Add --bundle-install option to reduce distributable package size

* Downloads libraries from internet if they are missing from the system
* Libraries are shared among all sibs projects as long as they use same
library versions

6 files changed, 292 insertions, 27 deletions

diff --git a/README.md b/README.md
index 967f3b8..c578468 100644
--- a/README.md
+++ b/README.md
@@ -42,6 +42,9 @@ Sibs supports creating a redistributable packages of projects (currently only on
 Currently a script file is generated which should be used to run the project. The name of the script file is the same as project. This script file will most likely to be removed later. Do NOT run the executable called "program".
 Because creating a package is currently done by copying c/c++ libraries and precompiled shared libraries on Linux usually depend on gcc runtime libraries which are very large, the distributable package becomes very large; a hello world application extracted from its archive is 6 megabytes...
 If you want to reduce the size of your package then you will have to compile your project and each dependency from source with clang/musl (gcc c++ runtime is 14mb while clang c++ runtime is 800kb!).
+
+The package command also comes with --bundle-install option which reduces the size of the distributable package by removing libraries in the package that can be downloaded online, and instead the user will download missing libraries when launching the application for the first time (the libraries are cached). This option is good because if the user already has the libraries installed on their system with a package managed then the user dont have to download the libraries and if the user has other software that was distributed using sibs, then their libraries will be shared with your projects; meaning if one project has a library of one version then it's shared with all software that uses same version of the library.
+Libraries that are downloaded are available at: https://github.com/DEC05EBA/libraries
 # IDE support
 Sibs generates a compile_commands.json in the project root directory when executing `sibs build` and tools that support clang completion can be used, such as YouCompleteMe.
 There are several editors that support YouCompleteMe, including Vim, Emacs and Visual Studio Code. Visual studio code now also supports clang completion with C/C++ extension by Microsoft; the extension will ask you which compile_commands.json file you want to use and you can choose the compile_commands.json in the project root directory.
diff --git a/include/Conf.hpp b/include/Conf.hpp
index d23c655..17ecc6b 100644
--- a/include/Conf.hpp
+++ b/include/Conf.hpp
@@ -503,6 +503,7 @@ namespace sibs
         bool zigTestAllFiles;
         bool packaging;
         bool bundling;
+        std::string version;
     protected:
         virtual void processObject(StringView name) override;
         virtual void processField(StringView name, const ConfigValue &value) override;
diff --git a/scripts/download_dependencies.sh b/scripts/download_dependencies.sh
new file mode 100755
index 0000000..8349068
--- /dev/null
+++ b/scripts/download_dependencies.sh
@@ -0,0 +1,124 @@
+#!/bin/bash
+
+if (( "$#" != 1 )); then
+    echo "usage: download_dependencies.sh <program_name>"
+    exit 1
+fi
+
+is_root=0
+if [ $(id -u) = 0 ]; then
+    is_root=1
+fi
+
+program_name="$1"
+script_path=`readlink -f $0`
+script_dir=`dirname $script_path`
+
+if [ -f /usr/lib/sibs/"$program_name".cache ]; then
+    #echo "No need to download dependencies, all dependencies exist in cache"
+    exit 0
+fi
+
+if [ $is_root -eq 0 ] && [ -f ~/.local/lib/sibs/"$program_name".cache ]; then
+    #echo "No need to download dependencies, all dependencies exist in cache"
+    exit 0
+fi
+
+command -v sha1sum > /dev/null || { echo "Missing program: sha1sum"; exit 1; }
+command -v wget > /dev/null || { echo "Missing program: wget"; exit 1; }
+
+set -e
+IFS=$'\n' GLOBIGNORE='*' command eval 'dependencies=($(cat "$script_dir"/dependencies.conf))'
+IFS=$'\n' GLOBIGNORE='*' command eval 'urls=($(cat "$script_dir"/urls.conf))'
+IFS=$'\n' GLOBIGNORE='*' command eval 'libmaps=($(cat "$script_dir"/libmap.conf))'
+
+if (( "${#dependencies[@]}" % 2 != 0 )); then
+    echo "Invalid number of arguments in dependencies.conf file. Expected multiple libraries which are the dependencies, followed by their checksum"
+    exit 4
+fi
+
+program_libs_dir=""
+if [ $is_root -eq 0 ]; then
+    mkdir -p ~/.local/lib/sibs
+    program_libs_dir=~/.local/lib/sibs/"$program_name"
+else
+    mkdir -p /usr/lib/sibs
+    program_libs_dir=/usr/lib/sibs/"$program_name"
+fi
+mkdir -p "$program_libs_dir"
+set +e
+
+for (( i=0; i<${#dependencies[@]}; i=$i+2 ))
+do
+    file=${dependencies[i]}
+    checksum_file="$file".sha1
+    checksum=${dependencies[i+1]}
+
+    if [ -f /usr/lib/sibs/"$file" ] && [ "$checksum"  == "$(cat /usr/lib/sibs/$checksum_file)" ]; then
+        #echo "Using sibs global lib file $file"
+    elif [ $is_root -eq 0 ] && [ -f ~/.local/lib/sibs/"$file" ] && [ "$checksum"  == "$(cat ~/.local/lib/sibs/$checksum_file)" ]; then
+        #echo "Using sibs user lib file $file"
+    elif [ -f /usr/lib/"$file" ] && echo "$checksum" /usr/lib/"$file" | sha1sum -c --status; then
+        #echo "Using system lib file $file"
+        set -e
+        if [ $is_root -eq 0 ]; then
+            cp /usr/lib/"$file" ~/.local/lib/sibs/"$file"
+            echo "$checksum" > ~/.local/lib/sibs/"$checksum_file"
+        else
+            cp /usr/lib/"$file" /usr/lib/sibs/"$file"
+            echo "$checksum" > /usr/lib/sibs/"$checksum_file"
+        fi
+        set +e
+    else
+        downloaded=0
+        for url in "${urls[@]}"; do
+            dst_dir=""
+            if [ $is_root -eq 0 ]; then
+                dst_dir=~/.local/lib/sibs
+            else
+                dst_dir=/usr/lib/sibs
+            fi
+
+            #echo "Downloading missing library from $url/$file"
+            if wget -q --show-progress -O "$dst_dir/$file" "$url/$file"; then
+                echo "$checksum" > "$dst_dir/$checksum_file"
+                downloaded=1
+                break
+            fi
+        done
+
+        if [ $downloaded -eq 0 ]; then
+            echo "Failed to download dependency $file from all urls"
+            exit 2
+        fi
+    fi
+done
+
+for (( i=0; i<${#libmaps[@]}; i=$i+2 ))
+do
+    src=${libmaps[i]}
+    dst=${libmaps[i+1]}
+
+    lib_file=""
+    if [ -f /usr/lib/sibs/"$src" ]; then
+        lib_file=/usr/lib/sibs/"$src"
+    elif [ $is_root -eq 0 ] && [ -f ~/.local/lib/sibs/"$src" ]; then
+        lib_file=~/.local/lib/sibs/"$src"
+    elif [ -f /usr/lib/"$src" ]; then
+        lib_file=/usr/lib/"$src"
+    fi
+
+    #echo "Creating symlink for lib file $dst"
+    ln -sf "$lib_file" "$program_libs_dir/$dst"
+    if [ $? -ne 0 ]; then
+        echo "Failed to create symlink for program"
+        exit 3
+    fi
+done
+
+set -e
+if [ $is_root -eq 0 ]; then
+    touch ~/.local/lib/sibs/"$program_name".cache
+else
+    touch /usr/lib/sibs/"$program_name".cache
+fi
\ No newline at end of file
diff --git a/scripts/package.py b/scripts/package.py
index f2841b8..3d2ae92 100755
--- a/scripts/package.py
+++ b/scripts/package.py
@@ -7,15 +7,19 @@ import shutil
 import re
 import stat
 import tarfile
+import requests
+import hashlib
 
-run_script_linux = """
-#!/bin/sh
+run_script_linux = """#!/bin/bash
 
 set -e
 
 script_path=`readlink -f $0`
 script_dir=`dirname $script_path`
-"$script_dir/$SO_LOADER" --library-path "$script_dir/libs" "$script_dir/$PROGRAM_NAME" "$@"
+
+program_full_name="$PROGRAM_FULL_NAME"
+$DOWNLOAD_DEPENDENCIES_COMMAND
+"$script_dir/$SO_LOADER" --library-path "$script_dir/libs":~/.local/lib/sibs/"$program_full_name":/usr/lib/sibs/"$program_full_name" "$script_dir/$PROGRAM_NAME" "$@"
 """
 
 def get_executable_dynamic_libraries(filepath):
@@ -45,13 +49,32 @@ def get_libnss_files():
             files.append(os.path.join("/usr/lib", filepath))
     return files
 
+def file_get_sha1(filepath):
+    with open(filepath, "rb") as f:
+        sha1 = hashlib.sha1()
+        sha1.update(f.read())
+        return sha1.hexdigest()
+    raise RuntimeError("No such file: %s" % filepath)
+
+def usage():
+    print("usage: package.py executable_path program_version destination_path <--bundle|--bundle-install>")
+    exit(1)
+
 def main():
-    if len(sys.argv) <= 2:
-        print("usage: %s executable_path destination_path" % sys.argv[0])
-        exit(1)
+    if len(sys.argv) <= 4:
+        usage()
 
-    os.makedirs(sys.argv[2], exist_ok=True)
-    libs = get_executable_dynamic_libraries(sys.argv[1])
+    script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
+    executable_path = sys.argv[1]
+    program_version = sys.argv[2]
+    destination_path = sys.argv[3]
+    package_type = sys.argv[4]
+
+    if package_type != "--bundle" and package_type != "--bundle-install":
+        usage()
+
+    os.makedirs(destination_path, exist_ok=True)
+    libs = get_executable_dynamic_libraries(executable_path)
 
     so_loader_pattern = re.compile("ld-linux-x86-64\\.so.*")
     so_loader = None
@@ -66,11 +89,34 @@ def main():
         print("Unexpected error: no so loader found, unable to recover")
         exit(5)
 
+    print("So loader: %s" % so_loader)
+
+    mapped_libs = []
+    dependencies = []
+    if package_type == "--bundle-install":
+        new_libs = []
+        # Remove libraries from the package that can be downloaded remotely (and which user most likely has if they have another program that uses sibs on their computer)
+        for lib in libs:
+            lib_name = os.path.basename(lib[0])
+            r = requests.get("https://raw.githubusercontent.com/DEC05EBA/libraries/master/linux/x86_64/" + lib_name + ".sha1")
+            # TODO: Remove check if it's so_loader or not, we can use so loader in sibs downloaded lib directory
+            if r.ok and lib[1] != so_loader:
+                external_checksum = r.text.splitlines()[0]
+                file_checksum = file_get_sha1(lib[0])
+                if external_checksum == file_checksum:
+                    mapped_libs.append([ lib_name, lib[1] ])
+                    dependencies.append([ lib_name, external_checksum ])
+                else:
+                    new_libs.append(lib)
+                print("Checksum for file %s: %s, equals? %s" % (lib_name, external_checksum, "yes" if external_checksum == file_checksum else "no"))
+            else:
+                new_libs.append(lib)
+        libs = new_libs
     so_loader = os.path.join("libs", so_loader)
 
     libnss_files = get_libnss_files()
     for idx, libnss_file in enumerate(libnss_files):
-        new_file_path = os.path.join(sys.argv[2], os.path.basename(libnss_file))
+        new_file_path = os.path.join(destination_path, os.path.basename(libnss_file))
         libnss_files[idx] = new_file_path
         if os.path.islink(libnss_file):
             target_name = os.path.basename(os.readlink(libnss_file))
@@ -78,9 +124,9 @@ def main():
         else:
             shutil.copyfile(libnss_file, new_file_path)
 
-    executable_filename = os.path.basename(sys.argv[1])
-    new_executable_path = os.path.join(sys.argv[2], executable_filename)
-    shutil.copyfile(sys.argv[1], new_executable_path)
+    executable_filename = os.path.basename(executable_path)
+    new_executable_path = os.path.join(destination_path, executable_filename)
+    shutil.copyfile(executable_path, new_executable_path)
     make_executable(new_executable_path)
 
     print("Patching executable")
@@ -90,17 +136,35 @@ def main():
         print("Failed to execute patchelf --set-interpreter on executable %s, error: %s" % (new_executable_path, stderr))
         exit(3)
     
-    process = subprocess.Popen(["patchelf", "--set-rpath", "libs", new_executable_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stdout, stderr) = process.communicate()
-    if process.returncode != 0:
-        print("Failed to execute patchelf --set-rpath on executable %s, error: %s" % (new_executable_path, stderr))
-        exit(4)
+    #process = subprocess.Popen(["patchelf", "--set-rpath", "libs", new_executable_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    #(stdout, stderr) = process.communicate()
+    #if process.returncode != 0:
+    #    print("Failed to execute patchelf --set-rpath on executable %s, error: %s" % (new_executable_path, stderr))
+    #    exit(4)
 
-    run_script_path = os.path.join(sys.argv[2], "run.sh")
+    run_script_path = os.path.join(destination_path, "run.sh")
     with open(run_script_path, "wb") as run_script:
-        run_script.write(run_script_linux.replace("$SO_LOADER", so_loader).replace("$PROGRAM_NAME", "program").encode("UTF-8"))
+        run_script.write(run_script_linux.replace("$SO_LOADER", so_loader)\
+            .replace("$PROGRAM_NAME", "program")\
+            .replace("$PROGRAM_FULL_NAME", executable_filename + "." + program_version)\
+            .replace("$DOWNLOAD_DEPENDENCIES_COMMAND", "\"$script_dir/download_dependencies.sh\" \"$program_full_name\"" if package_type == "--bundle-install" else "")\
+            .encode("UTF-8"))
     make_executable(run_script_path)
 
+    urls_filepath = os.path.join(destination_path, "urls.conf")
+    with open(urls_filepath, "wb") as urls_file:
+        urls_file.write("https://raw.githubusercontent.com/DEC05EBA/libraries/master/linux/x86_64/\n".encode("UTF-8"))
+
+    library_mapping_filepath = os.path.join(destination_path, "libmap.conf")
+    with open(library_mapping_filepath, "wb") as library_mapping_file:
+        for mapped_lib in mapped_libs:
+            library_mapping_file.write((mapped_lib[0] + "\n" + mapped_lib[1] + "\n").encode("UTF-8"))
+
+    dependencies_filepath = os.path.join(destination_path, "dependencies.conf")
+    with open(dependencies_filepath, "wb") as dependencies_file:
+        for dependency in dependencies:
+            dependencies_file.write((dependency[0] + "\n" + dependency[1] + "\n").encode("UTF-8"))
+
     package_name = new_executable_path + ".tar.gz"
     print("Creating archive %s" % os.path.basename(package_name))
     with tarfile.open(package_name, "w:gz") as tar:
@@ -113,14 +177,27 @@ def main():
             libnss_name = os.path.basename(libnss_file)
             tar.add(libnss_file, arcname=os.path.join("libs", libnss_name))
 
-        print("Adding executable %s to package" % sys.argv[1])
+        print("Adding executable %s to package" % executable_path)
         tar.add(new_executable_path, arcname="program")
         print("Adding run script %s to package" % run_script_path)
         tar.add(run_script_path, arcname=executable_filename)
+        if package_type == "--bundle-install":
+            print("Adding urls file %s to package" % urls_filepath)
+            tar.add(urls_filepath, arcname="urls.conf")
+            print("Adding library mapping file %s to package" % library_mapping_filepath)
+            tar.add(library_mapping_filepath, arcname="libmap.conf")
+            print("Adding dependencies file %s to package" % dependencies_filepath)
+            tar.add(dependencies_filepath, arcname="dependencies.conf")
+            download_dependencies_script_filepath = os.path.join(script_dir, "download_dependencies.sh")
+            print("Adding download dependencies script file %s to package" % download_dependencies_script_filepath)
+            tar.add(download_dependencies_script_filepath, arcname="download_dependencies.sh")
     
     print("Removing temporary files")
     os.remove(new_executable_path)
     os.remove(run_script_path)
+    os.remove(urls_filepath)
+    os.remove(library_mapping_filepath)
+    os.remove(dependencies_filepath)
     for libnss_file in libnss_files:
         os.remove(libnss_file)
     print("Package has been created at %s" % package_name)
diff --git a/src/Conf.cpp b/src/Conf.cpp
index fa92bf5..aabb957 100644
--- a/src/Conf.cpp
+++ b/src/Conf.cpp
@@ -461,12 +461,30 @@ namespace sibs
         // Do not free file content (fileContentResult) on purpose, since we are using the data and sibs is short lived
         Result<bool> parseResult = Parser::parse(code, config);
         if(!parseResult)
-            return Result<bool>::Err("Failed to read config, reason: " + parseResult.getErrMsg());
+        {
+            string errMsg = "Failed while parsing project.conf for project ";
+            errMsg += config.isTest() ? "tests" : config.getPackageName();
+            errMsg += ", reason: " + parseResult.getErrMsg();
+            return Result<bool>::Err(errMsg);
+        }
 
         if(!config.isTest())
         {
             if(config.getPackageName().empty())
-                return Result<bool>::Err("project.conf is missing required field package.name");
+            {
+                string errMsg = "The project ";
+                errMsg += config.getPackageName();
+                errMsg += " is missing required field package.name is project.conf";
+                return Result<bool>::Err(errMsg);
+            }
+
+            if(config.version.empty())
+            {
+                string errMsg = "The project ";
+                errMsg += config.getPackageName();
+                errMsg += " is missing required field package.version is project.conf";
+                return Result<bool>::Err(errMsg);
+            }
 
             if (!containsPlatform(config.getPlatforms(), SYSTEM_PLATFORM))
             {
@@ -668,6 +686,17 @@ namespace sibs
         return result;
     }
 
+    static bool isVersionStringValid(const string &version)
+    {
+        for(char c : version)
+        {
+            bool isValidChar = (c == '.' || (c >= '0' && c <= '9'));
+            if(!isValidChar)
+                return false;
+        }
+        return true;
+    }
+
     void SibsConfig::processObject(StringView name)
     {
         currentObject = name;
@@ -713,7 +742,13 @@ namespace sibs
             }
             else if(name.equals("version"))
             {
-                // TODO: Use version for info output when building
+                if (value.isSingle())
+                    version = string(value.asSingle().data, value.asSingle().size);
+                else
+                    throw ParserException("Expected package.version to be a single value, was a list");
+
+                if(!isVersionStringValid(version))
+                    throw ParserException("package.version is in invalid format. Version string can only contain numbers and dots");
             }
             else if(name.equals("authors"))
             {
diff --git a/src/main.cpp b/src/main.cpp
index 6a807f0..e55014e 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -184,12 +184,14 @@ static void usageInit()
 
 static void usagePackage()
 {
-    printf("Usage: sibs package [project_path] <--static|--bundle>\n\n");
+    printf("Usage: sibs package [project_path] <--static|--bundle|--bundle-install>\n\n");
     printf("Create a redistributable package from a sibs project. Note: Redistributable packages can't use system packages to build if packaging using --static\n\n");
     printf("Options:\n");
     printf("  project_path\t\tThe directory containiung a project.conf file - Optional (default: current directory)\n");
     printf("  --static\t\tPackage project by building everything statically. Note: can't use system packages when using this option (meaning no pkg-config support)\n\n");
     printf("  --bundle\t\tPackage project by copying all dynamic libraries into one location and creating an archive of all files. The executable is patched to use the dynamic libraries in the same directory. Note: if your project loads dynamic libraries at runtime (for example using dlopen) then you need to manually copy those libraries to the archive\n\n");
+    printf("  --bundle-install\t\tPackage project by copying all dynamic libraries into one location, except libraries that can automatically be downloaded online by the user. Then create an archive of all files - Use this option if you want to reduce the size of the distributed package and also if user already has some of the libraries installed/downloaded on their system, then they are used."
+            "Note: if your project loads dynamic libraries at runtime (for example using dlopen) then you need to manually copy those libraries to the archive");
     printf("Examples:\n");
     printf("  sibs package --static\n");
     printf("  sibs package dirA/dirB --bundle\n");
@@ -937,7 +939,8 @@ enum class PackagingType
 {
     NONE,
     STATIC,
-    BUNDLE
+    BUNDLE,
+    BUNDLE_INSTALL
 };
 
 static const char* asString(PackagingType packagingType)
@@ -1025,6 +1028,15 @@ static int packageProject(int argc, const _tinydir_char_t **argv)
             }
             packagingType = PackagingType::BUNDLE;
         }
+        else if(_tinydir_strcmp(arg, TINYDIR_STRING("--bundle-install")) == 0)
+        {
+            if(packagingType != PackagingType::NONE)
+            {
+                ferr << "Error: Project packaging type was defined more than once. First as " << asString(packagingType) << " then as " << "bundle-install" << endl;
+                usagePackage();
+            }
+            packagingType = PackagingType::BUNDLE_INSTALL;
+        }
         else
         {
             if(!projectPath.empty())
@@ -1073,7 +1085,7 @@ static int packageProject(int argc, const _tinydir_char_t **argv)
     SibsConfig sibsConfig(compiler, projectPath, OPT_LEV_RELEASE, false);
     sibsConfig.showWarnings = true;
     sibsConfig.packaging = packagingType == PackagingType::STATIC;
-    sibsConfig.bundling = packagingType == PackagingType::BUNDLE;
+    sibsConfig.bundling = (packagingType == PackagingType::BUNDLE) || (packagingType == PackagingType::BUNDLE_INSTALL);
     int result = buildProject(projectPath, projectConfFilePath, sibsConfig);
     if(result != 0)
         return result;
@@ -1087,11 +1099,24 @@ static int packageProject(int argc, const _tinydir_char_t **argv)
             break;
         }
         case PackagingType::BUNDLE:
+        case PackagingType::BUNDLE_INSTALL:
         {
+            const char *bundleType = nullptr;
+            switch(packagingType)
+            {
+                case PackagingType::BUNDLE:
+                    bundleType = "--bundle";
+                    break;
+                case PackagingType::BUNDLE_INSTALL:
+                    bundleType = "--bundle-install";
+                    break;
+            }
+
             string packagePath = toUtf8(projectPath + TINYDIR_STRING("/sibs-build/package"));
             string executablePath = toUtf8(projectPath + TINYDIR_STRING("/sibs-build/release/") + sibsConfig.getPackageName());
             printf("Creating a package from project and dependencies...\n");
-            FileString cmd = "python3 \"" + packageScriptPath + "\" \"" + executablePath + "\" \"" + packagePath + "\"";
+            // args: executable_path program_version destination_path <--bundle|--bundle-install>
+            FileString cmd = "python3 \"" + packageScriptPath + "\" \"" + executablePath + "\" \"" + sibsConfig.version + "\" \"" + packagePath + "\" " + bundleType;
             Result<ExecResult> bundleResult = exec(cmd.c_str(), true);
             if(!bundleResult)
             {