Add --bundle-install option to reduce distributable package size

* Downloads libraries from internet if they are missing from the system
* Libraries are shared among all sibs projects as long as they use same
library versions

2 files changed, 221 insertions, 20 deletions

diff --git a/scripts/download_dependencies.sh b/scripts/download_dependencies.sh
new file mode 100755
index 0000000..8349068
--- /dev/null
+++ b/scripts/download_dependencies.sh
@@ -0,0 +1,124 @@
+#!/bin/bash
+
+if (( "$#" != 1 )); then
+    echo "usage: download_dependencies.sh <program_name>"
+    exit 1
+fi
+
+is_root=0
+if [ $(id -u) = 0 ]; then
+    is_root=1
+fi
+
+program_name="$1"
+script_path=`readlink -f $0`
+script_dir=`dirname $script_path`
+
+if [ -f /usr/lib/sibs/"$program_name".cache ]; then
+    #echo "No need to download dependencies, all dependencies exist in cache"
+    exit 0
+fi
+
+if [ $is_root -eq 0 ] && [ -f ~/.local/lib/sibs/"$program_name".cache ]; then
+    #echo "No need to download dependencies, all dependencies exist in cache"
+    exit 0
+fi
+
+command -v sha1sum > /dev/null || { echo "Missing program: sha1sum"; exit 1; }
+command -v wget > /dev/null || { echo "Missing program: wget"; exit 1; }
+
+set -e
+IFS=$'\n' GLOBIGNORE='*' command eval 'dependencies=($(cat "$script_dir"/dependencies.conf))'
+IFS=$'\n' GLOBIGNORE='*' command eval 'urls=($(cat "$script_dir"/urls.conf))'
+IFS=$'\n' GLOBIGNORE='*' command eval 'libmaps=($(cat "$script_dir"/libmap.conf))'
+
+if (( "${#dependencies[@]}" % 2 != 0 )); then
+    echo "Invalid number of arguments in dependencies.conf file. Expected multiple libraries which are the dependencies, followed by their checksum"
+    exit 4
+fi
+
+program_libs_dir=""
+if [ $is_root -eq 0 ]; then
+    mkdir -p ~/.local/lib/sibs
+    program_libs_dir=~/.local/lib/sibs/"$program_name"
+else
+    mkdir -p /usr/lib/sibs
+    program_libs_dir=/usr/lib/sibs/"$program_name"
+fi
+mkdir -p "$program_libs_dir"
+set +e
+
+for (( i=0; i<${#dependencies[@]}; i=$i+2 ))
+do
+    file=${dependencies[i]}
+    checksum_file="$file".sha1
+    checksum=${dependencies[i+1]}
+
+    if [ -f /usr/lib/sibs/"$file" ] && [ "$checksum"  == "$(cat /usr/lib/sibs/$checksum_file)" ]; then
+        #echo "Using sibs global lib file $file"
+    elif [ $is_root -eq 0 ] && [ -f ~/.local/lib/sibs/"$file" ] && [ "$checksum"  == "$(cat ~/.local/lib/sibs/$checksum_file)" ]; then
+        #echo "Using sibs user lib file $file"
+    elif [ -f /usr/lib/"$file" ] && echo "$checksum" /usr/lib/"$file" | sha1sum -c --status; then
+        #echo "Using system lib file $file"
+        set -e
+        if [ $is_root -eq 0 ]; then
+            cp /usr/lib/"$file" ~/.local/lib/sibs/"$file"
+            echo "$checksum" > ~/.local/lib/sibs/"$checksum_file"
+        else
+            cp /usr/lib/"$file" /usr/lib/sibs/"$file"
+            echo "$checksum" > /usr/lib/sibs/"$checksum_file"
+        fi
+        set +e
+    else
+        downloaded=0
+        for url in "${urls[@]}"; do
+            dst_dir=""
+            if [ $is_root -eq 0 ]; then
+                dst_dir=~/.local/lib/sibs
+            else
+                dst_dir=/usr/lib/sibs
+            fi
+
+            #echo "Downloading missing library from $url/$file"
+            if wget -q --show-progress -O "$dst_dir/$file" "$url/$file"; then
+                echo "$checksum" > "$dst_dir/$checksum_file"
+                downloaded=1
+                break
+            fi
+        done
+
+        if [ $downloaded -eq 0 ]; then
+            echo "Failed to download dependency $file from all urls"
+            exit 2
+        fi
+    fi
+done
+
+for (( i=0; i<${#libmaps[@]}; i=$i+2 ))
+do
+    src=${libmaps[i]}
+    dst=${libmaps[i+1]}
+
+    lib_file=""
+    if [ -f /usr/lib/sibs/"$src" ]; then
+        lib_file=/usr/lib/sibs/"$src"
+    elif [ $is_root -eq 0 ] && [ -f ~/.local/lib/sibs/"$src" ]; then
+        lib_file=~/.local/lib/sibs/"$src"
+    elif [ -f /usr/lib/"$src" ]; then
+        lib_file=/usr/lib/"$src"
+    fi
+
+    #echo "Creating symlink for lib file $dst"
+    ln -sf "$lib_file" "$program_libs_dir/$dst"
+    if [ $? -ne 0 ]; then
+        echo "Failed to create symlink for program"
+        exit 3
+    fi
+done
+
+set -e
+if [ $is_root -eq 0 ]; then
+    touch ~/.local/lib/sibs/"$program_name".cache
+else
+    touch /usr/lib/sibs/"$program_name".cache
+fi
\ No newline at end of file
diff --git a/scripts/package.py b/scripts/package.py
index f2841b8..3d2ae92 100755
--- a/scripts/package.py
+++ b/scripts/package.py
@@ -7,15 +7,19 @@ import shutil
 import re
 import stat
 import tarfile
+import requests
+import hashlib
 
-run_script_linux = """
-#!/bin/sh
+run_script_linux = """#!/bin/bash
 
 set -e
 
 script_path=`readlink -f $0`
 script_dir=`dirname $script_path`
-"$script_dir/$SO_LOADER" --library-path "$script_dir/libs" "$script_dir/$PROGRAM_NAME" "$@"
+
+program_full_name="$PROGRAM_FULL_NAME"
+$DOWNLOAD_DEPENDENCIES_COMMAND
+"$script_dir/$SO_LOADER" --library-path "$script_dir/libs":~/.local/lib/sibs/"$program_full_name":/usr/lib/sibs/"$program_full_name" "$script_dir/$PROGRAM_NAME" "$@"
 """
 
 def get_executable_dynamic_libraries(filepath):
@@ -45,13 +49,32 @@ def get_libnss_files():
             files.append(os.path.join("/usr/lib", filepath))
     return files
 
+def file_get_sha1(filepath):
+    with open(filepath, "rb") as f:
+        sha1 = hashlib.sha1()
+        sha1.update(f.read())
+        return sha1.hexdigest()
+    raise RuntimeError("No such file: %s" % filepath)
+
+def usage():
+    print("usage: package.py executable_path program_version destination_path <--bundle|--bundle-install>")
+    exit(1)
+
 def main():
-    if len(sys.argv) <= 2:
-        print("usage: %s executable_path destination_path" % sys.argv[0])
-        exit(1)
+    if len(sys.argv) <= 4:
+        usage()
 
-    os.makedirs(sys.argv[2], exist_ok=True)
-    libs = get_executable_dynamic_libraries(sys.argv[1])
+    script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
+    executable_path = sys.argv[1]
+    program_version = sys.argv[2]
+    destination_path = sys.argv[3]
+    package_type = sys.argv[4]
+
+    if package_type != "--bundle" and package_type != "--bundle-install":
+        usage()
+
+    os.makedirs(destination_path, exist_ok=True)
+    libs = get_executable_dynamic_libraries(executable_path)
 
     so_loader_pattern = re.compile("ld-linux-x86-64\\.so.*")
     so_loader = None
@@ -66,11 +89,34 @@ def main():
         print("Unexpected error: no so loader found, unable to recover")
         exit(5)
 
+    print("So loader: %s" % so_loader)
+
+    mapped_libs = []
+    dependencies = []
+    if package_type == "--bundle-install":
+        new_libs = []
+        # Remove libraries from the package that can be downloaded remotely (and which user most likely has if they have another program that uses sibs on their computer)
+        for lib in libs:
+            lib_name = os.path.basename(lib[0])
+            r = requests.get("https://raw.githubusercontent.com/DEC05EBA/libraries/master/linux/x86_64/" + lib_name + ".sha1")
+            # TODO: Remove check if it's so_loader or not, we can use so loader in sibs downloaded lib directory
+            if r.ok and lib[1] != so_loader:
+                external_checksum = r.text.splitlines()[0]
+                file_checksum = file_get_sha1(lib[0])
+                if external_checksum == file_checksum:
+                    mapped_libs.append([ lib_name, lib[1] ])
+                    dependencies.append([ lib_name, external_checksum ])
+                else:
+                    new_libs.append(lib)
+                print("Checksum for file %s: %s, equals? %s" % (lib_name, external_checksum, "yes" if external_checksum == file_checksum else "no"))
+            else:
+                new_libs.append(lib)
+        libs = new_libs
     so_loader = os.path.join("libs", so_loader)
 
     libnss_files = get_libnss_files()
     for idx, libnss_file in enumerate(libnss_files):
-        new_file_path = os.path.join(sys.argv[2], os.path.basename(libnss_file))
+        new_file_path = os.path.join(destination_path, os.path.basename(libnss_file))
         libnss_files[idx] = new_file_path
         if os.path.islink(libnss_file):
             target_name = os.path.basename(os.readlink(libnss_file))
@@ -78,9 +124,9 @@ def main():
         else:
             shutil.copyfile(libnss_file, new_file_path)
 
-    executable_filename = os.path.basename(sys.argv[1])
-    new_executable_path = os.path.join(sys.argv[2], executable_filename)
-    shutil.copyfile(sys.argv[1], new_executable_path)
+    executable_filename = os.path.basename(executable_path)
+    new_executable_path = os.path.join(destination_path, executable_filename)
+    shutil.copyfile(executable_path, new_executable_path)
     make_executable(new_executable_path)
 
     print("Patching executable")
@@ -90,17 +136,35 @@ def main():
         print("Failed to execute patchelf --set-interpreter on executable %s, error: %s" % (new_executable_path, stderr))
         exit(3)
     
-    process = subprocess.Popen(["patchelf", "--set-rpath", "libs", new_executable_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stdout, stderr) = process.communicate()
-    if process.returncode != 0:
-        print("Failed to execute patchelf --set-rpath on executable %s, error: %s" % (new_executable_path, stderr))
-        exit(4)
+    #process = subprocess.Popen(["patchelf", "--set-rpath", "libs", new_executable_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    #(stdout, stderr) = process.communicate()
+    #if process.returncode != 0:
+    #    print("Failed to execute patchelf --set-rpath on executable %s, error: %s" % (new_executable_path, stderr))
+    #    exit(4)
 
-    run_script_path = os.path.join(sys.argv[2], "run.sh")
+    run_script_path = os.path.join(destination_path, "run.sh")
     with open(run_script_path, "wb") as run_script:
-        run_script.write(run_script_linux.replace("$SO_LOADER", so_loader).replace("$PROGRAM_NAME", "program").encode("UTF-8"))
+        run_script.write(run_script_linux.replace("$SO_LOADER", so_loader)\
+            .replace("$PROGRAM_NAME", "program")\
+            .replace("$PROGRAM_FULL_NAME", executable_filename + "." + program_version)\
+            .replace("$DOWNLOAD_DEPENDENCIES_COMMAND", "\"$script_dir/download_dependencies.sh\" \"$program_full_name\"" if package_type == "--bundle-install" else "")\
+            .encode("UTF-8"))
     make_executable(run_script_path)
 
+    urls_filepath = os.path.join(destination_path, "urls.conf")
+    with open(urls_filepath, "wb") as urls_file:
+        urls_file.write("https://raw.githubusercontent.com/DEC05EBA/libraries/master/linux/x86_64/\n".encode("UTF-8"))
+
+    library_mapping_filepath = os.path.join(destination_path, "libmap.conf")
+    with open(library_mapping_filepath, "wb") as library_mapping_file:
+        for mapped_lib in mapped_libs:
+            library_mapping_file.write((mapped_lib[0] + "\n" + mapped_lib[1] + "\n").encode("UTF-8"))
+
+    dependencies_filepath = os.path.join(destination_path, "dependencies.conf")
+    with open(dependencies_filepath, "wb") as dependencies_file:
+        for dependency in dependencies:
+            dependencies_file.write((dependency[0] + "\n" + dependency[1] + "\n").encode("UTF-8"))
+
     package_name = new_executable_path + ".tar.gz"
     print("Creating archive %s" % os.path.basename(package_name))
     with tarfile.open(package_name, "w:gz") as tar:
@@ -113,14 +177,27 @@ def main():
             libnss_name = os.path.basename(libnss_file)
             tar.add(libnss_file, arcname=os.path.join("libs", libnss_name))
 
-        print("Adding executable %s to package" % sys.argv[1])
+        print("Adding executable %s to package" % executable_path)
         tar.add(new_executable_path, arcname="program")
         print("Adding run script %s to package" % run_script_path)
         tar.add(run_script_path, arcname=executable_filename)
+        if package_type == "--bundle-install":
+            print("Adding urls file %s to package" % urls_filepath)
+            tar.add(urls_filepath, arcname="urls.conf")
+            print("Adding library mapping file %s to package" % library_mapping_filepath)
+            tar.add(library_mapping_filepath, arcname="libmap.conf")
+            print("Adding dependencies file %s to package" % dependencies_filepath)
+            tar.add(dependencies_filepath, arcname="dependencies.conf")
+            download_dependencies_script_filepath = os.path.join(script_dir, "download_dependencies.sh")
+            print("Adding download dependencies script file %s to package" % download_dependencies_script_filepath)
+            tar.add(download_dependencies_script_filepath, arcname="download_dependencies.sh")
     
     print("Removing temporary files")
     os.remove(new_executable_path)
     os.remove(run_script_path)
+    os.remove(urls_filepath)
+    os.remove(library_mapping_filepath)
+    os.remove(dependencies_filepath)
     for libnss_file in libnss_files:
         os.remove(libnss_file)
     print("Package has been created at %s" % package_name)