From 7cfe40b987bf08053487f4a72fafe11de219b583 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Thu, 4 Oct 2018 02:43:30 +0200 Subject: Add package lib blacklist, remove patchelf dependency Some libraries such as gpu driver libraries cant be distributed because even though the program is run on the same cpu arch, gpu can differ. Removed dependency on patchelf, as it's not needed when executing ld-so directly with --library-path --- README.md | 5 ++++- scripts/package.py | 31 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c578468..1b24b35 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,15 @@ Ccache is currently required on non-windows platforms but it will later be remov # Usage After you have installed sibs, execute `sibs` without any arguments and you will get a list of commands and description for them. # Package -Sibs supports creating a redistributable packages of projects (currently only on Linux, run `sibs package --bundle`). Packaging is in testing phase and may not work for all projects. Currently you need to have python3, ldd and patchelf installed and also set the environment variable SIBS_SCRIPT_DIR to scripts sub directory which is in sibs root directory (the directory that contains package.py). +Sibs supports creating a redistributable packages of projects (currently only on Linux, run `sibs package --bundle`). Packaging is in testing phase and may not work for all projects. Currently you need to have python3 and ldd installed and also set the environment variable SIBS_SCRIPT_DIR to scripts sub directory which is in sibs root directory (the directory that contains package.py). Currently a script file is generated which should be used to run the project. The name of the script file is the same as project. This script file will most likely to be removed later. Do NOT run the executable called "program". Because creating a package is currently done by copying c/c++ libraries and precompiled shared libraries on Linux usually depend on gcc runtime libraries which are very large, the distributable package becomes very large; a hello world application extracted from its archive is 6 megabytes... If you want to reduce the size of your package then you will have to compile your project and each dependency from source with clang/musl (gcc c++ runtime is 14mb while clang c++ runtime is 800kb!). The package command also comes with --bundle-install option which reduces the size of the distributable package by removing libraries in the package that can be downloaded online, and instead the user will download missing libraries when launching the application for the first time (the libraries are cached). This option is good because if the user already has the libraries installed on their system with a package managed then the user dont have to download the libraries and if the user has other software that was distributed using sibs, then their libraries will be shared with your projects; meaning if one project has a library of one version then it's shared with all software that uses same version of the library. + +Users are required to manually install some libraries as they can't be included in a distributed package (install with their package manager). These libraries are commonly gpu driver libraries, which vary even if you have the same cpu architecture. +This requirement might be removed later, if the gpu driver libraries required can somehow be detected and downloaded cross platform. Libraries that are downloaded are available at: https://github.com/DEC05EBA/libraries # IDE support Sibs generates a compile_commands.json in the project root directory when executing `sibs build` and tools that support clang completion can be used, such as YouCompleteMe. diff --git a/scripts/package.py b/scripts/package.py index 61f12c7..ce7eb71 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -22,6 +22,8 @@ $DOWNLOAD_DEPENDENCIES_COMMAND "$script_dir/$SO_LOADER" --library-path "$script_dir/libs":$HOME/.local/lib/sibs/"$program_full_name":/usr/lib/sibs/"$program_full_name" "$script_dir/$PROGRAM_NAME" "$@" """ +blacklisted_libs = [ "libGL\\.so.*", "libGLX.*", "libGLdispatch.*" ] + def get_executable_dynamic_libraries(filepath): libs = [] process = subprocess.Popen(["ldd", filepath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -76,6 +78,18 @@ def fetch_checksums(url): return checksums +def remove_blacklisted_libs(libs): + new_libs = [] + for lib in libs: + is_blacklisted = False + for blacklisted_lib in blacklisted_libs: + if re.match(blacklisted_lib, os.path.basename(lib[0])): + is_blacklisted = True + break + if not is_blacklisted: + new_libs.append(lib) + return new_libs + def main(): if len(sys.argv) <= 4: usage() @@ -92,6 +106,10 @@ def main(): os.makedirs(destination_path, exist_ok=True) libs = get_executable_dynamic_libraries(executable_path) + # Some libraries can't be distributed because different hardware. Since we are operating on a specific architecture, the types of libraries + # that can't be distributed are commonly gpu driver libraries + libs = remove_blacklisted_libs(libs) + so_loader_pattern = re.compile("ld-linux-x86-64\\.so.*") so_loader = None for lib in libs: @@ -147,19 +165,6 @@ def main(): shutil.copyfile(executable_path, new_executable_path) make_executable(new_executable_path) - print("Patching executable") - process = subprocess.Popen(["patchelf", "--set-interpreter", so_loader, new_executable_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = process.communicate() - if process.returncode != 0: - print("Failed to execute patchelf --set-interpreter on executable %s, error: %s" % (new_executable_path, stderr)) - exit(3) - - #process = subprocess.Popen(["patchelf", "--set-rpath", "libs", new_executable_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - #(stdout, stderr) = process.communicate() - #if process.returncode != 0: - # print("Failed to execute patchelf --set-rpath on executable %s, error: %s" % (new_executable_path, stderr)) - # exit(4) - run_script_path = os.path.join(destination_path, "run.sh") with open(run_script_path, "wb") as run_script: run_script.write(run_script_linux.replace("$SO_LOADER", so_loader)\ -- cgit v1.2.3