#include #include "Ninja.hpp" #include "../../include/FileUtil.hpp" #include "../../include/Exec.hpp" #include "../../include/PkgConfig.hpp" #include "../../include/GlobalLib.hpp" using namespace std; using namespace sibs; namespace backend { string join(const vector &list, const char *joinStr) { if(list.empty()) return ""; string result; long stringSize = 0; long joinStrLen = strlen(joinStr); int i = 0; for(const string &str : list) { stringSize += str.size(); if(!str.empty() && i > 0) stringSize += joinStrLen; ++i; } result.reserve(stringSize); i = 0; for(const string &str : list) { if(i > 0) result += joinStr; result += str; ++i; } return result; } Ninja::LibraryType getNinjaLibraryType(PackageType packageType) { switch(packageType) { case PackageType::EXECUTABLE: return Ninja::LibraryType::EXECUTABLE; case PackageType::STATIC: return Ninja::LibraryType::STATIC; case PackageType::DYNAMIC: case PackageType::LIBRARY: return Ninja::LibraryType::DYNAMIC; } } Ninja::Ninja() { } void Ninja::addSourceFile(const char *filepath) { string filePathStr = filepath ? filepath : ""; if(filepath && !containsSourceFile(filePathStr)) { sourceFiles.emplace_back(filePathStr); printf("Adding source file: %s\n", filepath); } } void Ninja::addTestSourceDir(const char *dir) { string dirStr = dir ? dir : ""; if(dir && !containsTestSourceDir(dirStr)) { testSourceDirs.emplace_back(dirStr); printf("Adding test source directory: %s\n", dir); } } void Ninja::addDependency(const std::string &binaryFile) { if(!containsDependency(binaryFile)) binaryDependencies.emplace_back(binaryFile); } const std::vector& Ninja::getSourceFiles() const { return sourceFiles; } bool Ninja::containsSourceFile(const string &filepath) const { for(const string &sourceFile : sourceFiles) { if(sourceFile == filepath) return true; } return false; } bool Ninja::containsTestSourceDir(const string &dir) const { for(const string &testSourceDir : testSourceDirs) { if(testSourceDir == dir) return true; } return false; } bool Ninja::containsDependency(const string &dependency) const { for(const string &binaryDependency : binaryDependencies) { if(binaryDependency == dependency) return true; } return false; } Result validatePkgConfigPackageVersionExists(const Dependency &dependency) { Result dependencyValidationResult = PkgConfig::validatePackageExists(dependency.name); if(dependencyValidationResult.isErr()) return Result::Err(dependencyValidationResult.getErrMsg()); Result dependencyVersionValidationResult = PkgConfig::validatePackageVersionAtLeast(dependency.name, dependency.version); if(dependencyVersionValidationResult.isErr()) return Result::Err(dependencyVersionValidationResult.getErrMsg()); return Result::Ok(true); } // TODO: First check if pkg-config is installed. If it's not, only check dependencies that exists in the dependencies sub directory. // If pkg-config is installed and dependency is not installed, check in dependencies sub directory. Result Ninja::getLinkerFlags(const SibsConfig &config, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallback) const { const vector &dependencies = config.getDependencies(); if(dependencies.empty()) return Result::Ok(true); string globalLibDir = getHomeDir(); globalLibDir += "/.sibs/lib"; Result createGlobalLibDirResult = createDirectoryRecursive(globalLibDir.c_str()); if(createGlobalLibDirResult.isErr()) return createGlobalLibDirResult; vector pkgConfigDependencies; vector globalLibDependencies; for(const Dependency &dependency : dependencies) { Result pkgConfigDependencyValidation = validatePkgConfigPackageVersionExists(dependency); if(pkgConfigDependencyValidation.isOk()) { pkgConfigDependencies.push_back(dependency); } else { globalLibDependencies.push_back(dependency); } } Result pkgConfigLinkerFlagsResult = PkgConfig::getDynamicLibsLinkerFlags(pkgConfigDependencies); if(pkgConfigLinkerFlagsResult.isErr()) { printf("%s, using global lib...\n", pkgConfigLinkerFlagsResult.getErrMsg().c_str()); globalLibDependencies.reserve(globalLibDependencies.size() + pkgConfigDependencies.size()); for(const Dependency &pkgConfigDependency : pkgConfigDependencies) { globalLibDependencies.push_back(pkgConfigDependency); } pkgConfigDependencies.clear(); } else { if(!pkgConfigLinkerFlagsResult.unwrap().empty()) dynamicLinkerFlagCallback(pkgConfigLinkerFlagsResult.unwrap()); } for(const Dependency &globalLibDependency : globalLibDependencies) { printf("Dependency %s is missing from pkg-config, trying global lib\n", globalLibDependency.name.c_str()); Result globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(config, globalLibDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallback); if(globalLibLinkerFlagsResult.isErr()) { if(globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_NOT_FOUND || globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_VERSION_NO_MATCH) { printf("Dependency not found in global lib, trying to download from github\n"); // TODO: Download several dependencies at the same time by adding them to a list // and then iterate them and download them all using several threads. // All dependecies should be downloaded at the same time, this includes dependencies of dependencies. // If a dependency is missing, fail build BEFORE downloading dependencies and before compiling anything. // You do not want to possibly wait several minutes only for build to fail when there is no compilation error. // TODO: If return error is invalid url, then the message should be converted to // invalid package name/version. A check should be done if it is the name or version // that is invalid. Result downloadDependencyResult = GlobalLib::downloadDependency(globalLibDependency); if(downloadDependencyResult.isErr()) return downloadDependencyResult; globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(config, globalLibDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallback); if(globalLibLinkerFlagsResult.isErr()) return Result::Err(globalLibLinkerFlagsResult); } else { return Result::Err(globalLibLinkerFlagsResult); } } } return Result::Ok(true); } Result Ninja::build(const SibsConfig &config, const char *savePath, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallback) { // TODO: Do not quit here if no source files are provided. The source-less project could have dependencies if(sourceFiles.empty()) return Result::Err("No source files provided"); Result createBuildDirResult = createDirectoryRecursive(savePath); if(createBuildDirResult.isErr()) return createBuildDirResult; LibraryType libraryType = getNinjaLibraryType(config.getPackageType()); string ninjaBuildFilePath = savePath; ninjaBuildFilePath += "/build.ninja"; string result; result.reserve(16384); string globalIncDir = getHomeDir(); globalIncDir += "/.sibs/lib"; result += "globalIncDir = '-I"; result += globalIncDir; result += "'"; for(const auto &includeDir : config.getIncludeDirs()) { result += " '-I../../"; result += includeDir; result += "'"; } result += "\n\n"; string defines; for(const auto &definePair : config.getDefines()) { defines += " '-D"; defines += definePair.first; defines += "="; defines += definePair.second; defines += "'"; } if(!defines.empty()) printf("Custom define: %s\n", defines.c_str()); string buildJob; switch(libraryType) { case LibraryType::EXECUTABLE: { result += "rule cpp_COMPILER\n"; result += " command = ccache c++ $ARGS -c $in -o $out\n\n"; result += "rule cpp_BUILD_EXEC\n"; result += " command = ccache c++ $ARGS -o $out $in $LINK_ARGS $aliasing\n\n"; buildJob = "cpp_BUILD_EXEC"; break; } case LibraryType::STATIC: { result += "rule cpp_COMPILER\n"; result += " command = ccache c++ $ARGS -c -fPIC $in -o $out\n\n"; result += "rule cpp_BUILD_STATIC\n"; result += " command = ar rcs lib"; result += config.getPackageName(); result += ".a"; result += " $in\n\n"; buildJob = "cpp_BUILD_STATIC"; break; } case LibraryType::DYNAMIC: { result += "rule cpp_COMPILER\n"; result += " command = ccache c++ $ARGS -c -fPIC $in -o $out\n\n"; // --whole-archive result += "rule cpp_BUILD_DYNAMIC\n"; result += " command = ccache c++ $in -shared -o $out $LINK_ARGS $aliasing\n\n"; buildJob = "cpp_BUILD_DYNAMIC"; break; } default: assert(false); return Result::Err("Unexpected error"); } string optimizationFlags; switch(config.getOptimizationLevel()) { case OPT_LEV_DEBUG: optimizationFlags = "'-O0'"; break; case OPT_LEV_RELEASE: optimizationFlags = "'-O3' '-DNDEBUG'"; break; } vector objectNames; objectNames.reserve(sourceFiles.size()); for(const string &sourceFile : sourceFiles) { //string sourceFileEncoded = sourceFile; //replace(sourceFileEncoded, '/', '@'); string objectName = config.getPackageName() + "@exe/" + sourceFile + ".o"; result += "build "; result += objectName; result += ": cpp_COMPILER ../../"; result += sourceFile; result += "\n"; result += " ARGS = $globalIncDir '-I" + config.getPackageName() + "@exe' '-I..' "; if(!defines.empty()) result += defines; result += " '-fdiagnostics-color=always' '-pipe' '-D_FILE_OFFSET_BITS=64' '-Wall' '-Winvalid-pch' '-Wnon-virtual-dtor' " + optimizationFlags + " '-g'\n\n"; objectNames.emplace_back(objectName); } // TODO: Allow configuring default linking flags. Maybe have `package.useThreads = false` to disable this flag string allLinkerFlags = "-pthread"; // TODO: Somehow check loading order, because it has to be correct to work.. Or does it for dynamic libraries? // Anyways it's required for static libraries (especially on Windows) for(const string &binaryDependency : binaryDependencies) { allLinkerFlags += " "; allLinkerFlags += binaryDependency; } if(!staticLinkerFlagCallbackFunc || libraryType == LibraryType::DYNAMIC) { staticLinkerFlagCallbackFunc = [&allLinkerFlags](const string &linkerFlag) { allLinkerFlags += " "; allLinkerFlags += linkerFlag; }; } // TODO: If project contains no source files, then we shouldn't override this function dynamicLinkerFlagCallback = [&allLinkerFlags](const string &linkerFlag) { allLinkerFlags += " "; allLinkerFlags += linkerFlag; }; Result linkerFlags = getLinkerFlags(config, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallback); if(linkerFlags.isErr()) return Result::Err(linkerFlags.getErrMsg()); string projectGeneratedBinary = allLinkerFlags; projectGeneratedBinary += " '"; projectGeneratedBinary += savePath; projectGeneratedBinary += "/"; switch(libraryType) { case LibraryType::EXECUTABLE: { result += "build "; result += config.getPackageName(); result += ": " + buildJob + " "; result += join(objectNames, " "); result += "\n"; result += " LINK_ARGS = '-Wl,--no-undefined,--as-needed' "; if(!allLinkerFlags.empty()) { result += allLinkerFlags; } result += "\n\n"; projectGeneratedBinary += config.getPackageName(); break; } case LibraryType::STATIC: { result += "build "; result += config.getPackageName(); result += ": " + buildJob + " "; result += join(objectNames, " "); result += "\n\n"; projectGeneratedBinary += config.getPackageName() + ".a"; break; } case LibraryType::DYNAMIC: { result += "build lib"; result += config.getPackageName(); result += ".so: " + buildJob + " "; result += join(objectNames, " "); result += "\n"; result += " LINK_ARGS = '-Wl,--no-undefined,--as-needed' "; if(!allLinkerFlags.empty()) { result += allLinkerFlags; //result += " '-Wl,--no-whole-archive'"; } result += "\n\n"; projectGeneratedBinary += "lib" + config.getPackageName() + ".so"; break; } default: assert(false); return Result::Err("Unexpected error"); } projectGeneratedBinary += "'"; Result fileOverwriteResult = sibs::fileOverwrite(ninjaBuildFilePath.c_str(), sibs::StringView(result.data(), result.size())); if(fileOverwriteResult.isErr()) return fileOverwriteResult; printf("Created ninja build file: %s\n", ninjaBuildFilePath.c_str()); Result buildResult = build(savePath); if(!buildResult) return buildResult; Result buildTestResult = buildTests(projectGeneratedBinary, config, savePath); if(!buildTestResult) return buildTestResult; return Result::Ok(true); } const char *sourceFileExtensions[] = { "c", "cc", "cpp", "cxx" }; bool isSourceFile(tinydir_file *file) { if(!file->is_reg) return false; for(const char *sourceFileExtension : sourceFileExtensions) { if(_tinydir_strcmp(sourceFileExtension, file->extension) == 0) return true; } return false; } Result Ninja::buildTests(const std::string &projectGeneratedBinary, const SibsConfig &config, const char *savePath) { if(testSourceDirs.empty()) return Result::Ok(true); // Tests need parent project as dependency. Executables can't be included as dependency so we build it as dynamic library. // `build` also builds tests if(getNinjaLibraryType(config.getPackageType()) == LibraryType::EXECUTABLE) { SibsConfig parentProjConfigLib = config; parentProjConfigLib.setPackageType(PackageType::DYNAMIC); // HACK: We can build a package that is defined as executable and contains main function by redefining `main` // as something else. // TODO: Do not allow defining `main` in project.conf or as program argument to sibs (when sibs supports defines). // It's ok if `define` fails. It could fail if `main` has already been replaced by other tests somehow. parentProjConfigLib.define("main", "sibs_lib_ignore_main"); return build(parentProjConfigLib, savePath, nullptr, nullptr); } for(const string &testSourceDir : testSourceDirs) { string projectConfFilePath = testSourceDir; projectConfFilePath += "/project.conf"; FileType projectConfFileType = getFileType(projectConfFilePath.c_str()); SibsTestConfig sibsTestConfig(testSourceDir); if(projectConfFileType == FileType::REGULAR) { Result result = Config::readFromFile(projectConfFilePath.c_str(), sibsTestConfig); if(!result) return result; } backend::Ninja ninja; ninja.addDependency(projectGeneratedBinary); walkDirFilesRecursive(testSourceDir.c_str(), [&ninja, &sibsTestConfig](tinydir_file *file) { if (isSourceFile(file)) { ninja.addSourceFile(file->path + sibsTestConfig.getProjectPath().size() + 1); } else { //printf("Ignoring non-source file: %s\n", file->path + projectPath.size()); } }); if(!ninja.getSourceFiles().empty()) { string debugBuildPath = testSourceDir + "/sibs-build/debug"; Result buildFileResult = ninja.build(sibsTestConfig, debugBuildPath.c_str()); if (!buildFileResult) return buildFileResult; Result buildResult = ninja.build(debugBuildPath.c_str()); if (!buildResult) return buildResult; } } return Result::Ok(true); } Result Ninja::build(const char *buildFilePath) { string command = "ninja -C '"; command += buildFilePath; command += "'"; Result execResult = exec(command.c_str(), true); if(execResult.isOk()) { if(execResult.unwrap().exitCode == 0) return Result::Ok(true); else return Result::Err(execResult.unwrap().execStdout); } else return Result::Err(execResult.getErrMsg()); } }