#include #include "Ninja.hpp" #include "../../include/FileUtil.hpp" #include "../../include/Exec.hpp" #include "../../include/PkgConfig.hpp" #include "../../include/GlobalLib.hpp" using namespace std; using namespace sibs; #if OS_FAMILY == OS_FAMILY_POSIX #define nprintf printf #else #define nprintf wprintf #endif namespace backend { string join(const vector &list, const char *joinStr) { if(list.empty()) return ""; string result; long stringSize = 0; long joinStrLen = strlen(joinStr); int i = 0; for(const string &str : list) { stringSize += str.size(); if(!str.empty() && i > 0) stringSize += joinStrLen; ++i; } result.reserve(stringSize); i = 0; for(const string &str : list) { if(i > 0) result += joinStr; result += str; ++i; } return result; } bool endsWith(const string &str, const string &endWithStr) { if(endWithStr.size() > str.size()) return false; else return strncmp(&str[str.size() - endWithStr.size()], &endWithStr[0], endWithStr.size()) == 0; } Ninja::LibraryType getNinjaLibraryType(PackageType packageType) { switch(packageType) { case PackageType::EXECUTABLE: return Ninja::LibraryType::EXECUTABLE; case PackageType::STATIC: return Ninja::LibraryType::STATIC; case PackageType::DYNAMIC: case PackageType::LIBRARY: return Ninja::LibraryType::DYNAMIC; default: assert(false); return (Ninja::LibraryType)-1; } } string getIncludeOptionFlag(Compiler compiler, const string &filepath) { string result; switch (compiler) { case Compiler::GCC: { result = "'-I"; result += filepath; result += "'"; break; } case Compiler::MSVC: { result = "/I \""; result += filepath; result += "\""; break; } } return result; } string getDefineFlag(Compiler compiler, const string &name, const string &value) { string result; switch (compiler) { case Compiler::GCC: { result = "'-D"; result += name; result += "="; result += value; result += "'"; break; } case Compiler::MSVC: { result = "\"/D"; result += name; result += "="; result += value; result += "\""; break; } } return result; } string getCompileWithoutLinkingFlag(Compiler compiler) { string result; switch (compiler) { case Compiler::GCC: { result = "-c"; break; } case Compiler::MSVC: { result = "/c"; break; } } return result; } string getObjectFileNameFlag(Compiler compiler, const string &objectFileName) { string result; switch (compiler) { case Compiler::GCC: { result = "-o "; result += objectFileName; break; } case Compiler::MSVC: { result = "/Fo"; result += objectFileName; break; } } return result; } const char* getObjectFileExtension(Compiler compiler) { switch (compiler) { case Compiler::GCC: return ".o"; case Compiler::MSVC: return ".obj"; default: return nullptr; } } Ninja::Ninja() { } void Ninja::addSourceFile(const char *filepath) { string filePathStr = filepath ? filepath : ""; if(filepath && !containsSourceFile(filePathStr)) { sourceFiles.emplace_back(filePathStr); printf("Adding source file: %s\n", filepath); } } void Ninja::addTestSourceDir(const char *dir) { string dirStr = dir ? dir : ""; if(dir && !containsTestSourceDir(dirStr)) { testSourceDirs.emplace_back(dirStr); printf("Adding test source directory: %s\n", dir); } } void Ninja::addDependency(const std::string &binaryFile) { if(!containsDependency(binaryFile)) binaryDependencies.emplace_back(binaryFile); } const std::vector& Ninja::getSourceFiles() const { return sourceFiles; } bool Ninja::containsSourceFile(const string &filepath) const { for(const string &sourceFile : sourceFiles) { if(sourceFile == filepath) return true; } return false; } bool Ninja::containsTestSourceDir(const string &dir) const { for(const string &testSourceDir : testSourceDirs) { if(testSourceDir == dir) return true; } return false; } bool Ninja::containsDependency(const string &dependency) const { for(const string &binaryDependency : binaryDependencies) { if(binaryDependency == dependency) return true; } return false; } #if OS_FAMILY == OS_FAMILY_POSIX Result validatePkgConfigPackageVersionExists(const Dependency &dependency) { Result dependencyValidationResult = PkgConfig::validatePackageExists(dependency.name); if(dependencyValidationResult.isErr()) return Result::Err(dependencyValidationResult.getErrMsg()); Result dependencyVersionValidationResult = PkgConfig::validatePackageVersionAtLeast(dependency.name, dependency.version); if(dependencyVersionValidationResult.isErr()) return Result::Err(dependencyVersionValidationResult.getErrMsg()); return Result::Ok(true); } #endif // TODO: First check if pkg-config is installed. If it's not, only check dependencies that exists in the dependencies sub directory. // If pkg-config is installed and dependency is not installed, check in dependencies sub directory. Result Ninja::getLinkerFlags(const SibsConfig &config, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallback, GlobalIncludeDirCallbackFunc globalIncludeDirCallback) const { const vector &dependencies = config.getDependencies(); if(dependencies.empty()) return Result::Ok(true); Result globalLibDirResult = getHomeDir(); if (!globalLibDirResult) return Result::Err(globalLibDirResult); FileString globalLibDir = globalLibDirResult.unwrap(); globalLibDir += TINYDIR_STRING("/.sibs/lib"); Result createGlobalLibDirResult = createDirectoryRecursive(globalLibDir.c_str()); if(createGlobalLibDirResult.isErr()) return createGlobalLibDirResult; vector globalLibDependencies; #if OS_FAMILY == OS_FAMILY_POSIX vector pkgConfigDependencies; for(const Dependency &dependency : dependencies) { Result pkgConfigDependencyValidation = validatePkgConfigPackageVersionExists(dependency); if(pkgConfigDependencyValidation.isOk()) { pkgConfigDependencies.push_back(dependency); } else { globalLibDependencies.push_back(dependency); } } Result pkgConfigLinkerFlagsResult = PkgConfig::getDynamicLibsLinkerFlags(pkgConfigDependencies); if (pkgConfigLinkerFlagsResult.isErr()) { printf("%s, using global lib...\n", pkgConfigLinkerFlagsResult.getErrMsg().c_str()); globalLibDependencies.reserve(globalLibDependencies.size() + pkgConfigDependencies.size()); for (const Dependency &pkgConfigDependency : pkgConfigDependencies) { globalLibDependencies.push_back(pkgConfigDependency); } pkgConfigDependencies.clear(); } else { if (!pkgConfigLinkerFlagsResult.unwrap().empty()) dynamicLinkerFlagCallback(pkgConfigLinkerFlagsResult.unwrap()); } #else for (const Dependency &dependency : dependencies) { globalLibDependencies.push_back(dependency); } #endif for(const Dependency &globalLibDependency : globalLibDependencies) { printf("Dependency %s is missing from pkg-config, trying global lib\n", globalLibDependency.name.c_str()); Result globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(config, globalLibDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallback, globalIncludeDirCallback); if(globalLibLinkerFlagsResult.isErr()) { if(globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_NOT_FOUND || globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_VERSION_NO_MATCH) { printf("Dependency not found in global lib, trying to download from github\n"); // TODO: Download several dependencies at the same time by adding them to a list // and then iterate them and download them all using several threads. // All dependecies should be downloaded at the same time, this includes dependencies of dependencies. // If a dependency is missing, fail build BEFORE downloading dependencies and before compiling anything. // You do not want to possibly wait several minutes only for build to fail when there is no compilation error. // TODO: If return error is invalid url, then the message should be converted to // invalid package name/version. A check should be done if it is the name or version // that is invalid. Result downloadDependencyResult = GlobalLib::downloadDependency(globalLibDependency); if(downloadDependencyResult.isErr()) return downloadDependencyResult; globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(config, globalLibDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallback, globalIncludeDirCallback); if(globalLibLinkerFlagsResult.isErr()) return Result::Err(globalLibLinkerFlagsResult); } else { return Result::Err(globalLibLinkerFlagsResult); } } } return Result::Ok(true); } Result Ninja::build(const SibsConfig &config, const _tinydir_char_t *savePath, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallback, GlobalIncludeDirCallbackFunc globalIncludeDirCallback) { if (!sourceFiles.empty()) { Result createBuildDirResult = createDirectoryRecursive(savePath); if (createBuildDirResult.isErr()) return createBuildDirResult; } LibraryType libraryType = getNinjaLibraryType(config.getPackageType()); string savePathUtf8 = toUtf8(savePath); string projectPathUtf8 = toUtf8(config.getProjectPath()); FileString ninjaBuildFilePath = savePath; ninjaBuildFilePath += TINYDIR_STRING("/build.ninja"); string result; result.reserve(16384); Result globalIncDirResult = getHomeDir(); if (!globalIncDirResult) return Result::Err(globalIncDirResult); FileString globalIncDir = globalIncDirResult.unwrap(); globalIncDir += TINYDIR_STRING("/.sibs/lib"); string globalIncDirUtf8 = toUtf8(globalIncDir); result += "globalIncDir = "; result += getIncludeOptionFlag(config.getCompiler(), globalIncDirUtf8); for(const auto &includeDir : config.getIncludeDirs()) { string includeDirRelative = "../../"; includeDirRelative += includeDir; result += " "; result += getIncludeOptionFlag(config.getCompiler(), includeDirRelative); } auto parentGlobalIncludeDirCallback = globalIncludeDirCallback; for (const string &globalIncludeDir : config.getGlobalIncludeDirs()) { string globalIncludeDirFull = projectPathUtf8; globalIncludeDirFull += "/"; globalIncludeDirFull += globalIncludeDir; if(parentGlobalIncludeDirCallback) parentGlobalIncludeDirCallback(globalIncludeDirFull); } globalIncludeDirCallback = [&parentGlobalIncludeDirCallback, &globalIncludeDirCallback, &result, &config](const string &globalIncludeDir) { result += " "; result += getIncludeOptionFlag(config.getCompiler(), globalIncludeDir); if (parentGlobalIncludeDirCallback) parentGlobalIncludeDirCallback(globalIncludeDir); }; #if OS_TYPE == OS_TYPE_LINUX // TODO: Allow configuring default linking flags. Maybe have `package.useThreads = false` to disable this flag string allLinkerFlags = "-pthread"; #else string allLinkerFlags = ""; #endif // TODO: Somehow check loading order, because it has to be correct to work.. Or does it for dynamic libraries? // Anyways it's required for static libraries (especially on Windows) for (const string &binaryDependency : binaryDependencies) { allLinkerFlags += " "; allLinkerFlags += binaryDependency; } if (!staticLinkerFlagCallbackFunc || libraryType == LibraryType::DYNAMIC) { staticLinkerFlagCallbackFunc = [&allLinkerFlags](const string &linkerFlag) { allLinkerFlags += " "; allLinkerFlags += linkerFlag; }; } // TODO: If project contains no source files, then we shouldn't override this function dynamicLinkerFlagCallback = [&allLinkerFlags](const string &linkerFlag) { allLinkerFlags += " "; allLinkerFlags += linkerFlag; }; Result linkerFlags = getLinkerFlags(config, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallback, globalIncludeDirCallback); if (linkerFlags.isErr()) return Result::Err(linkerFlags.getErrMsg()); result += "\n\n"; string defines; for(const auto &definePair : config.getDefines()) { defines += " "; defines += getDefineFlag(config.getCompiler(), definePair.first, definePair.second); } switch (SYSTEM_PLATFORM) { case PLATFORM_WIN64: defines += " /DWIN64 /D_WIN64"; case PLATFORM_WIN32: defines += " /DWIN32 /D_WIN32"; break; } #if OS_TYPE == OS_TYPE_WINDOWS switch(libraryType) { case LibraryType::EXECUTABLE: defines += " _CONSOLE"; break; case LibraryType::STATIC: defines += " _LIB"; break; } #endif if(!defines.empty()) printf("Custom define: %s\n", defines.c_str()); string compilerName; switch (config.getCompiler()) { case Compiler::GCC: compilerName = "ccache c++"; break; case Compiler::MSVC: compilerName = "cl.exe"; break; } string buildJob; switch(libraryType) { case LibraryType::EXECUTABLE: { switch (config.getCompiler()) { case Compiler::GCC: { result += "rule cpp_COMPILER\n"; result += " command = ccache c++ $ARGS -c $in -o $out\n\n"; result += "rule cpp_BUILD_EXEC\n"; result += " command = ccache c++ $ARGS -o $out $in $LINK_ARGS $aliasing\n\n"; break; } case Compiler::MSVC: { result += "rule cpp_COMPILER\n"; result += " command = cl.exe $ARGS /c $in /Fo$out\n\n"; result += "rule cpp_BUILD_EXEC\n"; result += " command = cl.exe $ARGS $in /Fe$out $LINK_ARGS\n\n"; break; } } buildJob = "cpp_BUILD_EXEC"; break; } case LibraryType::STATIC: { switch (config.getCompiler()) { case Compiler::GCC: { result += "rule cpp_COMPILER\n"; result += " command = ccache c++ $ARGS -c -fPIC $in -o $out\n\n"; result += "rule cpp_BUILD_STATIC\n"; result += " command = ar rcs lib"; result += config.getPackageName(); result += ".a"; result += " $in\n\n"; break; } case Compiler::MSVC: { result += "rule cpp_COMPILER\n"; result += " command = cl.exe $ARGS /c $in /Fo$out\n\n"; result += "rule cpp_BUILD_STATIC\n"; result += " command = lib.exe /OUT:$out $in\n\n"; break; } } buildJob = "cpp_BUILD_STATIC"; break; } case LibraryType::DYNAMIC: { switch (config.getCompiler()) { case Compiler::GCC: { result += "rule cpp_COMPILER\n"; result += " command = ccache c++ $ARGS -c -fPIC $in -o $out\n\n"; // --whole-archive result += "rule cpp_BUILD_DYNAMIC\n"; result += " command = ccache c++ $in -shared -o $out $LINK_ARGS $aliasing\n\n"; break; } case Compiler::MSVC: { result += "rule cpp_COMPILER\n"; result += " command = cl.exe $ARGS /c $in /Fo$out\n\n"; //result += "rule cpp_BUILD_DYNAMIC\n"; //result += " command = cl.exe /LD $in /Fe$out $LINK_ARGS\n\n"; result += "rule cpp_BUILD_DYNAMIC\n"; result += " command = lib.exe /OUT:$out $in\n\n"; break; } } buildJob = "cpp_BUILD_DYNAMIC"; break; } default: assert(false); return Result::Err("Unexpected error"); } // TODO: Add equivalent functionality for msvc. Currently msvc always builds as debug build string optimizationFlags; switch (config.getOptimizationLevel()) { case OPT_LEV_DEBUG: { switch (config.getCompiler()) { case Compiler::GCC: { optimizationFlags = "'-O0'"; break; } case Compiler::MSVC: { optimizationFlags = "/Od"; break; } } break; } case OPT_LEV_RELEASE: { switch (config.getCompiler()) { case Compiler::GCC: { optimizationFlags = "'-O3' '-DNDEBUG'"; break; } case Compiler::MSVC: { optimizationFlags = "/Ox /DNDEBUG"; break; } } break; } } vector objectNames; objectNames.reserve(sourceFiles.size()); for(const string &sourceFile : sourceFiles) { string sourceFileLanguage = "c++"; if(endsWith(sourceFile, ".c")) sourceFileLanguage = "c"; //string sourceFileEncoded = sourceFile; //replace(sourceFileEncoded, '/', '@'); string objectName = config.getPackageName() + "@exe/" + sourceFile; objectName += getObjectFileExtension(config.getCompiler()); result += "build "; result += objectName; result += ": cpp_COMPILER ../../"; result += sourceFile; result += "\n"; result += " ARGS = $globalIncDir "; if(!defines.empty()) result += defines; switch (config.getCompiler()) { case Compiler::GCC: { // -Werror result += " '-I" + config.getPackageName() + "@exe' '-I..' -Wall -Wextra -Werror=return-type -fexceptions -fdiagnostics-show-option '-fdiagnostics-color=always' '-pipe' '-D_FILE_OFFSET_BITS=64' '-Winvalid-pch' '-Wnon-virtual-dtor' " + optimizationFlags + " '-g'"; break; } case Compiler::MSVC: { result += " "; result += optimizationFlags; result += " /EHsc"; switch (config.getOptimizationLevel()) { case OPT_LEV_DEBUG: result += " /MTd"; break; case OPT_LEV_RELEASE: result += " /MT"; break; } switch (SYSTEM_PLATFORM) { case PLATFORM_WIN32: result += " /MACHINE:X86"; break; case PLATFORM_WIN64: result += " /MACHINE:X64"; break; } break; } } result += "\n\n"; objectNames.emplace_back(objectName); } string projectGeneratedBinary; if (!sourceFiles.empty()) { projectGeneratedBinary = allLinkerFlags; projectGeneratedBinary += " \""; projectGeneratedBinary += savePathUtf8; projectGeneratedBinary += "/"; switch (libraryType) { case LibraryType::EXECUTABLE: { result += "build "; result += config.getPackageName(); result += ": " + buildJob + " "; result += join(objectNames, " "); result += "\n"; switch (config.getCompiler()) { case Compiler::GCC: { result += " LINK_ARGS = '-Wl,--no-undefined,--as-needed' "; break; } case Compiler::MSVC: { // TODO: Do not link all of these. Find a way to only link the ones that are needed result += " LINK_ARGS = Ws2_32.lib Wldap32.lib Crypt32.lib Advapi32.lib Gdi32.lib User32.lib Userenv.lib OpenGL32.lib GlU32.lib Shell32.lib "; break; } } if (!allLinkerFlags.empty()) { result += allLinkerFlags; } result += "\n\n"; projectGeneratedBinary += config.getPackageName(); break; } case LibraryType::STATIC: { switch (config.getCompiler()) { case Compiler::GCC: { result += "build lib"; result += config.getPackageName(); result += ".a: " + buildJob + " "; result += join(objectNames, " "); result += "\n\n"; projectGeneratedBinary += config.getPackageName() + ".a"; break; } case Compiler::MSVC: { result += "build "; result += config.getPackageName(); result += ".lib: " + buildJob + " "; result += join(objectNames, " "); result += "\n\n"; projectGeneratedBinary += config.getPackageName() + ".lib"; break; } } break; } case LibraryType::DYNAMIC: { switch (config.getCompiler()) { case Compiler::GCC: { result += "build lib"; result += config.getPackageName(); result += ".so: " + buildJob + " "; result += join(objectNames, " "); result += "\n"; result += " LINK_ARGS = '-Wl,--no-undefined,--as-needed' "; projectGeneratedBinary += "lib" + config.getPackageName() + ".so"; break; } case Compiler::MSVC: { result += "build "; result += config.getPackageName(); result += ".lib: " + buildJob + " "; result += join(objectNames, " "); result += "\n"; // TODO: Do not link all of these. Find a way to only link the ones that are needed result += " LINK_ARGS = Ws2_32.lib Wldap32.lib Crypt32.lib Advapi32.lib Gdi32.lib User32.lib Userenv.lib OpenGL32.lib GlU32.lib Shell32.lib "; projectGeneratedBinary += config.getPackageName() + ".lib"; break; } } if (!allLinkerFlags.empty()) { result += allLinkerFlags; //result += " '-Wl,--no-whole-archive'"; } result += "\n\n"; break; } default: assert(false); return Result::Err("Unexpected error"); } projectGeneratedBinary += "\""; Result fileOverwriteResult = sibs::fileOverwrite(ninjaBuildFilePath.c_str(), sibs::StringView(result.data(), result.size())); if (fileOverwriteResult.isErr()) return fileOverwriteResult; nprintf(TINYDIR_STRING("Created ninja build file: %s\n"), ninjaBuildFilePath.c_str()); Result buildResult = compile(savePath); if (!buildResult) return buildResult; } Result buildTestResult = buildTests(projectGeneratedBinary, config, savePath); if(!buildTestResult) return buildTestResult; return Result::Ok(true); } const _tinydir_char_t *sourceFileExtensions[] = { TINYDIR_STRING("c"), TINYDIR_STRING("cc"), TINYDIR_STRING("cpp"), TINYDIR_STRING("cxx") }; bool isSourceFile(tinydir_file *file) { if(!file->is_reg) return false; for(const _tinydir_char_t *sourceFileExtension : sourceFileExtensions) { if(_tinydir_strcmp(sourceFileExtension, file->extension) == 0) return true; } return false; } Result Ninja::buildTests(const std::string &projectGeneratedBinary, const SibsConfig &config, const _tinydir_char_t *savePath) { if(testSourceDirs.empty() || !config.shouldBuildTests()) return Result::Ok(true); // Tests need parent project as dependency. Executables can't be included as dependency so we build it as dynamic library. // `build` also builds tests if(getNinjaLibraryType(config.getPackageType()) == LibraryType::EXECUTABLE) { SibsConfig parentProjConfigLib = config; parentProjConfigLib.setPackageType(PackageType::DYNAMIC); // HACK: We can build a package that is defined as executable and contains main function by redefining `main` // as something else. // TODO: Do not allow defining `main` in project.conf or as program argument to sibs (when sibs supports defines). // It's ok if `define` fails. It could fail if `main` has already been replaced by other tests somehow. parentProjConfigLib.define("main", "sibs_lib_ignore_main"); parentProjConfigLib.define("wmain", "sibs_lib_ignore_wmain"); return build(parentProjConfigLib, savePath, nullptr, nullptr); } for(const string &testSourceDir : testSourceDirs) { #if OS_FAMILY == OS_FAMILY_POSIX FileString testSourceDirNative = testSourceDir; FileString projectConfFilePath = testSourceDir; #else FileString testSourceDirNative = utf8To16(testSourceDir); FileString projectConfFilePath = testSourceDirNative; #endif projectConfFilePath += TINYDIR_STRING("/project.conf"); FileType projectConfFileType = getFileType(projectConfFilePath.c_str()); SibsTestConfig sibsTestConfig(config.getCompiler(), testSourceDirNative, config.getOptimizationLevel()); if(projectConfFileType == FileType::REGULAR) { Result result = Config::readFromFile(projectConfFilePath.c_str(), sibsTestConfig); if(!result) return result; } backend::Ninja ninja; if(!projectGeneratedBinary.empty()) ninja.addDependency(projectGeneratedBinary); // TODO: Use same source file finder as in main.cpp walkDirFilesRecursive(testSourceDirNative.c_str(), [&ninja, &sibsTestConfig](tinydir_file *file) { if (isSourceFile(file)) { string filePathUtf8 = toUtf8(file->path + sibsTestConfig.getProjectPath().size() + 1); ninja.addSourceFile(filePathUtf8.c_str()); } else { //printf("Ignoring non-source file: %s\n", file->path + projectPath.size()); } }); if(!ninja.getSourceFiles().empty()) { FileString buildPath = testSourceDirNative; switch(sibsTestConfig.getOptimizationLevel()) { case OPT_LEV_DEBUG: buildPath += TINYDIR_STRING("/sibs-build/debug"); break; case OPT_LEV_RELEASE: buildPath += TINYDIR_STRING("/sibs-build/release"); break; } Result buildFileResult = ninja.build(sibsTestConfig, buildPath.c_str()); if (!buildFileResult) return buildFileResult; Result buildResult = ninja.compile(buildPath.c_str()); if (!buildResult) return buildResult; FileString testExecutableName = buildPath; testExecutableName += TINYDIR_STRING("/"); testExecutableName += toFileString(sibsTestConfig.getPackageName()); Result runTestResult = exec(testExecutableName.c_str(), true); if(!runTestResult) return Result::Err(runTestResult); } } return Result::Ok(true); } Result Ninja::compile(const _tinydir_char_t *buildFilePath) { FileString command = TINYDIR_STRING("ninja -C \""); command += buildFilePath; command += TINYDIR_STRING("\""); Result execResult = exec(command.c_str(), true); if(execResult.isOk()) { if(execResult.unwrap().exitCode == 0) return Result::Ok(true); else return Result::Err(execResult.unwrap().execStdout); } else return Result::Err(execResult.getErrMsg()); } }