Add support for running zig tests

7 files changed, 267 insertions, 35 deletions

diff --git a/README.md b/README.md
index d32a934..98cd766 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,9 @@ Sibs is still in very early testing phase, should only be used if you want to to
 
 Sibs is inspired by [Cargo](https://github.com/rust-lang/cargo/), you can think of it like a C/C++/Zig version of Cargo.
 Zig support has not been tested properly yet and currently always links to c library and requires a C/C++ compiler to build (for linking).
+You can run zig tests with `sibs test --file filepath` or `sibs test --all-files`.
+Currently zig tests are cached because ninja build system is used, which means if source files do not change between runs,
+then tests wont run. This should be fixed later...
 
 The CMakeLists.txt is only for development purpose and only compiles on linux.
 
diff --git a/backend/BackendUtils.cpp b/backend/BackendUtils.cpp
index ac71b07..3c9dd71 100644
--- a/backend/BackendUtils.cpp
+++ b/backend/BackendUtils.cpp
@@ -27,28 +27,43 @@ namespace backend
         TINYDIR_STRING("c++")
     };
 
-    sibs::Language BackendUtils::getFileLanguage(tinydir_file *file)
+    sibs::FileString BackendUtils::getFileExtension(const sibs::FileString &filepath)
     {
-        if(!file->is_reg)
-            return sibs::Language::NONE;
+        size_t indexOfDot = filepath.find_last_of('.');
+        if(indexOfDot == sibs::FileString::npos)
+            return TINYDIR_STRING("");
+        
+        indexOfDot += 1;
+        return filepath.substr(indexOfDot);
+    }
 
+    sibs::Language BackendUtils::getFileLanguage(const _tinydir_char_t *extension)
+    {
         for(const _tinydir_char_t *sourceFileExtension : cFileExtensions)
         {
-            if(_tinydir_strcmp(sourceFileExtension, file->extension) == 0)
+            if(_tinydir_strcmp(sourceFileExtension, extension) == 0)
                 return sibs::Language::C;
         }
 
         for(const _tinydir_char_t *sourceFileExtension : cppFileExtensions)
         {
-            if(_tinydir_strcmp(sourceFileExtension, file->extension) == 0)
+            if(_tinydir_strcmp(sourceFileExtension, extension) == 0)
                 return sibs::Language::CPP;
         }
 
-        if(_tinydir_strcmp(TINYDIR_STRING("zig"), file->extension) == 0)
+        if(_tinydir_strcmp(TINYDIR_STRING("zig"), extension) == 0)
             return sibs::Language::ZIG;
 
         return sibs::Language::NONE;
     }
+
+    sibs::Language BackendUtils::getFileLanguage(tinydir_file *file)
+    {
+        if(!file->is_reg)
+            return sibs::Language::NONE;
+
+        return getFileLanguage(file->extension);
+    }
     
     void BackendUtils::collectSourceFiles(const _tinydir_char_t *projectPath, Ninja *ninjaProject, const SibsConfig &sibsConfig, bool recursive)
     {
diff --git a/backend/BackendUtils.hpp b/backend/BackendUtils.hpp
index acef0ca..f99d7e5 100644
--- a/backend/BackendUtils.hpp
+++ b/backend/BackendUtils.hpp
@@ -9,6 +9,8 @@ namespace backend
     class BackendUtils
     {
     public:
+        static sibs::FileString getFileExtension(const sibs::FileString &filepath);
+        static sibs::Language getFileLanguage(const _tinydir_char_t *extension);
         static sibs::Language getFileLanguage(tinydir_file *file);
         static void collectSourceFiles(const _tinydir_char_t *projectPath, Ninja *ninjaProject, const sibs::SibsConfig &sibsConfig, bool recursive = true);
     };
diff --git a/backend/ninja/Ninja.cpp b/backend/ninja/Ninja.cpp
index ca88f40..c642a4c 100644
--- a/backend/ninja/Ninja.cpp
+++ b/backend/ninja/Ninja.cpp
@@ -474,6 +474,7 @@ namespace backend
             case Compiler::MSVC:
             {
                 includeStartStr = "/I";
+                break;
             }
             default:
                 assert(false);
@@ -527,6 +528,72 @@ namespace backend
         return result;
     }
 
+    static vector<ninja::NinjaArg> extractLibraryPathsWithoutFlags(Compiler compiler, const string &cLibraries)
+    {
+        vector<ninja::NinjaArg> result;
+
+        size_t i = 0;
+        while(i < cLibraries.size())
+        {
+            char c = cLibraries[i];
+            if(c == '\'' || c == '"')
+            {
+                // TODO: Handle quote escaping
+                char quoteSymbol = c;
+                ++i;
+                size_t libraryIndexStart = i;
+                while(i < cLibraries.size() && cLibraries[i] != quoteSymbol)
+                {
+                    ++i;
+                }
+                size_t libraryIndexEnd = i;
+                ++i;
+                
+                if(strncmp(&cLibraries[libraryIndexStart], "-l", 2) == 0)
+                    libraryIndexStart += 2;
+                else if(strcmp(&cLibraries[libraryIndexStart], "-pthread") == 0)
+                    continue;
+
+                size_t libraryPathLength = libraryIndexEnd - libraryIndexStart;
+                if(libraryPathLength > 0)
+                    result.push_back(ninja::NinjaArg::createRaw(cLibraries.substr(libraryIndexStart, libraryPathLength)));
+            }
+            else if(c != ' ')
+            {
+                // TODO: Handle space escaping
+                size_t libraryIndexStart = i;
+                while(i < cLibraries.size() && cLibraries[i] != ' ')
+                {
+                    ++i;
+                }
+                size_t libraryIndexEnd = i;
+                ++i;
+
+                if(strncmp(&cLibraries[libraryIndexStart], "-l", 2) == 0)
+                    libraryIndexStart += 2;
+                else if(strcmp(&cLibraries[libraryIndexStart], "-pthread") == 0)
+                    continue;
+                
+                size_t libraryPathLength = libraryIndexEnd - libraryIndexStart;
+                if(libraryPathLength > 0)
+                    result.push_back(ninja::NinjaArg::createRaw(cLibraries.substr(libraryIndexStart, libraryPathLength)));
+            }
+        }
+
+        return result;
+    }
+
+    static vector<ninja::NinjaArg> convertCLibrariesToZigLibraries(Compiler compiler, const string &cLibraries)
+    {
+        vector<ninja::NinjaArg> result;
+        result = extractLibraryPathsWithoutFlags(compiler, cLibraries);
+        for(ninja::NinjaArg &include : result)
+        {
+            include.arg = "--library \"" + include.arg + "\"";
+        }
+        return result;
+    }
+
     Result<bool> Ninja::build(const SibsConfig &config, const _tinydir_char_t *savePath, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallback, GlobalIncludeDirCallbackFunc globalIncludeDirCallback)
     {
         if (!sourceFiles.empty())
@@ -809,18 +876,29 @@ namespace backend
         // TODO: Specify -mconsole or -mwindows for windows.
         // TODO: Convert sibs defines to const variables in a zig file that other zig files can include (like a config file).
 
+        // TODO: Remove --library c if project does not depend on c libraries or project only has .zig files
+        vector<ninja::NinjaArg> zigTestArgs = {
+            ninja::NinjaArg::createRaw("zig test $in --output $out -isystem ../../ --color on --library c $globalIncDirZig")
+        };
+
         // TODO: Find a way to do this more efficiently
         vector<ninja::NinjaArg> cflagsIncludes = convertCFlagsIncludesToZigIncludes(config.getCompiler(), cflags);
+        vector<ninja::NinjaArg> zigLibraryFlags = convertCLibrariesToZigLibraries(config.getCompiler(), allLinkerFlags);
+
         vector<ninja::NinjaArg> compileZigArgs = {
             ninja::NinjaArg::createRaw("zig build-obj"),
             ninja::NinjaArg::createRaw("$in"),
             ninja::NinjaArg::createRaw("--output $out"),
-            ninja::NinjaArg::createRaw("-isystem ."),
+            ninja::NinjaArg::createRaw("-isystem ../../"),
             ninja::NinjaArg::createRaw("--color on"),
             ninja::NinjaArg::createRaw("--library c"), // TODO: Remove this if project does not depend on c libraries or project only has .zig files
             ninja::NinjaArg::createRaw("$globalIncDirZig")
         };
+        // TODO: Verify if we really need to add all libraries for every object file
         compileZigArgs.insert(compileZigArgs.end(), cflagsIncludes.begin(), cflagsIncludes.end());
+        compileZigArgs.insert(compileZigArgs.end(), zigLibraryFlags.begin(), zigLibraryFlags.end());
+        zigTestArgs.insert(zigTestArgs.end(), cflagsIncludes.begin(), cflagsIncludes.end());
+        zigTestArgs.insert(zigTestArgs.end(), zigLibraryFlags.begin(), zigLibraryFlags.end());
 
         if(config.getOptimizationLevel() == sibs::OPT_LEV_RELEASE)
         {
@@ -829,6 +907,11 @@ namespace backend
                 ninja::NinjaArg::createRaw("--release-safe"),
                 ninja::NinjaArg::createRaw("--strip")
             });
+
+            zigTestArgs.insert(zigTestArgs.end(), { 
+                ninja::NinjaArg::createRaw("--release-safe"),
+                ninja::NinjaArg::createRaw("--strip")
+            });
         }
 
         if(libraryType == LibraryType::STATIC)
@@ -837,9 +920,11 @@ namespace backend
             compileZigArgs.push_back(ninja::NinjaArg::createRaw("-rdynamic"));
 
         ninja::NinjaRule *compileZigRule = ninjaBuildFile.createRule("compile_zig", compileZigArgs);
+        ninja::NinjaRule *testZigRule = ninjaBuildFile.createRule("zig_test", zigTestArgs);
 
         bool usesCFiles = false;
         bool usesCppFiles = false;
+        bool zigTest = (config.zigTestAllFiles || !config.zigTestFiles.empty());
 
         vector<string> objectNames;
         objectNames.reserve(sourceFiles.size());
@@ -868,7 +953,10 @@ namespace backend
                 {
                     objectName += "zig-cache/" + sourceFile.filepath;
                     objectName += getObjectFileExtension(config.getCompiler());
-                    ninjaBuildFile.build(compileZigRule, "../../" + sourceFile.filepath, objectName, {});
+                    if(zigTest)
+                        ninjaBuildFile.build(testZigRule, "../../" + sourceFile.filepath, objectName, {});
+                    else
+                        ninjaBuildFile.build(compileZigRule, "../../" + sourceFile.filepath, objectName, {});
                     break;
                 }
                 default:
@@ -882,7 +970,7 @@ namespace backend
         // they should be built with zig if project only contains zig files.
         // But how to combine object files with zig? build-exe only wants to accept .zig files
         string projectGeneratedBinaryFlags;
-        if (!sourceFiles.empty())
+        if (!sourceFiles.empty() && !zigTest)
         {
             string projectGeneratedBinary = "\"";
             projectGeneratedBinary += savePathUtf8;
@@ -1070,6 +1158,10 @@ namespace backend
             }
             
             projectGeneratedBinaryFlags = allLinkerFlags + " " + projectGeneratedBinary;
+        }
+
+        if(!sourceFiles.empty())
+        {
             string result = ninjaBuildFile.generate();
             Result<bool> fileOverwriteResult = sibs::fileOverwrite(ninjaBuildFilePath.c_str(), sibs::StringView(result.data(), result.size()));
             if (fileOverwriteResult.isErr())
@@ -1089,9 +1181,6 @@ namespace backend
             }
         }
 
-        // TODO: If tests are being run (sibs test) and root project is an executable, do not run compile (above code) as executable. 
-        // Sibs test will compile root project as dynamic library so you end up compiling the project twice, first as an executable and then as a dynamic library.
-        // Even if the root project has been built before and there is cached object, it will take a few seconds to run compile
         Result<bool> buildTestResult = buildTests(projectGeneratedBinaryFlags, config, savePath, dependencyExportIncludeDirs);
         if(!buildTestResult)
             return buildTestResult;
@@ -1132,6 +1221,8 @@ namespace backend
             FileType projectConfFileType = getFileType(projectConfFilePath.c_str());
             SibsTestConfig sibsTestConfig(config.getCompiler(), testSourceDirNative, config.getOptimizationLevel());
             sibsTestConfig.setSanitize(config.getSanitize());
+            sibsTestConfig.zigTestFiles = move(config.zigTestFiles);
+            sibsTestConfig.zigTestAllFiles = config.zigTestAllFiles;
             if(projectConfFileType == FileType::REGULAR)
             {
                 Result<bool> result = Config::readFromFile(projectConfFilePath.c_str(), sibsTestConfig);
@@ -1144,7 +1235,38 @@ namespace backend
             if(!projectGeneratedBinaryFlags.empty())
                 ninja.addDependency(projectGeneratedBinaryFlags);
 
-            backend::BackendUtils::collectSourceFiles(testSourceDirNative.c_str(), &ninja, sibsTestConfig, false);
+            bool zigTest = false;
+            if(config.zigTestAllFiles)
+            {
+                backend::BackendUtils::collectSourceFiles(testSourceDirNative.c_str(), &ninja, sibsTestConfig, false);
+                // TODO: This can be optimized as well. No need to insert non-zig files if we are going to remove them.
+                // Maybe pass a filter callback function to @collectSourceFiles.
+                for(auto it = ninja.sourceFiles.begin(); it != ninja.sourceFiles.end(); )
+                {
+                    if(it->language != sibs::Language::ZIG)
+                    {
+                        it = ninja.sourceFiles.erase(it);
+                    }
+                    else
+                    {
+                        ++it;
+                    }
+                }
+                zigTest = true;
+            }
+            else if(!config.zigTestFiles.empty())
+            {
+                ninja.sourceFiles.reserve(config.zigTestFiles.size());
+                for(const sibs::FileString &testFile : config.zigTestFiles)
+                {
+                    ninja.addSourceFile(sibs::Language::ZIG, testFile.c_str());
+                }
+                zigTest = true;
+            }
+            else
+            {
+                backend::BackendUtils::collectSourceFiles(testSourceDirNative.c_str(), &ninja, sibsTestConfig, false);
+            }
 
             if(!ninja.getSourceFiles().empty())
             {
@@ -1170,16 +1292,19 @@ namespace backend
                     if(!buildFileResult)
                         return buildFileResult;
                 }
-                
-                FileString testExecutableName = buildPath;
-                testExecutableName += TINYDIR_STRING("/");
-                testExecutableName += toFileString(sibsTestConfig.getPackageName());
-                Result<ExecResult> runTestResult = exec(testExecutableName.c_str(), true);
-                if(!runTestResult)
-                    return Result<bool>::Err(runTestResult);
-                
-                if(runTestResult.unwrap().exitCode != 0)
-                    return Result<bool>::Err("Tests failed", runTestResult.unwrap().exitCode);
+
+                if(!zigTest)
+                {
+                    FileString testExecutableName = buildPath;
+                    testExecutableName += TINYDIR_STRING("/");
+                    testExecutableName += toFileString(sibsTestConfig.getPackageName());
+                    Result<ExecResult> runTestResult = exec(testExecutableName.c_str(), true);
+                    if(!runTestResult)
+                        return Result<bool>::Err(runTestResult);
+                    
+                    if(runTestResult.unwrap().exitCode != 0)
+                        return Result<bool>::Err("Tests failed", runTestResult.unwrap().exitCode);
+                }
             }
         }
 
diff --git a/include/Conf.hpp b/include/Conf.hpp
index 278acf2..951def9 100644
--- a/include/Conf.hpp
+++ b/include/Conf.hpp
@@ -229,7 +229,8 @@ namespace sibs
             cppVersion(CPPVersion::CPP14),
             mainProject(false),
             sanitize(false),
-            showWarnings(false)
+            showWarnings(false),
+            zigTestAllFiles(false)
         {
             cmakeDirGlobal = projectPath;
             cmakeDirStatic = cmakeDirGlobal;
@@ -417,12 +418,16 @@ namespace sibs
         virtual bool isDefined(const std::string &name) const;
         virtual bool define(const std::string &name, const std::string &value);
         virtual const std::unordered_map<std::string, std::string>& getDefines() const;
+
+        virtual bool isTest() { return false; }
         
         // Get define value by name. 
         // Return empty string if the value is empty or if the defined value doesn't exist
         const std::string& getDefinedValue(const std::string &name) const;
         
+        std::vector<FileString> zigTestFiles;
         bool showWarnings;
+        bool zigTestAllFiles;
     protected:
         virtual void processObject(StringView name) override;
         virtual void processField(StringView name, const ConfigValue &value) override;
@@ -474,7 +479,7 @@ namespace sibs
     class SibsTestConfig : public SibsConfig
     {
     public:
-        SibsTestConfig(Compiler _compiler, const FileString &_projectPath, OptimizationLevel _optimizationLevel) : SibsConfig(_compiler, _projectPath, _optimizationLevel, true)
+        SibsTestConfig(Compiler _compiler, const FileString &_projectPath, OptimizationLevel _optimizationLevel) : SibsConfig(_compiler, _projectPath, _optimizationLevel, false)
         {
             packageName = "test";
             showWarnings = true;
@@ -482,6 +487,8 @@ namespace sibs
 
         virtual ~SibsTestConfig(){}
 
+        bool isTest() override { return true; }
+
         PackageType getPackageType() const override
         {
             return PackageType::EXECUTABLE;
diff --git a/src/Conf.cpp b/src/Conf.cpp
index 6a8de19..695bc9e 100644
--- a/src/Conf.cpp
+++ b/src/Conf.cpp
@@ -508,12 +508,6 @@ namespace sibs
                 buildPath += TINYDIR_STRING("release");
                 break;
         }
-        
-        if(sibsConfig.shouldBuildTests() && sibsConfig.getTestPath().empty())
-        {
-            printf("Project is missing package.tests config. No tests to build\n");
-            exit(0);
-        }
     }
 
     const char* asString(Platform platform)
diff --git a/src/main.cpp b/src/main.cpp
index d8ce41c..6c9df97 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -93,6 +93,8 @@ using namespace std::chrono;
 // TODO: Make Process::exec safe to use. Currently you pass an argument and it's run as a command, but the string can be escaped to perform malicious acts.
 // Process::exec should be modified to take a list of arguments to execute command with.
 
+// TODO: Verify paths (test path, ignore dirs, include dirs, expose include dir, sibs test --file dir) are sub directory of the project
+
 #if OS_FAMILY == OS_FAMILY_POSIX
 #define fout std::cout
 #define ferr std::cerr
@@ -139,7 +141,7 @@ void usageBuild()
 
 void usageNew()
 {
-    printf("Usage: sibs new <project_name> <--exec|--static|--dynamic> <--lang [c|c++|zig]>\n\n");
+    printf("Usage: sibs new <project_name> <--exec|--static|--dynamic> [--lang c|c++|zig]\n\n");
     printf("Create a new sibs project\n\n");
     printf("Options:\n");
     printf("  project_name\t\tThe name of the project you want to create\n");
@@ -154,11 +156,13 @@ void usageNew()
 
 void usageTest()
 {
-    printf("Usage: sibs test [project_path] [--no-sanitize]\n\n");
+    printf("Usage: sibs test [project_path] [--no-sanitize] [--file filepath...|--all-files]\n\n");
     printf("Build and run tests for a sibs project\n\n");
     printf("Options:\n");
     printf("  project_path\t\tThe directory containing a project.conf file - Optional (default: current directory)\n");
-    printf("  --no-sanitize\t\tDisable runtime address/undefined behavior - Optional\n");
+    printf("  --no-sanitize\t\tDisable runtime address/undefined behavior - Optional (default: enabled), Only applicable for C and C++\n");
+    printf("  --file\t\t\tSpecify file to test, path to test file should be defined after this. Can be defined multiple times to test multiple files - Optional (default: not used), Only applicable for Zig\n");
+    printf("  --all-files\t\t\tTest all files - Optional (default: not used), Only applicable for Zig\n");
     printf("Examples:\n");
     printf("  sibs test\n");
     printf("  sibs test dirA/dirB\n");
@@ -167,7 +171,7 @@ void usageTest()
 
 void usageInit()
 {
-    printf("Usage: sibs init [project_path] <--exec|--static|--dynamic> <--lang [c|c++|zig]>\n\n");
+    printf("Usage: sibs init [project_path] <--exec|--static|--dynamic> [--lang c|c++|zig]\n\n");
     printf("Create sibs project structure in an existing directory\n\n");
     printf("Options:\n");
     printf("  project_path\t\tThe directory where you want to initialize sibs project - Optional (default: current  directory)\n");
@@ -390,6 +394,8 @@ int testProject(int argc, const _tinydir_char_t **argv)
         usageTest();
     
     FileString projectPath;
+    vector<FileString> filesToTest;
+    bool testAllFiles = false;
     bool sanitize = true;
 
     for(int i = 0; i < argc; ++i)
@@ -399,6 +405,39 @@ int testProject(int argc, const _tinydir_char_t **argv)
         {
             sanitize = false;
         }
+        else if(_tinydir_strcmp(arg, TINYDIR_STRING("--file")) == 0)
+        {
+            if(i == argc - 1)
+            {
+                ferr << "Error: Expected path to file to test after --file " << endl;
+                usageTest();
+            }
+
+            ++i;
+            arg = argv[i];
+            filesToTest.push_back(arg);
+
+            if(testAllFiles)
+            {
+                ferr << "Error: --file can't be used together with --all-files " << endl;
+                usageTest();
+            }
+        }
+        else if(_tinydir_strcmp(arg, TINYDIR_STRING("--all-files")) == 0)
+        {
+            if(testAllFiles)
+            {
+                ferr << "Error: --all-files defined twice " << endl;
+                usageTest();
+            }
+            testAllFiles = true;
+
+            if(!filesToTest.empty())
+            {
+                ferr << "Error: --all-files can't be used together with --file " << endl;
+                usageTest();
+            }
+        }
         else if(_tinydir_strncmp(arg, TINYDIR_STRING("--"), 2) == 0)
         {
             ferr << "Error: Invalid argument " << arg << endl;
@@ -431,6 +470,44 @@ int testProject(int argc, const _tinydir_char_t **argv)
     }
     projectPath = projectRealPathResult.unwrap();
 
+    for(const FileString &testFile : filesToTest)
+    {
+        if(testFile.empty())
+        {
+            ferr << "Error: Test filepath can't be empty" << endl;
+            exit(20);
+        }
+
+        FileType fileType = getFileType(testFile.c_str());
+        switch(fileType)
+        {
+            case FileType::FILE_NOT_FOUND:
+            {
+                ferr << "Error: Test file not found: " << testFile << endl;
+                exit(20);
+                break;
+            }
+            case FileType::DIRECTORY:
+            {
+                ferr << "Error: Test file " << testFile << " is a directory, expected to be a file" << endl;
+                exit(20);
+                break;
+            }
+            case FileType::REGULAR:
+            {
+                // TODO: This can be optimized, there is no need to create a copy to check file extension
+                FileString fileExtension = backend::BackendUtils::getFileExtension(testFile);
+                sibs::Language fileLanguage = backend::BackendUtils::getFileLanguage(fileExtension.c_str());
+                if(fileLanguage != sibs::Language::ZIG)
+                {
+                    ferr << "Error: file specific testing can only be done on zig files. " << testFile << " is not a zig file" << endl;
+                    exit(42);
+                }
+                break;
+            }
+        }
+    }
+
     FileString projectConfFilePath = projectPath;
     projectConfFilePath += TINYDIR_STRING("/project.conf");
     validateFilePath(projectConfFilePath.c_str());
@@ -446,6 +523,15 @@ int testProject(int argc, const _tinydir_char_t **argv)
     SibsConfig sibsConfig(compiler, projectPath, OPT_LEV_DEBUG, true);
     sibsConfig.showWarnings = true;
     sibsConfig.setSanitize(sanitize);
+    sibsConfig.zigTestFiles = move(filesToTest);
+    sibsConfig.zigTestAllFiles = testAllFiles;
+
+    if(sibsConfig.getTestPath().empty() && !sibsConfig.zigTestAllFiles && sibsConfig.zigTestFiles.empty())
+    {
+        printf("Project is missing package.tests config. No tests to build\n");
+        exit(50);
+    }
+
     return buildProject(projectPath, projectConfFilePath, sibsConfig);
 }