#include "../include/Conf.hpp" #include "../include/types.hpp" #include "../external/utf8/unchecked.h" using namespace std; using u8string = utf8::unchecked::iterator; namespace sibs { class UnexpectedTokenException : public std::runtime_error { public: UnexpectedTokenException(const string &errMsg) : runtime_error(errMsg) { } }; enum class Token { NONE, END_OF_FILE, IDENTIFIER, OPEN_BRACKET, CLOSING_BRACKET, EQUALS, STRING, COMMA }; const char *getTokenName(Token token) { switch(token) { case Token::NONE: return "NONE"; case Token::END_OF_FILE: return ""; case Token::IDENTIFIER: return "identifier"; case Token::OPEN_BRACKET: return "["; case Token::CLOSING_BRACKET: return "]"; case Token::EQUALS: return "="; case Token::STRING: return "string"; case Token::COMMA: return ","; default: return "Unknown"; } } class Tokenizer { public: Tokenizer(const char *_code) : currentToken(Token::NONE), code((char*)_code) { } Token nextToken() { u32 c = *code; while(isWhitespace(c)) { ++code; c = *code; } if(isIdentifierChar(c)) { char *startOfIdentifier = code.base(); ++code; c = *code; while(isIdentifierChar(c)) { ++code; c = *code; } char *endOfIdentifier = code.base(); identifier = StringView(startOfIdentifier, endOfIdentifier - startOfIdentifier); return Token::IDENTIFIER; } else if(c == '[') { ++code; return Token::OPEN_BRACKET; } else if(c == ']') { ++code; return Token::CLOSING_BRACKET; } else if(c == '=') { ++code; return Token::EQUALS; } else if(c == '"') { bool escapeQuote = false; ++code; char *startOfStr = code.base(); while(true) { c = *code; if(c == '"' && !escapeQuote) break; else if(c == '\\') escapeQuote = !escapeQuote; else if(c == '\0') throw UnexpectedTokenException("Reached end of file before string end"); else escapeQuote = false; ++code; } str = StringView(startOfStr, code.base() - startOfStr); ++code; return Token::STRING; } else if (c == ',') { ++code; return Token::COMMA; } else if(c == '#') { ++code; while(true) { c = *code; if(c == '\n') return nextToken(); else if(c == '\0') return Token::END_OF_FILE; ++code; } } else if(c == '\0') { return Token::END_OF_FILE; } else { string errMsg = "Unexpected token: "; errMsg += (char)c; throw UnexpectedTokenException(errMsg); } } StringView getIdentifier() const { return identifier; } StringView getString() const { return str; } private: bool isWhitespace(u32 c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } bool isAlpha(u32 c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); } bool isDigit(u32 c) { return c >= '0' && c <= '9'; } bool isIdentifierChar(u32 c) { return isAlpha(c) || isDigit(c) || c == '_' || c == '-' || c == '.'; } private: Token currentToken; u8string code; union { StringView identifier; StringView str; }; }; class Parser { public: static Result parse(const char *code, const ConfigCallback &callback) { try { Parser parser(code, (ConfigCallback*)&callback); parser.parse(); return Result::Ok(true); } catch (const UnexpectedTokenException &e) { return Result::Err(e.what()); } catch (const ParserException &e) { return Result::Err(e.what()); } } private: Parser(const char *code, ConfigCallback *_callback) : tokenizer(code), callback(_callback), objectDefined(false) { } void parse() { while(true) { Token token = tokenizer.nextToken(); switch(token) { case Token::IDENTIFIER: { parseConfigField(); break; } case Token::OPEN_BRACKET: { parseConfigObject(); break; } case Token::END_OF_FILE: { callback->finished(); return; } default: { string errMsg = "Expected identifier or object, got: "; errMsg += getTokenName(token); throw ParserException(errMsg); } } } } void parseConfigField() { StringView fieldName = tokenizer.getIdentifier(); if (!objectDefined) { string errMsg = "An object has to be the first element defined in a config file"; throw ParserException(errMsg); } Token token = tokenizer.nextToken(); if(token == Token::EQUALS) { parseConfigFieldRhs(fieldName); } else { string errMsg = "Expected '=' after identifier, got: "; errMsg += getTokenName(token); throw ParserException(errMsg); } } void parseConfigFieldRhs(const StringView &fieldName) { Token token = tokenizer.nextToken(); if(token == Token::STRING) { callback->processField(fieldName, tokenizer.getString()); } else if(token == Token::OPEN_BRACKET) { parseConfigFieldRhsList(fieldName); } else { string errMsg = "Expected string on right-hand side of field '"; errMsg += string(fieldName.data, fieldName.size); errMsg += "', got: "; errMsg += getTokenName(token); throw ParserException(errMsg); } } void parseConfigFieldRhsList(const StringView &fieldName) { Token token = tokenizer.nextToken(); if (token == Token::CLOSING_BRACKET) return; vector values; while (true) { if (token == Token::STRING) values.push_back(tokenizer.getString()); token = tokenizer.nextToken(); if (token == Token::COMMA) { token = tokenizer.nextToken(); continue; } else if (token == Token::CLOSING_BRACKET) { break; } else { string errMsg = "Expected list value to be followed by ']' or ',', got: "; errMsg += getTokenName(token); throw ParserException(errMsg); } } callback->processField(fieldName, values); } void parseConfigObject() { Token token = tokenizer.nextToken(); if(token == Token::IDENTIFIER) { StringView objectName = tokenizer.getIdentifier(); token = tokenizer.nextToken(); if(token == Token::CLOSING_BRACKET) { objectDefined = true; callback->processObject(objectName); } else { string errMsg = "Expected ']' after identifier to close object definition, got: "; errMsg += getTokenName(token); throw ParserException(errMsg); } } else { string errMsg = "Expected identifier after '[', got: "; errMsg += getTokenName(token); throw ParserException(errMsg); } } private: Tokenizer tokenizer; ConfigCallback *callback; bool objectDefined; }; Result Config::readFromFile(const _tinydir_char_t *filepath, const ConfigCallback &callback) { Result fileContentResult = getFileContent(filepath); if(fileContentResult.isErr()) return Result::Err(fileContentResult.getErrMsg()); const char *code = fileContentResult.unwrap().data; if(!utf8::is_valid(code, code + fileContentResult.unwrap().size)) return Result::Err("File is not in valid utf8 format"); if(fileContentResult.unwrap().size >= 3 && utf8::is_bom(code)) code += 3; return Parser::parse(code, callback); } bool containsPlatform(const vector &platforms, Platform platform) { for (Platform vecPlatform : platforms) { if (vecPlatform == platform || vecPlatform == PLATFORM_ANY) return true; } return false; } const char* asString(Platform platform) { switch (platform) { case PLATFORM_ANY: return "any"; case PLATFORM_LINUX32: return "linux32"; case PLATFORM_LINUX64: return "linux64"; case PLATFORM_WIN32: return "win32"; case PLATFORM_WIN64: return "win64"; } } const char* asString(OptimizationLevel optLevel) { switch(optLevel) { case OPT_LEV_NONE: return "none"; case OPT_LEV_DEBUG: return "debug"; case OPT_LEV_RELEASE: return "release"; } } bool directoryToIgnore(const FileString &dir, const vector &ignoreDirList) { string dirUtf8 = toUtf8(dir); for(const string &ignoreDir : ignoreDirList) { if(pathEquals(dirUtf8, ignoreDir)) return true; } return false; } bool isProjectNameValid(const string &projectName) { for(int i = 0; i < projectName.size(); ++i) { char c = projectName[i]; if(!isalpha(c) && !isdigit(c) && c != '-' && c != '_') return false; } return true; } bool SibsConfig::isDefined(const std::string &name) const { return defines.find(name) != defines.end(); } bool SibsConfig::define(const std::string &name, const std::string &value) { if(isDefined(name)) return false; else { defines[name] = value; return true; } } const std::unordered_map& SibsConfig::getDefines() const { return defines; } void getLibFiles(const string &libPath, vector &outputFiles) { FileString nativePath = toFileString(libPath); FileType fileType = getFileType(nativePath.c_str()); switch (fileType) { case FileType::FILE_NOT_FOUND: { string errMsg = "Library path not found: "; errMsg += libPath; throw ParserException(errMsg); } case FileType::REGULAR: { string errMsg = "Expected library path "; errMsg += libPath; errMsg += " to be a directory, was a regular file"; throw ParserException(errMsg); } } walkDirFiles(nativePath.c_str(), [&outputFiles](tinydir_file *file) { if(_tinydir_strcmp(file->extension, CONFIG_STATIC_LIB_FILE_EXTENSION) == 0) outputFiles.push_back(toUtf8(file->path)); }); } void SibsConfig::processObject(StringView name) { currentObject = name; if(currentObject.equals("cmake") || currentObject.equals("cmake.static") || currentObject.equals("cmake.dynamic")) useCmake = true; //printf("Process object: %.*s\n", name.size, name.data); } void SibsConfig::processField(StringView name, const ConfigValue &value) { /* printf("Process field: %.*s, value: ", name.size, name.data); if(value.isSingle()) { printf("\"%.*s\"", value.asSingle().size, value.asSingle().data); } else { printf("["); int i = 0; for(auto listElement : value.asList()) { if(i > 0) printf(", "); printf("\"%.*s\"", listElement.size, listElement.data); ++i; } printf("]"); } printf("\n"); */ if(currentObject.equals("package")) { if(name.equals("name")) { if (value.isSingle()) packageName = string(value.asSingle().data, value.asSingle().size); else throw ParserException("Expected package.name to be a single value, was a list"); validatePackageName(); } else if(name.equals("version")) { // TODO: Use version for info output when building } else if(name.equals("authors")) { // TODO: Use authors for something? } else if(name.equals("type")) { if (value.isSingle()) { const StringView &packageTypeStr = value.asSingle(); if(packageTypeStr.equals("executable")) packageType = PackageType::EXECUTABLE; else if(packageTypeStr.equals("static")) packageType = PackageType::STATIC; else if(packageTypeStr.equals("dynamic")) packageType = PackageType::DYNAMIC; else if(packageTypeStr.equals("library")) packageType = PackageType::LIBRARY; else { string errMsg = "Expected package.type to be either 'executable', 'static', 'dynamic' or 'library', was: "; errMsg += string(packageTypeStr.data, packageTypeStr.size); throw ParserException(errMsg); } } else throw ParserException("Expected package.type to be a single value, was a list"); } else if(name.equals("tests")) { if (value.isSingle()) { testPath = projectPath; testPath += TINYDIR_STRING("/"); #if OS_FAMILY == OS_FAMILY_POSIX testPath += FileString(value.asSingle().data, value.asSingle().size); #else testPath += utf8To16(value.asSingle()); #endif Result testRealPathResult = getRealPath(testPath.c_str()); if(!testRealPathResult) { string errMsg = "Failed to resolve package.tests path: "; errMsg += testRealPathResult.getErrMsg(); throw ParserException(errMsg); } testPath = testRealPathResult.unwrap(); } else throw ParserException("Expected package.tests to be a single value, was a list"); } else if(name.equals("include_dirs")) { if(value.isList()) { // TODO: Checking for duplicate declaration should be done in the config parser if(!includeDirs.empty()) throw ParserException("Found duplicate declaration of package.include_dirs"); for(const StringView &includeDir : value.asList()) { includeDirs.emplace_back(string(includeDir.data, includeDir.size)); } } else throw ParserException("Expected package.include_dirs to be a list, was a single value"); } else if (name.equals("platforms")) { if (value.isList()) { // TODO: Checking for duplicate declaration should be done in the config parser if (!platforms.empty()) throw ParserException("Found duplicate declaration of package.platforms"); for (const StringView &platform : value.asList()) { if (platform.equals("any")) { platforms.push_back(PLATFORM_ANY); } else if (platform.equals("linux32")) { platforms.push_back(PLATFORM_LINUX32); } else if (platform.equals("linux64")) { platforms.push_back(PLATFORM_LINUX64); } else if (platform.equals("win32")) { platforms.push_back(PLATFORM_WIN32); } else if (platform.equals("win64")) { platforms.push_back(PLATFORM_WIN64); } else { string errMsg = "package.platforms contains invalid platform \""; errMsg += string(platform.data, platform.size); errMsg += "\". Expected platform to be one of: any, linux32, linux64, win32 or win64"; throw ParserException(errMsg); } } } else throw ParserException("Expected package.platforms to be a list, was a single value"); } else if(name.equals("ignore_dirs")) { if (value.isList()) { string projectPathUtf8 = toUtf8(projectPath); // TODO: Checking for duplicate declaration should be done in the config parser if (!ignoreDirs.empty()) throw ParserException("Found duplicate declaration of package.ignore_dirs"); for (const StringView &ignoreDir : value.asList()) { string ignoreDirFull = projectPathUtf8; ignoreDirFull += "/"; ignoreDirFull += string(ignoreDir.data, ignoreDir.size); ignoreDirs.emplace_back(ignoreDirFull); } } else throw ParserException("Expected package.ignore_dirs to be a list, was a single value"); } else failInvalidFieldUnderObject(name); } else if(currentObject.size >= 6 && strncmp(currentObject.data, "config", 6) == 0) { if(currentObject.size == 6) // [config] { if (name.equals("expose_include_dirs")) { if (value.isList()) { for (const StringView &includeDir : value.asList()) { exposeIncludeDirs.emplace_back(string(includeDir.data, includeDir.size)); } } else { string errMsg = "Expected "; errMsg += string(currentObject.data, currentObject.size); errMsg += " to be a list, was a single value"; throw ParserException(errMsg); } } else failInvalidFieldUnderObject(name); } else // [config.*] { parsePlatformConfigs(name, value); } } else if(currentObject.equals("dependencies")) { if(value.isSingle()) { // TODO: Validate version is number in correct format Dependency dependency; dependency.name = string(name.data, name.size); dependency.version = string(value.asSingle().data, value.asSingle().size); dependencies.emplace_back(dependency); } else throw ParserException("Expected field under dependencies to be a single value, was a list"); } else if(currentObject.equals("cmake")) { parseCmake(name, value, cmakeDirGlobal, cmakeArgsGlobal); } else if(currentObject.equals("cmake.static")) { parseCmake(name, value, cmakeDirStatic, cmakeArgsStatic); } else if(currentObject.equals("cmake.dynamic")) { parseCmake(name, value, cmakeDirDynamic, cmakeArgsDynamic); } else { string errMsg = "Invalid config object \""; errMsg += string(currentObject.data, currentObject.size); errMsg += "\""; throw ParserException(errMsg); } } void SibsConfig::parsePlatformConfigs(const StringView &fieldName, const ConfigValue &fieldValue) { for(int i = 0; i < NUM_CONFIGS; ++i) { const StringView &config = CONFIGS[i]; if(currentObject.equals(config)) { switch(i) { case CONFIG_SYSTEM_PLATFORM: return parsePlatformConfig(fieldName, fieldValue); case CONFIG_STATIC_DEBUG_PLATFORM: return parsePlatformConfigStaticDebug(fieldName, fieldValue); case CONFIG_STATIC_RELEASE_PLATFORM: return parsePlatformConfigStaticRelease(fieldName, fieldValue); default: return; } } } string errMsg = "Invalid config object \""; errMsg += string(currentObject.data, currentObject.size); errMsg += "\""; throw ParserException(errMsg); } void SibsConfig::parsePlatformConfig(const StringView &fieldName, const ConfigValue &fieldValue) { if (fieldName.equals("expose_include_dirs")) { if (fieldValue.isList()) { for (const StringView &includeDir : fieldValue.asList()) { exposeIncludeDirs.emplace_back(string(includeDir.data, includeDir.size)); } } else { string errMsg = "Expected "; errMsg += string(currentObject.data, currentObject.size); errMsg += " to be a list, was a single value"; throw ParserException(errMsg); } } else failInvalidFieldUnderObject(fieldName); } void SibsConfig::parsePlatformConfigStaticDebug(const StringView &fieldName, const ConfigValue &fieldValue) { if (fieldName.equals("lib")) { if (fieldValue.isSingle()) { string debugStaticLibPath = toUtf8(projectPath); debugStaticLibPath += "/"; debugStaticLibPath += string(fieldValue.asSingle().data, fieldValue.asSingle().size); getLibFiles(debugStaticLibPath, debugStaticLibs); } else { string errMsg = "Expected "; errMsg += string(currentObject.data, currentObject.size); errMsg += " to be a single value, was a list"; throw ParserException(errMsg); } } else failInvalidFieldUnderObject(fieldName); } void SibsConfig::parsePlatformConfigStaticRelease(const StringView &fieldName, const ConfigValue &fieldValue) { if (fieldName.equals("lib")) { if (fieldValue.isSingle()) { string releaseStaticLibPath = toUtf8(projectPath); releaseStaticLibPath += "/"; releaseStaticLibPath += string(fieldValue.asSingle().data, fieldValue.asSingle().size); getLibFiles(releaseStaticLibPath, releaseStaticLibs); } else { string errMsg = "Expected "; errMsg += string(currentObject.data, currentObject.size); errMsg += " to be a single value, was a list"; throw ParserException(errMsg); } } else failInvalidFieldUnderObject(fieldName); } void SibsConfig::parseCmake(const StringView &fieldName, const ConfigValue &fieldValue, string &cmakeDir, string &cmakeArgs) { if(fieldName.equals("dir")) { if(fieldValue.isSingle()) { cmakeDir = projectPath; cmakeDir += TINYDIR_STRING("/"); cmakeDir += toFileString(fieldValue.asSingle()); // No need to validate if CMakeLists.txt exists here, cmake will tell us if the file doesn't exist } else { string errMsg = "Expected "; errMsg.append(currentObject.data, currentObject.size); errMsg += "."; errMsg.append(fieldName.data, fieldName.size); errMsg += " to be a single value, was a list"; throw ParserException(errMsg); } } else if(fieldName.equals("args")) { if(fieldValue.isList()) { for(const StringView &arg : fieldValue.asList()) { bool prependSpace = !cmakeArgs.empty(); cmakeArgs.reserve(cmakeArgs.size() + 4 + (prependSpace ? 1 : 0) + arg.size); if(prependSpace) cmakeArgs += " "; cmakeArgs += "\"-D"; cmakeArgs.append(arg.data, arg.size); cmakeArgs += "\""; } } else { string errMsg = "Expected "; errMsg.append(currentObject.data, currentObject.size); errMsg += "."; errMsg.append(fieldName.data, fieldName.size); errMsg += " to be a list, was a single value"; throw ParserException(errMsg); } } else failInvalidFieldUnderObject(fieldName); } void SibsConfig::finished() { if((int)packageType == -1) throw ParserException("Missing required config package.type. Expected to be one either 'executable', 'static', 'dynamic' or 'library'"); finishedProcessing = true; if(platforms.empty()) throw ParserException("Missing required config package.platforms. If the package supports all platforms, add:\nplatforms = [\"any\"]\nto project.conf under [package]"); } void SibsConfig::validatePackageName() const { if(!isProjectNameValid(packageName)) { string errMsg = "Invalid package name: "; errMsg += packageName; errMsg += ". Package name can only contain alphanumerical characters, dash (-) or underscore (_)"; throw ParserException(errMsg); } } void SibsConfig::failInvalidFieldUnderObject(const StringView &fieldName) const { string errMsg = "Invalid field \""; errMsg += string(fieldName.data, fieldName.size); errMsg += "\" under object \""; errMsg += string(currentObject.data, currentObject.size); errMsg += "\""; throw ParserException(errMsg); } void SibsTestConfig::processObject(StringView name) { currentObject = name; } void SibsTestConfig::processField(StringView name, const ConfigValue &value) { if(currentObject.equals("dependencies")) { if(value.isSingle()) { // TODO: Validate version is number in correct format Dependency dependency; dependency.name = string(name.data, name.size); dependency.version = string(value.asSingle().data, value.asSingle().size); dependencies.emplace_back(dependency); } else throw ParserException("Expected field under dependencies to be a single value, was a list"); } else { string errMsg = "project.conf: Expected category to be 'dependencies', was: '"; errMsg += string(currentObject.data, currentObject.size); errMsg += "'"; throw ParserException(errMsg); } } void SibsTestConfig::finished() { finishedProcessing = true; } }