diff --git a/.run/spice.run.xml b/.run/spice.run.xml index cd31dddf0..542c9d043 100644 --- a/.run/spice.run.xml +++ b/.run/spice.run.xml @@ -1,5 +1,5 @@ - + diff --git a/media/test-project/os-test.spice b/media/test-project/os-test.spice index 820b51ba7..b9916b1e4 100644 --- a/media/test-project/os-test.spice +++ b/media/test-project/os-test.spice @@ -1,8 +1,9 @@ import "os-test2" as s1; f main() { - dyn s = Vector{Vector1{123}}; - printf("Result: %d\n", s.i1.i2); + //dyn s = Vector{Vector1{123}}; + //printf("Result: %d\n", s.i1.i2); + printf("Test"); } /*f isValid(int input = 12) { diff --git a/setup-libs.bat b/setup-libs.bat index c806a2b93..1c5e8f7a1 100644 --- a/setup-libs.bat +++ b/setup-libs.bat @@ -10,6 +10,9 @@ git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest mkdir json curl -SsL "https://github.com/nlohmann/json/releases/download/v3.11.2/json.hpp" --output json/json.hpp +mkdir thread-pool +curl -SsL "https://raw.githubusercontent.com/bshoshany/thread-pool/master/BS_thread_pool.hpp" --output thread-pool/thread-pool.hpp + mkdir cli11 curl -SsL "https://github.com/spicelang/CLI11/releases/download/v2.3.0-spice/CLI11.hpp" --output cli11/CLI11.hpp diff --git a/setup-libs.sh b/setup-libs.sh index 4059a291a..62a519a9c 100644 --- a/setup-libs.sh +++ b/setup-libs.sh @@ -10,6 +10,9 @@ git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest mkdir json curl -SsL "https://github.com/nlohmann/json/releases/download/v3.11.2/json.hpp" --output json/json.hpp +mkdir thread-pool +curl -SsL "https://raw.githubusercontent.com/bshoshany/thread-pool/master/BS_thread_pool.hpp" --output thread-pool/thread-pool.hpp + mkdir cli11 curl -SsL "https://github.com/spicelang/CLI11/releases/download/v2.3.0-spice/CLI11.hpp" --output cli11/CLI11.hpp diff --git a/src/SourceFile.cpp b/src/SourceFile.cpp index 8cbf93dbe..eb721a07f 100644 --- a/src/SourceFile.cpp +++ b/src/SourceFile.cpp @@ -25,7 +25,8 @@ SourceFile::SourceFile(GlobalResourceManager &resourceManager, SourceFile *parent, std::string name, const std::string &filePath, bool stdFile) - : resourceManager(resourceManager), name(std::move(name)), filePath(filePath), stdFile(stdFile), parent(parent) { + : resourceManager(resourceManager), tout(resourceManager.tout), name(std::move(name)), filePath(filePath), stdFile(stdFile), + parent(parent) { this->objectFilePath = resourceManager.cliOptions.outputDir + FileUtil::DIR_SEPARATOR + FileUtil::getFileName(filePath) + ".o"; // Deduce fileName and fileDir @@ -317,7 +318,7 @@ void SourceFile::runEscapeAnalyzer() { // NOLINT(misc-no-recursion) printStatusMessage("Escape Analyzer", IO_AST, IO_AST, &timer, compilerOutput.times.escapeAnalyzer); } -void SourceFile::runIRGenerator() { // NOLINT(misc-no-recursion) +void SourceFile::runIRGenerator() { // Skip if restored from cache if (restoredFromCache) return; @@ -325,10 +326,6 @@ void SourceFile::runIRGenerator() { // NOLINT(misc-no-recursion) Timer timer; timer.start(); - // Generate the imported source files - for (const auto &sourceFile : dependencies) - sourceFile.second.first->runIRGenerator(); - // Create LLVM module for this source file llvmModule = std::make_unique(FileUtil::getFileName(filePath), resourceManager.context); @@ -340,17 +337,14 @@ void SourceFile::runIRGenerator() { // NOLINT(misc-no-recursion) compilerOutput.irString = irGenerator.getIRString(); // Dump unoptimized IR code - if (resourceManager.cliOptions.dumpIR) { // GCOV_EXCL_START - std::cout << "\nUnoptimized IR code:\n"; - irGenerator.dumpIR(); - std::cout << "\n"; - } // GCOV_EXCL_STOP + if (resourceManager.cliOptions.dumpIR) // GCOV_EXCL_LINE + tout.println("\nUnoptimized IR code:\n" + irGenerator.getIRString()); // GCOV_EXCL_LINE timer.stop(); - printStatusMessage("IR Generator", IO_AST, IO_IR, &timer, compilerOutput.times.irGenerator); + printStatusMessage("IR Generator", IO_AST, IO_IR, &timer, compilerOutput.times.irGenerator, true); } -void SourceFile::runIROptimizer() { // NOLINT(misc-no-recursion) +void SourceFile::runIROptimizer() { // Skip if restored from cache if (restoredFromCache) return; @@ -362,10 +356,6 @@ void SourceFile::runIROptimizer() { // NOLINT(misc-no-recursion) if (resourceManager.cliOptions.optLevel < 1 || resourceManager.cliOptions.optLevel > 5) return; - // Optimize the imported source files - for (const auto &sourceFile : dependencies) - sourceFile.second.first->runIROptimizer(); - // Optimize this source file IROptimizer irOptimizer(resourceManager, this); irOptimizer.optimize(); @@ -374,17 +364,14 @@ void SourceFile::runIROptimizer() { // NOLINT(misc-no-recursion) compilerOutput.irOptString = irOptimizer.getOptimizedIRString(); // Dump optimized IR code - if (resourceManager.cliOptions.dumpIR) { // GCOV_EXCL_START - std::cout << "\nOptimized IR code:\n"; - irOptimizer.dumpOptimizedIR(); - std::cout << "\n"; - } // GCOV_EXCL_STOP + if (resourceManager.cliOptions.dumpIR) // GCOV_EXCL_LINE + tout.println("\nOptimized IR code:\n" + irOptimizer.getOptimizedIRString()); // GCOV_EXCL_LINE timer.stop(); - printStatusMessage("IR Optimizer", IO_IR, IO_IR, &timer, compilerOutput.times.irOptimizer); + printStatusMessage("IR Optimizer", IO_IR, IO_IR, &timer, compilerOutput.times.irOptimizer, true); } -void SourceFile::runObjectEmitter() { // NOLINT(misc-no-recursion) +void SourceFile::runObjectEmitter() { // Skip if restored from cache if (restoredFromCache) return; @@ -392,42 +379,35 @@ void SourceFile::runObjectEmitter() { // NOLINT(misc-no-recursion) Timer timer; timer.start(); - // Emit objects for the imported source files - for (const auto &sourceFile : dependencies) - sourceFile.second.first->runIROptimizer(); - // Emit object for this source file ObjectEmitter objectEmitter(resourceManager, this); objectEmitter.emit(); // Dump assembly code if (resourceManager.cliOptions.dumpAssembly) { // GCOV_EXCL_START - std::cout << "\nAssembly code:\n"; + tout.println("\nAssembly code:\n"); objectEmitter.dumpAsm(); } // GCOV_EXCL_STOP timer.stop(); - printStatusMessage("Object Emitter", IO_IR, IO_OBJECT_FILE, &timer, compilerOutput.times.objectEmitter); + printStatusMessage("Object Emitter", IO_IR, IO_OBJECT_FILE, &timer, compilerOutput.times.objectEmitter, true); } -void SourceFile::concludeCompilation() { // NOLINT(misc-no-recursion) - // Conclude all dependencies - for (const auto &sourceFile : dependencies) - sourceFile.second.first->concludeCompilation(); - +void SourceFile::concludeCompilation() { // Cache the source file if (!resourceManager.cliOptions.ignoreCache) resourceManager.cacheManager.cacheSourceFile(this); // Print warning if verifier is disabled if (parent == nullptr && resourceManager.cliOptions.disableVerifier) { - std::cout << "\n"; - CompilerWarning(VERIFIER_DISABLED, "The LLVM verifier passes are disabled. Please use this cli option carefully.").print(); - std::cout << "\n"; + const std::string warningMessage = + CompilerWarning(VERIFIER_DISABLED, "The LLVM verifier passes are disabled. Please use this cli option carefully.") + .warningMessage; + tout.println("\n" + warningMessage); } if (resourceManager.cliOptions.printDebugOutput) - std::cout << "Finished compiling " << fileName << ".\n"; + tout.println("Finished compiling " + fileName); } void SourceFile::runFrontEnd() { // NOLINT(misc-no-recursion) @@ -448,11 +428,22 @@ void SourceFile::runMiddleEnd() { runEscapeAnalyzer(); } -void SourceFile::runBackEnd() { - runIRGenerator(); - runIROptimizer(); - runObjectEmitter(); - concludeCompilation(); +void SourceFile::runBackEnd() { // NOLINT(misc-no-recursion) + // Run backend for all dependencies first + for (const auto &[importName, sourceFile] : dependencies) + sourceFile.first->runBackEnd(); + + // Submit source file compilation to the task queue + resourceManager.threadPool.push_task([&]() { + runIRGenerator(); + runIROptimizer(); + runObjectEmitter(); + concludeCompilation(); + }); + + // Wait until all compile tasks are done + if (mainFile) + resourceManager.threadPool.wait_for_tasks(); } std::shared_ptr SourceFile::createSourceFile(const std::string &dependencyName, const std::string &path, @@ -527,7 +518,6 @@ const NameRegistryEntry *SourceFile::getNameRegistryEntry(std::string symbolName void SourceFile::mergeNameRegistries(const SourceFile &importedSourceFile, const std::string &importName) { for (const auto &[originalName, entry] : importedSourceFile.exportedNameRegistry) { // Add fully qualified name - const bool isFunction = entry.targetEntry == nullptr || entry.targetEntry->getType().isOneOf({TY_FUNCTION, TY_PROCEDURE}); const std::string newName = importName + "::" + originalName; exportedNameRegistry.insert({newName, {newName, entry.targetEntry, entry.targetScope}}); // Add the shortened name, considering the name collision @@ -564,14 +554,26 @@ void SourceFile::visualizerOutput(std::string outputName, const std::string &out } void SourceFile::printStatusMessage(const std::string &stage, const CompilerStageIOType &in, const CompilerStageIOType &out, - const Timer *timer, uint64_t &timeCompilerOutput) const { + const Timer *timer, uint64_t &timeCompilerOutput, bool fromThread /*=false*/) const { if (resourceManager.cliOptions.printDebugOutput) { const char *const compilerStageIoTypeName[] = {"Code", "Tokens", "CST", "AST", "IR", "OBJECT_FILE"}; - std::cout << "[" << stage << "] for " << fileName << ": "; - std::cout << compilerStageIoTypeName[in] << " --> " << compilerStageIoTypeName[out]; - if (timer != nullptr) { - timeCompilerOutput = timer->getDurationMilliseconds(); - std::cout << " (" << std::to_string(timeCompilerOutput) << " s)\n"; + if (fromThread) { + if (timer != nullptr) { + tout.println("[" + stage + "] for " + fileName + ": " + compilerStageIoTypeName[in] + " --> " + + compilerStageIoTypeName[out] + " (" + std::to_string(timeCompilerOutput) + " s)"); + } else { + tout.println("[" + stage + "] for " + fileName + ": " + compilerStageIoTypeName[in] + " --> " + + compilerStageIoTypeName[out]); + } + } else { + if (timer != nullptr) { + std::cout << "[" << stage << "] for " << fileName << ": "; + std::cout << compilerStageIoTypeName[in] << " --> " << compilerStageIoTypeName[out]; + std::cout << " (" << std::to_string(timeCompilerOutput) << " s)\n"; + } else { + std::cout << "[" << stage << "] for " << fileName << ": "; + std::cout << compilerStageIoTypeName[in] << " --> " << compilerStageIoTypeName[out] << "\n"; + } } } } \ No newline at end of file diff --git a/src/SourceFile.h b/src/SourceFile.h index 1276482e2..242738123 100644 --- a/src/SourceFile.h +++ b/src/SourceFile.h @@ -18,6 +18,8 @@ #include +#include "../lib/thread-pool/thread-pool.hpp" + // Forward declarations class GlobalResourceManager; class EntryNode; @@ -149,11 +151,12 @@ class SourceFile { private: // Private fields GlobalResourceManager &resourceManager; + BS::synced_stream &tout; // Private methods void mergeNameRegistries(const SourceFile &importedSourceFile, const std::string &importName); void visualizerPreamble(std::stringstream &output) const; void visualizerOutput(std::string outputName, const std::string &output) const; void printStatusMessage(const std::string &stage, const CompilerStageIOType &in, const CompilerStageIOType &out, - const Timer *timer, uint64_t &timeCompilerOutput) const; + const Timer *timer, uint64_t &timeCompilerOutput, bool fromThread = false) const; }; \ No newline at end of file diff --git a/src/cli/CLIInterface.cpp b/src/cli/CLIInterface.cpp index afe8b48ba..919471b92 100644 --- a/src/cli/CLIInterface.cpp +++ b/src/cli/CLIInterface.cpp @@ -244,6 +244,8 @@ void CLIInterface::addCompileSubcommandOptions(CLI::App *subCmd) { // --dump-assembly subCmd->add_flag("--dump-assembly,-asm,-s", cliOptions.dumpAssembly, "Dump Assembly code"); + // --jobs + subCmd->add_option("--jobs,-j", cliOptions.compileJobCount, "Compile jobs (threads), used for compilation"); // --ignore-cache subCmd->add_option("--ignore-cache", cliOptions.ignoreCache, "Force re-compilation of all source files"); // --disable-ast-opt diff --git a/src/cli/CLIInterface.h b/src/cli/CLIInterface.h index 5dfa05431..6a19eedc8 100644 --- a/src/cli/CLIInterface.h +++ b/src/cli/CLIInterface.h @@ -16,9 +16,10 @@ struct CliOptions { std::string targetVendor; std::string targetOs; bool isNativeTarget = false; - std::string cacheDir; // Where the cache files go. Should always be a temp directory - std::string outputDir; // Where the object files go. Should always be a temp directory - std::string outputPath; // Where the output binary goes. + std::string cacheDir; // Where the cache files go. Should always be a temp directory + std::string outputDir; // Where the object files go. Should always be a temp directory + std::string outputPath; // Where the output binary goes. + unsigned short compileJobCount = 0; // 0 for auto bool ignoreCache = false; bool printDebugOutput = false; bool dumpCST = false; diff --git a/src/global/GlobalResourceManager.cpp b/src/global/GlobalResourceManager.cpp index 5e5d835cb..1a735ab83 100644 --- a/src/global/GlobalResourceManager.cpp +++ b/src/global/GlobalResourceManager.cpp @@ -11,17 +11,17 @@ GlobalResourceManager::GlobalResourceManager(const CliOptions &cliOptions) : cliOptions(cliOptions), linker(threadFactory, cliOptions), cacheManager(cliOptions.cacheDir) { // Initialize the required LLVM targets - //llvm::InitializeAllTargetInfos(); if (cliOptions.isNativeTarget) { llvm::InitializeNativeTarget(); llvm::InitializeNativeTargetAsmParser(); llvm::InitializeNativeTargetAsmPrinter(); } else { llvm::InitializeAllTargets(); + llvm::InitializeAllTargetInfos(); + llvm::InitializeAllTargetMCs(); llvm::InitializeAllAsmParsers(); llvm::InitializeAllAsmPrinters(); } - //llvm::InitializeAllTargetMCs(); // Search after selected target std::string error; diff --git a/src/global/GlobalResourceManager.h b/src/global/GlobalResourceManager.h index e2f286487..db51817a5 100644 --- a/src/global/GlobalResourceManager.h +++ b/src/global/GlobalResourceManager.h @@ -9,8 +9,11 @@ #include #include +#include #include +#include "../../lib/thread-pool/thread-pool.hpp" + // Forward declarations struct CliOptions; @@ -32,4 +35,7 @@ class GlobalResourceManager { llvm::LLVMContext context; llvm::IRBuilder<> builder = llvm::IRBuilder<>(context); llvm::TargetMachine *targetMachine; + BS::thread_pool threadPool = BS::thread_pool(cliOptions.compileJobCount); + BS::synced_stream tout; + std::mutex objectEmitLock; }; \ No newline at end of file diff --git a/src/objectemitter/ObjectEmitter.cpp b/src/objectemitter/ObjectEmitter.cpp index 08ca05c83..80225a7d2 100644 --- a/src/objectemitter/ObjectEmitter.cpp +++ b/src/objectemitter/ObjectEmitter.cpp @@ -10,32 +10,44 @@ void ObjectEmitter::emit() const { const std::string &objectFile = sourceFile->objectFilePath; + // Lock the mutex + resourceManager.objectEmitLock.lock(); + // GCOV_EXCL_START if (cliOptions.printDebugOutput && cliOptions.dumpAssembly) - std::cout << "\nEmitting object file for triplet '" << cliOptions.targetTriple << "' to path: " << objectFile << "\n"; + resourceManager.tout.println("\nEmitting object file for triplet '" + cliOptions.targetTriple + "' to path: " + objectFile); // GCOV_EXCL_STOP // Open file output stream std::error_code errorCode; - llvm::raw_fd_ostream dest(objectFile, errorCode, llvm::sys::fs::OF_None); - if (errorCode) + llvm::raw_fd_ostream stream(objectFile, errorCode, llvm::sys::fs::OF_None); + if (errorCode) // GCOV_EXCL_LINE throw IRError(CANT_OPEN_OUTPUT_FILE, "File '" + objectFile + "' could not be opened"); // GCOV_EXCL_LINE llvm::legacy::PassManager passManager; - if (resourceManager.targetMachine->addPassesToEmitFile(passManager, dest, nullptr, llvm::CGFT_ObjectFile)) - throw IRError(WRONG_TYPE, "Target machine can't emit a file of this type"); // GCOV_EXCL_LINE + // GCOV_EXCL_START + if (resourceManager.targetMachine->addPassesToEmitFile(passManager, stream, nullptr, llvm::CGFT_ObjectFile, + cliOptions.disableVerifier)) + throw IRError(WRONG_TYPE, "Target machine can't emit a file of this type"); + // GCOV_EXCL_STOP // Emit object file - passManager.run(*module); - dest.flush(); + passManager.run(module); + stream.flush(); + + // Unlock the mutex + resourceManager.objectEmitLock.unlock(); } void ObjectEmitter::dumpAsm() const { llvm::legacy::PassManager passManager; - if (resourceManager.targetMachine->addPassesToEmitFile(passManager, llvm::outs(), nullptr, llvm::CGFT_AssemblyFile)) - throw IRError(WRONG_TYPE, "Target machine can't emit a file of this type"); // GCOV_EXCL_LINE + // GCOV_EXCL_START + if (resourceManager.targetMachine->addPassesToEmitFile(passManager, llvm::outs(), nullptr, llvm::CGFT_AssemblyFile, + cliOptions.disableVerifier)) + throw IRError(WRONG_TYPE, "Target machine can't emit a file of this type"); + // GCOV_EXCL_STOP // Emit object file - passManager.run(*module); + passManager.run(module); llvm::outs().flush(); } \ No newline at end of file diff --git a/src/objectemitter/ObjectEmitter.h b/src/objectemitter/ObjectEmitter.h index afba2004c..a9ab758f9 100644 --- a/src/objectemitter/ObjectEmitter.h +++ b/src/objectemitter/ObjectEmitter.h @@ -10,7 +10,7 @@ class ObjectEmitter : private CompilerPass { public: // Constructors ObjectEmitter(GlobalResourceManager &resourceManager, SourceFile *sourceFile) - : CompilerPass(resourceManager, sourceFile), module(sourceFile->llvmModule.get()) {} + : CompilerPass(resourceManager, sourceFile), module(*sourceFile->llvmModule) {} // Public methods void emit() const; @@ -18,5 +18,5 @@ class ObjectEmitter : private CompilerPass { private: // Private members - llvm::Module *module; + llvm::Module &module; }; \ No newline at end of file diff --git a/test/TestRunner.cpp b/test/TestRunner.cpp index 99502bf6e..1426b98ac 100644 --- a/test/TestRunner.cpp +++ b/test/TestRunner.cpp @@ -36,6 +36,7 @@ void execTestCase(const TestCase &testCase) { /* cacheDir= */ "./cache", /* outputDir= */ ".", /* outputPath= */ ".", + /* compileJobCount= */ 0, /* ignoreCache */ true, /* printDebugOutput= */ false, /* dumpCST= */ false,