Merge a6323c7 into 0ca7912

lib/antlr/runtime/CMakeLists.txt

@@ -36,6 +36,7 @@ add_dependencies(antlr4_static make_lib_output_dir)
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
   target_link_libraries(antlr4_static ${UUID_LIBRARIES})
+  target_compile_options(antlr4_static PRIVATE "-fPIC")
 elseif(APPLE)
   target_link_libraries(antlr4_static ${COREFOUNDATION_LIBRARY})
 endif()
@@ -89,9 +90,9 @@ install(TARGETS antlr4_static
         DESTINATION lib
         EXPORT antlr4-targets)
 
-install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/" 
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/"
         DESTINATION "include/antlr4-runtime"
-        COMPONENT dev 
+        COMPONENT dev
         FILES_MATCHING PATTERN "*.h"
         )
 

pol-core/bscript/CMakeSources.cmake

@@ -27,6 +27,8 @@ set (bscript_sources    # sorted !
   compiler/analyzer/LocalVariableScopes.h
   compiler/analyzer/SemanticAnalyzer.cpp
   compiler/analyzer/SemanticAnalyzer.h
+  compiler/analyzer/SemanticTokensBuilder.cpp
+  compiler/analyzer/SemanticTokensBuilder.h
   compiler/analyzer/Variables.cpp
   compiler/analyzer/Variables.h
   compiler/ast/Argument.cpp
@@ -226,6 +228,8 @@ set (bscript_sources    # sorted !
   compiler/file/SourceFileCache.h
   compiler/file/SourceFileIdentifier.cpp
   compiler/file/SourceFileIdentifier.h
+  compiler/file/SourceFileLoader.cpp
+  compiler/file/SourceFileLoader.h
   compiler/file/SourceLocation.cpp
   compiler/file/SourceLocation.h
   compiler/format/CompiledScriptSerializer.cpp
@@ -243,6 +247,10 @@ set (bscript_sources    # sorted !
   compiler/model/FunctionLink.cpp
   compiler/model/FunctionLink.h
   compiler/model/LocalVariableScopeInfo.h
+  compiler/model/SemanticTokens.cpp
+  compiler/model/SemanticTokens.h
+  compiler/model/ScopeTree.cpp
+  compiler/model/ScopeTree.h
   compiler/model/SimpleTypes.h
   compiler/model/UserFunctionInclusion.h
   compiler/model/Variable.cpp

pol-core/bscript/compiler/Compiler.cpp

@@ -1,30 +1,34 @@
 #include "Compiler.h"
 
-#include "clib/fileutil.h"
-#include "clib/logfacility.h"
-#include "clib/timer.h"
 #include "bscript/compiler/Profile.h"
 #include "bscript/compiler/Report.h"
 #include "bscript/compiler/analyzer/Disambiguator.h"
 #include "bscript/compiler/analyzer/SemanticAnalyzer.h"
+#include "bscript/compiler/analyzer/SemanticTokensBuilder.h"
 #include "bscript/compiler/astbuilder/CompilerWorkspaceBuilder.h"
 #include "bscript/compiler/codegen/CodeGenerator.h"
 #include "bscript/compiler/file/SourceFileCache.h"
 #include "bscript/compiler/file/SourceFileIdentifier.h"
+#include "bscript/compiler/file/SourceFileLoader.h"
 #include "bscript/compiler/format/CompiledScriptSerializer.h"
 #include "bscript/compiler/format/DebugStoreSerializer.h"
 #include "bscript/compiler/format/ListingWriter.h"
 #include "bscript/compiler/model/CompilerWorkspace.h"
 #include "bscript/compiler/optimizer/Optimizer.h"
 #include "bscript/compiler/representation/CompiledScript.h"
+#include "clib/fileutil.h"
+#include "clib/logfacility.h"
+#include "clib/timer.h"
 #include "compilercfg.h"
 
 namespace Pol::Bscript::Compiler
 {
-Compiler::Compiler( SourceFileCache& em_cache, SourceFileCache& inc_cache, Profile& profile )
-  : em_cache( em_cache ),
-    inc_cache( inc_cache ),
-    profile( profile )
+Compiler::Compiler( SourceFileLoader& source_loader, SourceFileCache& em_cache,
+                    SourceFileCache& inc_cache, Profile& profile )
+    : source_loader( source_loader ),
+      em_cache( em_cache ),
+      inc_cache( inc_cache ),
+      profile( profile )
 {
 }
 
@@ -87,8 +91,13 @@ bool Compiler::compile_file( const std::string& filename )
   try
   {
     auto pathname = Clib::FullPath( filename.c_str() );
+    if ( pathname.empty() )
+    {
+      pathname = filename;
+    }
 
-    Report report( compilercfg.DisplayWarnings || compilercfg.ErrorOnWarning );
+    ConsoleReporter reporter( compilercfg.DisplayWarnings || compilercfg.ErrorOnWarning );
+    Report report( reporter );
 
     compile_file_steps( pathname, report );
     display_outcome( pathname, report );
@@ -106,7 +115,7 @@ bool Compiler::compile_file( const std::string& filename )
 
 void Compiler::compile_file_steps( const std::string& pathname, Report& report )
 {
-  std::unique_ptr<CompilerWorkspace> workspace = build_workspace( pathname, report );
+  std::unique_ptr<CompilerWorkspace> workspace = build_workspace( pathname, report, false );
   if ( report.error_count() )
     return;
 
@@ -126,15 +135,33 @@ void Compiler::compile_file_steps( const std::string& pathname, Report& report )
   if ( report.error_count() )
     return;
 
+  tokenize( *workspace, report );
+  if ( report.error_count() )
+    return;
+
   output = generate( std::move( workspace ) );
 }
 
+std::unique_ptr<CompilerWorkspace> Compiler::precompile( const std::string& pathname,
+                                                         Report& report, bool is_module )
+{
+  // Let's see how this explodes...
+  std::unique_ptr<CompilerWorkspace> workspace = build_workspace( pathname, report, is_module );
+  register_constants( *workspace, report );
+  optimize( *workspace, report );
+  disambiguate( *workspace, report );
+  analyze( *workspace, report );
+  tokenize( *workspace, report );
+  return workspace;
+}
+
 std::unique_ptr<CompilerWorkspace> Compiler::build_workspace( const std::string& pathname,
-                                                              Report& report )
+                                                              Report& report, bool is_module )
 {
   Pol::Tools::HighPerfTimer timer;
-  CompilerWorkspaceBuilder workspace_builder( em_cache, inc_cache, profile, report );
-  auto workspace = workspace_builder.build( pathname, user_function_inclusion );
+  CompilerWorkspaceBuilder workspace_builder( source_loader, em_cache, inc_cache, profile, report );
+  auto workspace = is_module ? workspace_builder.build_module( pathname )
+                             : workspace_builder.build( pathname, user_function_inclusion );
   profile.build_workspace_micros += timer.ellapsed().count();
   return workspace;
 }
@@ -170,6 +197,14 @@ void Compiler::analyze( CompilerWorkspace& workspace, Report& report )
   profile.analyze_micros += timer.ellapsed().count();
 }
 
+void Compiler::tokenize( CompilerWorkspace& workspace, Report& report )
+{
+  Pol::Tools::HighPerfTimer timer;
+  SemanticTokensBuilder tokenizer( workspace, report );
+  tokenizer.build();
+  profile.tokenize_micros += timer.ellapsed().count();
+}
+
 std::unique_ptr<CompiledScript> Compiler::generate( std::unique_ptr<CompilerWorkspace> workspace )
 {
   Pol::Tools::HighPerfTimer codegen_timer;

pol-core/bscript/compiler/Compiler.h

@@ -11,13 +11,15 @@ namespace Pol::Bscript::Compiler
 class CompiledScript;
 class CompilerWorkspace;
 class SourceFileCache;
+class SourceFileLoader;
 class Profile;
 class Report;
 
 class Compiler
 {
 public:
-  Compiler( SourceFileCache& em_cache, SourceFileCache& inc_cache, Profile& );
+  Compiler( SourceFileLoader& source_loader, SourceFileCache& em_cache, SourceFileCache& inc_cache,
+            Profile& );
   ~Compiler();
   Compiler( const Compiler& ) = delete;
   Compiler& operator=( const Compiler& ) = delete;
@@ -30,17 +32,21 @@ class Compiler
   void set_include_compile_mode();
 
   void compile_file_steps( const std::string& pathname, Report& );
+  std::unique_ptr<CompilerWorkspace> precompile( const std::string& pathname, Report&,
+                                                 bool is_module );
 
 private:
-  std::unique_ptr<CompilerWorkspace> build_workspace( const std::string&, Report& );
+  std::unique_ptr<CompilerWorkspace> build_workspace( const std::string&, Report&, bool );
   void register_constants( CompilerWorkspace&, Report& );
   void optimize( CompilerWorkspace&, Report& );
   void disambiguate( CompilerWorkspace&, Report& );
   void analyze( CompilerWorkspace&, Report& );
+  void tokenize( CompilerWorkspace&, Report& );
   std::unique_ptr<CompiledScript> generate( std::unique_ptr<CompilerWorkspace> );
 
   void display_outcome( const std::string& filename, Report& );
 
+  SourceFileLoader& source_loader;
   SourceFileCache& em_cache;
   SourceFileCache& inc_cache;
   Profile& profile;

pol-core/bscript/compiler/Profile.h

@@ -13,6 +13,7 @@ class Profile
   std::atomic<long long> optimize_micros;
   std::atomic<long long> disambiguate_micros;
   std::atomic<long long> analyze_micros;
+  std::atomic<long long> tokenize_micros;
   std::atomic<long long> codegen_micros;
   std::atomic<long long> prune_cache_select_micros;
   std::atomic<long long> prune_cache_delete_micros;

pol-core/bscript/compiler/Report.cpp

@@ -5,33 +5,76 @@
 
 namespace Pol::Bscript::Compiler
 {
-Report::Report( bool display_warnings )
-  : display_warnings( display_warnings ), errors( 0 ), warnings( 0 )
+ConsoleReporter::ConsoleReporter( bool display_warnings )
+    : ErrorReporter(), display_warnings( display_warnings )
 {
 }
 
-void Report::report_error( const SourceLocation& source_location, const char* msg )
+void ConsoleReporter::report_error( const SourceLocation& source_location, const std::string& msg )
 {
-  ++errors;
   try
   {
-    ERROR_PRINT << source_location << ": error: " << msg;
-  } catch (...)
+    ERROR_PRINT << source_location << ": error: " << msg << "\n";
+  }
+  catch ( ... )
   {
   }
 }
 
-void Report::report_warning( const SourceLocation& source_location, const char* msg )
+void ConsoleReporter::report_warning( const SourceLocation& source_location,
+                                      const std::string& msg )
 {
-  ++warnings;
-  try
-  {
-    ERROR_PRINT << source_location << ": warning: " << msg;
-  } catch (...)
+  if ( display_warnings )
   {
+    try
+    {
+      ERROR_PRINT << source_location << ": warning: " << msg << "\n";
+    }
+    catch ( ... )
+    {
+    }
   }
 }
 
+void ConsoleReporter::clear() {}
+
+void DiagnosticReporter::report_error( const SourceLocation& source_location,
+                                       const std::string& msg )
+{
+  diagnostics.push_back( Diagnostic{ Diagnostic::Severity::Error, source_location, msg } );
+}
+
+void DiagnosticReporter::report_warning( const SourceLocation& source_location,
+                                         const std::string& msg )
+{
+  diagnostics.push_back( Diagnostic{ Diagnostic::Severity::Warning, source_location, msg } );
+}
+
+void DiagnosticReporter::clear()
+{
+  diagnostics.clear();
+}
+
+Report::Report( ErrorReporter& reporter ) : reporter( reporter ), errors( 0 ), warnings( 0 ) {}
+
+void Report::report_error( const SourceLocation& source_location, const std::string& msg )
+{
+  ++errors;
+  reporter.report_error( source_location, msg );
+}
+
+void Report::report_warning( const SourceLocation& source_location, const std::string& msg )
+{
+  ++warnings;
+  reporter.report_warning( source_location, msg );
+}
+
+void Report::clear()
+{
+  warnings = errors = 0;
+  reporter.clear();
+}
+
 unsigned Report::error_count() const
 {
   return errors;