Use SourceFile token_stream to bypass lexer reset

polserver · Feb 3, 2024 · 2708801 · 2708801
1 parent 082bdda
commit 2708801
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 11 deletions.
diff --git a/pol-core/bscript/compiler/file/SourceFile.cpp b/pol-core/bscript/compiler/file/SourceFile.cpp
@@ -189,11 +189,10 @@ std::vector<antlr4::Token*> SourceFile::get_hidden_tokens_before( const Position
 
 std::vector<antlr4::Token*> SourceFile::get_hidden_tokens_before( size_t tokenIndex )
 {
-  token_stream.reset();
   return token_stream.getHiddenTokensToLeft( tokenIndex );
 }
 
-std::unique_ptr<antlr4::Token> SourceFile::get_token_at( const Position& position )
+antlr4::Token* SourceFile::get_token_at( const Position& position )
 {
   auto tokens = get_all_tokens();
   auto result =
@@ -207,22 +206,21 @@ std::unique_ptr<antlr4::Token> SourceFile::get_token_at( const Position& positio
                     } );
   if ( result != tokens.end() )
   {
-    return std::move( *result );
+    return *result;
   }
-  return {};
+  return nullptr;
 }
 
-std::vector<std::unique_ptr<antlr4::Token>> SourceFile::get_all_tokens()
+std::vector<antlr4::Token*> SourceFile::get_all_tokens()
 {
-  lexer.reset();
-  return lexer.getAllTokens();
+  return token_stream.getTokens();
 }
 
 SemanticTokens SourceFile::get_tokens()
 {
   SemanticTokens tokens;
-  lexer.reset();
-  for ( const auto& lexer_token : lexer.getAllTokens() )
+
+  for ( auto* lexer_token : token_stream.getTokens() )
   {
     auto semantic_token = SemanticToken::from_lexer_token( *lexer_token );
     if ( semantic_token )

diff --git a/pol-core/bscript/compiler/file/SourceFile.h b/pol-core/bscript/compiler/file/SourceFile.h
@@ -50,8 +50,8 @@ class SourceFile
   EscriptGrammar::EscriptParser::EvaluateUnitContext* get_evaluate_unit( Report& );
 
   SemanticTokens get_tokens();
-  std::unique_ptr<antlr4::Token> get_token_at( const Position& position );
-  std::vector<std::unique_ptr<antlr4::Token>> get_all_tokens();
+  antlr4::Token* get_token_at( const Position& position );
+  std::vector<antlr4::Token*> get_all_tokens();
   std::vector<antlr4::Token*> get_hidden_tokens_before( const Position& position );
   std::vector<antlr4::Token*> get_hidden_tokens_before( size_t tokenIndex );