LineLexer.Read() now assumes there is an arena.

I thought that this would make the Read() function faster, but it doesn't show up when parsing abuild (benchmarks/pytrace.sh time-parse-build.)
oilshell · Dec 21, 2017 · 4e8b9f15bcce390a420dc4d62d0b9f4c1f96a25b · 4e8b9f1
1 parent 7810a62
commit 4e8b9f15bcce390a420dc4d62d0b9f4c1f96a25b
Unified Split

Showing with 6 additions and 11 deletions.

+6 −11 core/lexer.py
diff --git a/core/lexer.py b/core/lexer.py
@@ -121,18 +121,13 @@ def Read(self, lex_mode):
    # want it to be "low level".  The only thing fabricated here is a newline
    # added at the last line, so we don't end with \0.

-    if self.arena is not None:
-      if self.arena_skip:
-        assert self.last_span_id != const.NO_INTEGER
-        span_id = self.last_span_id
-        self.arena_skip = False
-      else:
-        span_id = self.arena.AddLineSpan(line_span)
-        self.last_span_id = span_id
+    if self.arena_skip:
+      assert self.last_span_id != const.NO_INTEGER
+      span_id = self.last_span_id
+      self.arena_skip = False
    else:
-      # Completion parser might not have arena?
-      # We should probably get rid of this.
-      span_id = const.NO_INTEGER
+      span_id = self.arena.AddLineSpan(line_span)
+      self.last_span_id = span_id

    t = ast.token(tok_type, tok_val, span_id)