Refactoring: arena is no longer optional.

We could have a NullArena or CompletionArena later. Misc shell functions: - Trying to figure out why abuild runtime is also slow. - Look at what it takes to run Aboriginal Linux build scripts.
oilshell · Dec 20, 2017 · 7810a623ae70ef8be601b9d9a77591b089d3d57f · 7810a62
1 parent d1e6ebf
commit 7810a623ae70ef8be601b9d9a77591b089d3d57f
diff --git a/benchmarks/pytrace.sh b/benchmarks/pytrace.sh
@@ -14,12 +14,16 @@ set -o errexit
 readonly BIGGEST=benchmarks/testdata/configure-coreutils
 readonly ABUILD=benchmarks/testdata/abuild
 readonly -a RUN_ABUILD=(bin/oil.py osh $ABUILD -h)
+# Slightly faster but not significantly.
+#readonly -a RUN_ABUILD=(_bin/osh $ABUILD -h)
 readonly -a OSH_PARSE=(bin/oil.py osh --ast-format none -n)

 #
 # Use Python's cProfile, which uses _lsprof.  This is pretty fast.
 #

+time-bash-run-abuild() { time bash $ABUILD -h; }
+
 # Old: ~2.7 seconds (no tracing)
 # 2017/11/27, After ASDL optimization: 0.72 seconds.
 time-run-abuild() { time "${RUN_ABUILD[@]}"; }
@@ -51,6 +55,15 @@ cprofile-run-abuild() {
  _cprofile _tmp/abuild-run.cprofile "${RUN_ABUILD[@]}"
 }

+# TODO: Try uftrace?  I guess you can compare wait4() call duration with bash
+# vs. osh?
+strace-run-abuild() {
+  #local filter='read,wait4' 
+  local filter='execve,wait4' 
+  time strace -ff -e "$filter" "${RUN_ABUILD[@]}"
+  #time strace -c "${RUN_ABUILD[@]}"
+}
+
 # Yeah I understand from this why Chrome Tracing / Flame Graphs are better.
 # This format doesn't respect the stack!
 # cumtime: bin/oil.py is the top, obviously
diff --git a/bin/oil.py b/bin/oil.py
@@ -263,7 +263,7 @@ def OshMain(argv, login_shell):
    rc_path = 'oilrc'
    arena.PushSource(rc_path)
    with open(rc_path) as f:
-      rc_line_reader = reader.FileLineReader(f, arena=arena)
+      rc_line_reader = reader.FileLineReader(f, arena)
      _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
      try:
        rc_node = rc_c_parser.ParseWholeFile()
@@ -285,23 +285,23 @@ def OshMain(argv, login_shell):

  if opts.c is not None:
    arena.PushSource('<command string>')
-    line_reader = reader.StringLineReader(opts.c, arena=arena)
+    line_reader = reader.StringLineReader(opts.c, arena)
    interactive = False
  elif opts.i:  # force interactive
    arena.PushSource('<stdin -i>')
-    line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
+    line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
    interactive = True
  else:
    try:
      script_name = argv[opt_index]
    except IndexError:
      if sys.stdin.isatty():
        arena.PushSource('<interactive>')
-        line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
+        line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
        interactive = True
      else:
        arena.PushSource('<stdin>')
-        line_reader = reader.FileLineReader(sys.stdin, arena=arena)
+        line_reader = reader.FileLineReader(sys.stdin, arena)
        interactive = False
    else:
      arena.PushSource(script_name)
@@ -313,7 +313,7 @@ def OshMain(argv, login_shell):
      except IOError as e:
        util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno))
        return 1
-      line_reader = reader.FileLineReader(f, arena=arena)
+      line_reader = reader.FileLineReader(f, arena)
      interactive = False

  # TODO: assert arena.NumSourcePaths() == 1
diff --git a/core/cmd_exec_test.py b/core/cmd_exec_test.py
@@ -22,6 +22,7 @@
 from core import word_eval
 from core import runtime
 from core import process
+from core import test_lib

 from osh import ast_ as ast
 from osh import parse_lib
@@ -30,9 +31,10 @@
 def InitCommandParser(code_str):
  from osh.word_parse import WordParser
  from osh.cmd_parse import CommandParser
-  line_reader, lexer = parse_lib.InitLexer(code_str)
+  arena = test_lib.MakeArena('<cmd_exec_test.py>')
+  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
  w_parser = WordParser(lexer, line_reader)
-  c_parser = CommandParser(w_parser, lexer, line_reader)
+  c_parser = CommandParser(w_parser, lexer, line_reader, arena)
  return c_parser


diff --git a/core/completion.py b/core/completion.py
@@ -595,7 +595,7 @@ def __init__(self, pool, ev, comp_lookup, var_comp):

  def Matches(self, buf, status_out):
    arena = alloc.CompletionArena(self.pool)
-    w_parser, c_parser = parse_lib.MakeParserForCompletion(buf, arena=arena)
+    w_parser, c_parser = parse_lib.MakeParserForCompletion(buf, arena)
    comp_type, prefix, comp_words = _GetCompletionType(
        w_parser, c_parser, self.ev, status_out)

diff --git a/core/completion_test.py b/core/completion_test.py
@@ -152,7 +152,7 @@ def _MakeTestEvaluator():
 def _TestGetCompletionType(buf):
  ev = _MakeTestEvaluator()
  arena = test_lib.MakeArena('<completion_test.py>')
-  w_parser, c_parser = parse_lib.MakeParserForCompletion(buf, arena=arena)
+  w_parser, c_parser = parse_lib.MakeParserForCompletion(buf, arena)
  print('---', buf)
  return completion._GetCompletionType(w_parser, c_parser, ev, STATUS)

diff --git a/core/lexer.py b/core/lexer.py
@@ -37,7 +37,7 @@ def CompileAll(pat_list):


 class LineLexer(object):
-  def __init__(self, match_func, line, arena=None):
+  def __init__(self, match_func, line, arena):
    # Compile all regexes
    self.match_func = match_func
    self.arena = arena
diff --git a/core/reader.py b/core/reader.py
@@ -40,7 +40,7 @@ def Reset(self):


 class InteractiveLineReader(_Reader):
-  def __init__(self, ps1, arena=None):
+  def __init__(self, ps1, arena):
    _Reader.__init__(self, arena)
    self.ps1 = ps1
    self.prompt_str = ps1
@@ -64,7 +64,7 @@ def Reset(self):
 class FileLineReader(_Reader):
  """For -c and stdin?"""

-  def __init__(self, f, arena=None):
+  def __init__(self, f, arena):
    """
    Args:
      lines: List of (line_id, line) pairs
@@ -85,8 +85,8 @@ def _GetLine(self):
    return line


-def StringLineReader(s, arena=None):
-  return FileLineReader(cStringIO.StringIO(s), arena=arena)
+def StringLineReader(s, arena):
+  return FileLineReader(cStringIO.StringIO(s), arena)


 # C++ ownership notes:
diff --git a/core/reader_test.py b/core/reader_test.py
@@ -7,6 +7,7 @@
 import unittest

 from core import alloc
+from core import test_lib
 from core import reader  # module under test


@@ -17,19 +18,20 @@ def setUp(self):
    #self.arena = pool.NewArena()

  def testStringLineReader(self):
-    # No Arena, gives -1
-    r = reader.StringLineReader('one\ntwo')
-    self.assertEqual((-1, 'one\n'), r.GetLine())
-    self.assertEqual((-1, 'two\n'), r.GetLine())
+    arena = test_lib.MakeArena('<reader_test.py>')
+
+    r = reader.StringLineReader('one\ntwo', arena)
+    self.assertEqual((0, 'one\n'), r.GetLine())
+    self.assertEqual((1, 'two\n'), r.GetLine())
    self.assertEqual((-1, None), r.GetLine())

  def testLineReadersAreEquivalent(self):
    a1 = self.pool.NewArena()
-    r1 = reader.StringLineReader('one\ntwo', arena=a1)
+    r1 = reader.StringLineReader('one\ntwo', a1)

    a2 = self.pool.NewArena()
    f = cStringIO.StringIO('one\ntwo')
-    r2 = reader.FileLineReader(f, arena=a2)
+    r2 = reader.FileLineReader(f, a2)

    a3 = self.pool.NewArena()
    lines = [(0, 'one\n'), (1, 'two\n')]
diff --git a/core/shell_test.py b/core/shell_test.py
@@ -36,9 +36,10 @@
 class LineReaderTest(unittest.TestCase):

  def testGetLine(self):
-    r = reader.StringLineReader('foo\nbar')  # no trailing newline
-    self.assertEqual((-1, 'foo\n'), r.GetLine())
-    self.assertEqual((-1, 'bar\n'), r.GetLine())
+    arena = test_lib.MakeArena('<shell_test.py>')
+    r = reader.StringLineReader('foo\nbar', arena)  # no trailing newline
+    self.assertEqual((0, 'foo\n'), r.GetLine())
+    self.assertEqual((1, 'bar\n'), r.GetLine())

    # Keep returning EOF after exhausted
    self.assertEqual((-1, None), r.GetLine())
@@ -48,9 +49,9 @@ def testGetLine(self):
 def ParseAndExecute(code_str):
  arena = test_lib.MakeArena('<shell_test.py>')

-  line_reader, lexer = parse_lib.InitLexer(code_str, arena=arena)
+  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
  w_parser = WordParser(lexer, line_reader)
-  c_parser = CommandParser(w_parser, lexer, line_reader, arena=arena)
+  c_parser = CommandParser(w_parser, lexer, line_reader, arena)

  node = c_parser.ParseWholeFile()
  if not node:
diff --git a/osh/bool_parse_test.py b/osh/bool_parse_test.py
@@ -12,6 +12,7 @@
 import unittest

 from core.id_kind import Id
+from core import test_lib

 from osh import ast_ as ast
 from osh import parse_lib
@@ -36,7 +37,8 @@ def _ReadWords(w_parser):

 def _MakeParser(code_str):
  # NOTE: We need the extra ]] token
-  w_parser, _ = parse_lib.MakeParserForCompletion(code_str + ' ]]')
+  arena = test_lib.MakeArena('<bool_parse_test.py>')
+  w_parser, _ = parse_lib.MakeParserForCompletion(code_str + ' ]]', arena)
  w_parser._Next(lex_mode_e.DBRACKET)  # for tests only
  p = bool_parse.BoolParser(w_parser)
  if not p._Next():
diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py
@@ -37,7 +37,7 @@ class CommandParser(object):
    lexer: for lookahead in function def, PushHint of ()
    line_reader: for here doc
  """
-  def __init__(self, w_parser, lexer, line_reader, arena=None):
+  def __init__(self, w_parser, lexer, line_reader, arena):
    self.w_parser = w_parser  # for normal parsing
    self.lexer = lexer  # for fast lookahead to (, for function defs
    self.line_reader = line_reader  # for here docs
diff --git a/osh/cmd_parse_test.py b/osh/cmd_parse_test.py
@@ -25,9 +25,9 @@
 # TODO: Use parse_lib instead
 def InitCommandParser(code_str):
  arena = test_lib.MakeArena('<cmd_parse_test.py>')
-  line_reader, lexer = parse_lib.InitLexer(code_str, arena=arena)
+  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
  w_parser = WordParser(lexer, line_reader)
-  c_parser = CommandParser(w_parser, lexer, line_reader, arena=arena)
+  c_parser = CommandParser(w_parser, lexer, line_reader, arena)
  return arena, c_parser  # arena is returned for printing errors


diff --git a/osh/lex_test.py b/osh/lex_test.py
@@ -9,7 +9,6 @@
 from core.id_kind import Id, Kind, LookupKind
 from core.lexer import CompileAll, Lexer, LineLexer
 from core import test_lib
-from core.test_lib import TokensEqual

 from osh import parse_lib
 from osh import ast_ as ast
@@ -20,7 +19,7 @@

 def _InitLexer(s):
  arena = test_lib.MakeArena('<lex_test.py>')
-  _, lexer = parse_lib.InitLexer(s, arena=arena)
+  _, lexer = parse_lib.InitLexer(s, arena)
  return lexer


@@ -40,7 +39,7 @@ class LexerTest(unittest.TestCase):

  def assertTokensEqual(self, left, right):
    self.assertTrue(
-        TokensEqual(left, right), 'Expected %r, got %r' % (left, right))
+        test_lib.TokensEqual(left, right), 'Expected %r, got %r' % (left, right))

  def testRead(self):
    lexer = _InitLexer(CMD)
@@ -169,45 +168,48 @@ def testLookAhead(self):

 class LineLexerTest(unittest.TestCase):

+  def setUp(self):
+    self.arena = test_lib.MakeArena('<lex_test.py>')
+
  def assertTokensEqual(self, left, right):
-    self.assertTrue(TokensEqual(left, right))
+    self.assertTrue(test_lib.TokensEqual(left, right))

  def testReadOuter(self):
-    l = LineLexer(parse_lib._MakeMatcher(), '\n')
+    l = LineLexer(parse_lib._MakeMatcher(), '\n', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Op_Newline, '\n'), l.Read(lex_mode_e.OUTER))

  def testRead_VS_ARG_UNQ(self):
-    l = LineLexer(parse_lib._MakeMatcher(), "'hi'")
+    l = LineLexer(parse_lib._MakeMatcher(), "'hi'", self.arena)
    t = l.Read(lex_mode_e.VS_ARG_UNQ)
    self.assertEqual(Id.Left_SingleQuote, t.id)

  def testLookAhead(self):
    # Lines always end with '\n'
-    l = LineLexer(parse_lib._MakeMatcher(), '')
+    l = LineLexer(parse_lib._MakeMatcher(), '', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Unknown_Tok, ''), l.LookAhead(lex_mode_e.OUTER))

-    l = LineLexer(parse_lib._MakeMatcher(), 'foo')
+    l = LineLexer(parse_lib._MakeMatcher(), 'foo', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
        ast.token(Id.Unknown_Tok, ''), l.LookAhead(lex_mode_e.OUTER))

-    l = LineLexer(parse_lib._MakeMatcher(), 'foo  bar')
+    l = LineLexer(parse_lib._MakeMatcher(), 'foo  bar', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'bar'), l.LookAhead(lex_mode_e.OUTER))

    # No lookahead; using the cursor!
-    l = LineLexer(parse_lib._MakeMatcher(), 'func(')
+    l = LineLexer(parse_lib._MakeMatcher(), 'func(', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'func'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
        ast.token(Id.Op_LParen, '('), l.LookAhead(lex_mode_e.OUTER))

-    l = LineLexer(parse_lib._MakeMatcher(), 'func  (')
+    l = LineLexer(parse_lib._MakeMatcher(), 'func  (', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'func'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
diff --git a/osh/parse_lib.py b/osh/parse_lib.py
@@ -59,11 +59,11 @@ def _MakeMatcher():
    return MatchToken_Slow(lex.LEXER_DEF)


-def InitLexer(s, arena=None):
+def InitLexer(s, arena):
  """For tests only."""
  match_func = _MakeMatcher()
-  line_lexer = lexer.LineLexer(match_func, '', arena=arena)
-  line_reader = reader.StringLineReader(s, arena=arena)
+  line_lexer = lexer.LineLexer(match_func, '', arena)
+  line_reader = reader.StringLineReader(s, arena)
  lx = lexer.Lexer(line_lexer, line_reader)
  return line_reader, lx

@@ -88,10 +88,10 @@ def InitLexer(s, arena=None):
 def MakeParser(line_reader, arena):
  """Top level parser."""
  # AtEnd() is true
-  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena=arena)
+  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena)
  lx = lexer.Lexer(line_lexer, line_reader)
  w_parser = word_parse.WordParser(lx, line_reader)
-  c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena=arena)
+  c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena)
  return w_parser, c_parser


@@ -102,21 +102,21 @@ def MakeParser(line_reader, arena):
 #
 # NOTE: It probably needs to take a VirtualLineReader for $PS1, $PS2, ...
 # values.
-def MakeParserForCompletion(code_str, arena=None):
+def MakeParserForCompletion(code_str, arena):
  """Parser for partial lines."""
  # NOTE: We don't need to use a arena here?  Or we need a "scratch arena" that
  # doesn't interfere with the rest of the program.
-  line_reader = reader.StringLineReader(code_str)
-  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena=arena)  # AtEnd() is true
+  line_reader = reader.StringLineReader(code_str, arena)
+  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena)  # AtEnd() is true
  lx = lexer.Lexer(line_lexer, line_reader)
  w_parser = word_parse.WordParser(lx, line_reader)
-  c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena=arena)
+  c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena)
  return w_parser, c_parser


 def MakeWordParserForHereDoc(lines, arena):
  line_reader = reader.VirtualLineReader(lines, arena)
-  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena=arena)
+  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena)
  lx = lexer.Lexer(line_lexer, line_reader)
  return word_parse.WordParser(lx, line_reader)