Got a basic set -x tracer working, with tests. I folded spec/xtrace.sh

into spec/xtrace.test.sh. It parses and evaluates $PS4 at runtime. Some spec tests are still failing. Also: - Fix the reader not to add a \n to the last line. That was an anachronism related to ancient C++ input line handling. - Minor fix to the quick ref generator. - Rename executor member ev -> word_ev
oilshell · Dec 28, 2017 · 9887c70417c87815cd90f6fbef9cbe920ba88eae · 9887c70
1 parent 34508c8
commit 9887c70417c87815cd90f6fbef9cbe920ba88eae
diff --git a/build/quick_ref.py b/build/quick_ref.py
@@ -17,7 +17,7 @@
 # 2. lower-case or upper-case topic
 # 3. Optional: A SINGLE space, then punctuation

-TOPIC_RE = re.compile(r'\b(X[ ])?\@?([a-z\-]+|[A-Z0-9_]+)([ ]\S+)?', re.VERBOSE)
+TOPIC_RE = re.compile(r'\b(X[ ])?\@?([a-z_\-]+|[A-Z0-9_]+)([ ]\S+)?', re.VERBOSE)

 # Sections have alphabetical characters, spaces, and '/' for I/O.  They are
 # turned into anchors.
diff --git a/core/alloc.py b/core/alloc.py
@@ -120,6 +120,15 @@ def CompletionArena(pool):
  return arena


+def PluginArena():
+  """For PS4, etc."""
+  # TODO: Should there only be one pool?  This isn't worked out yet.
+  pool = Pool()
+  arena = pool.NewArena()
+  arena.PushSource('<plugin>')
+  return arena
+
+
 # In C++, InteractiveLineReader and StringLineReader should use the same
 # representation: std::string with internal NULs to terminate lines, and then
 # std::vector<char*> that points into to it.
diff --git a/core/cmd_exec.py b/core/cmd_exec.py
@@ -23,6 +23,7 @@

 from asdl import const

+from core import alloc
 from core import args
 from core import braces
 from core import expr_eval
@@ -52,6 +53,7 @@
 redir_e = ast.redir_e
 lhs_expr_e = ast.lhs_expr_e
 assign_op_e = ast.assign_op_e
+lex_mode_e = ast.lex_mode_e

 value_e = runtime.value_e
 scope_e = runtime.scope_e
@@ -121,9 +123,9 @@ def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
    self.exec_opts = exec_opts
    self.arena = arena

-    self.ev = word_eval.NormalWordEvaluator(mem, exec_opts, self)
-    self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.ev)
-    self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.ev)
+    self.word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, self)
+    self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.word_ev)
+    self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.word_ev)

    self.traps = {}  # signal/hook -> LST node
    self.fd_state = process.FdState()
@@ -141,6 +143,8 @@ def __init__(self, mem, status_lines, funcs, completion, comp_lookup,

    self.loop_level = 0  # for detecting bad top-level break/continue

+    self.tracer = Tracer(exec_opts, mem, self.word_ev)
+
  def _Complete(self, argv):
    """complete builtin - register a completion function.

@@ -379,8 +383,8 @@ def _EvalRedirect(self, n):

      if redir_type == RedirType.Path:
        # NOTE: no globbing.  You can write to a file called '*.py'.
-        val = self.ev.EvalWordToString(n.arg_word)
-        if val.tag != value_e.Str:
+        val = self.word_ev.EvalWordToString(n.arg_word)
+        if val.tag != value_e.Str:  # TODO: This error never fires
          util.warn("Redirect filename must be a string, got %s", val)
          return None
        filename = val.s
@@ -392,8 +396,8 @@ def _EvalRedirect(self, n):
        return runtime.PathRedirect(n.op_id, fd, filename)

      elif redir_type == RedirType.Desc:  # e.g. 1>&2
-        val = self.ev.EvalWordToString(n.arg_word)
-        if val.tag != value_e.Str:
+        val = self.word_ev.EvalWordToString(n.arg_word)
+        if val.tag != value_e.Str:  # TODO: This error never fires
          util.warn("Redirect descriptor should be a string, got %s", val)
          return None
        t = val.s
@@ -411,8 +415,8 @@ def _EvalRedirect(self, n):

      elif redir_type == RedirType.Here:  # here word
        # TODO: decay should be controlled by an option
-        val = self.ev.EvalWordToString(n.arg_word, decay=True)
-        if val.tag != value_e.Str:
+        val = self.word_ev.EvalWordToString(n.arg_word, decay=True)
+        if val.tag != value_e.Str:   # TODO: This error never fires
          util.warn("Here word body should be a string, got %s", val)
          return None
        # NOTE: bash and mksh both add \n
@@ -422,8 +426,8 @@ def _EvalRedirect(self, n):

    elif n.tag == redir_e.HereDoc:
      # TODO: decay shoudl be controlled by an option
-      val = self.ev.EvalWordToString(n.body, decay=True)
-      if val.tag != value_e.Str:
+      val = self.word_ev.EvalWordToString(n.body, decay=True)
+      if val.tag != value_e.Str:   # TODO: This error never fires
        util.warn("Here doc body should be a string, got %s", val)
        return None
      return runtime.HereRedirect(fd, val.s)
@@ -468,7 +472,7 @@ def _EvalEnv(self, node_env, out_env):
      rhs = env_pair.val

      # Could pass extra bindings like out_env here?  But PushTemp should work?
-      val = self.ev.EvalWordToString(rhs)
+      val = self.word_ev.EvalWordToString(rhs)

      # Set each var so the next one can reference it.  Example:
      # FOO=1 BAR=$FOO ls /
@@ -606,11 +610,6 @@ def _RunJobInBackground(self, node):
      log('Started background job with pid %d', pid)
    return 0

-  # TODO: This causes "bad descriptor errors"
-  #XFILE = open('/tmp/xtrace.log', 'w')
-  # typically fd 4, not sure why it interferes?
-  #log('*** XFILE %d', XFILE.fileno())
-
  def _Dispatch(self, node, fork_external):
    argv0 = None  # for error message
    check_errexit = False  # for errexit
@@ -630,22 +629,14 @@ def _Dispatch(self, node, fork_external):
      # to print the filename too.

      words = braces.BraceExpandWords(node.words)
-      argv = self.ev.EvalWordSequence(words)
+      argv = self.word_ev.EvalWordSequence(words)
      if argv:
        argv0 = argv[0]

      environ = self.mem.GetExported()
      self._EvalEnv(node.more_env, environ)

-      if self.exec_opts.xtrace:
-        # TODO: Eval PS4.  Using what evaluator?  I guess the same state.
-        # self.ev.
-        log('+ %s', argv)
-        # TODO:
-        #
-        # This is good enough for xtrace.
-        # But the tracer is more general than that.
-        # self.tracer.BeginSimpleCommand()
+      self.tracer.OnSimpleCommand(argv)

      status = self._RunSimpleCommand(argv, environ, fork_external)

@@ -720,7 +711,7 @@ def _Dispatch(self, node, fork_external):
      for pair in node.pairs:
        if pair.op == assign_op_e.PlusEqual:
          assert pair.rhs, pair.rhs  # I don't think a+= is valid?
-          val = self.ev.EvalWordToAny(pair.rhs)
+          val = self.word_ev.EvalWordToAny(pair.rhs)
          old_val, lval = expr_eval.EvalLhs(pair.lhs, self.arith_ev, self.mem,
                                            self.exec_opts)
          sig = (old_val.tag, val.tag)
@@ -743,7 +734,7 @@ def _Dispatch(self, node, fork_external):

          # RHS can be a string or array.
          if pair.rhs:
-            val = self.ev.EvalWordToAny(pair.rhs)
+            val = self.word_ev.EvalWordToAny(pair.rhs)
            assert isinstance(val, runtime.value), val
          else:
            # e.g. 'readonly x' or 'local x'
@@ -756,7 +747,7 @@ def _Dispatch(self, node, fork_external):

    elif node.tag == command_e.ControlFlow:
      if node.arg_word:  # Evaluate the argument
-        val = self.ev.EvalWordToString(node.arg_word)
+        val = self.word_ev.EvalWordToString(node.arg_word)
        assert val.tag == value_e.Str
        arg = int(val.s)  # They all take integers
      else:
@@ -856,7 +847,7 @@ def _Dispatch(self, node, fork_external):
        iter_list = self.mem.GetArgv()
      else:
        words = braces.BraceExpandWords(node.iter_words)
-        iter_list = self.ev.EvalWordSequence(words)
+        iter_list = self.word_ev.EvalWordSequence(words)
        # We need word splitting and so forth
        # NOTE: This expands globs too.  TODO: We should pass in a Globber()
        # object.
@@ -917,7 +908,7 @@ def _Dispatch(self, node, fork_external):
      status = 0  # make it true

    elif node.tag == command_e.Case:
-      val = self.ev.EvalWordToString(node.to_match)
+      val = self.word_ev.EvalWordToString(node.to_match)
      to_match = val.s

      status = 0  # If there are no arms, it should be zero?
@@ -926,7 +917,7 @@ def _Dispatch(self, node, fork_external):
      for arm in node.arms:
        for pat_word in arm.pat_list:
          # NOTE: Is it OK that we're evaluating these as we go?
-          pat_val = self.ev.EvalWordToString(pat_word, do_fnmatch=True)
+          pat_val = self.word_ev.EvalWordToString(pat_word, do_fnmatch=True)
          #log('Matching word %r against pattern %r', to_match, pat_val.s)
          if libc.fnmatch(pat_val.s, to_match):
            status = self._ExecuteList(arm.action)
@@ -1097,3 +1088,115 @@ def RunFunc(self, func_node, argv):
      self.fd_state.Pop()

    return status
+
+
+class Tracer(object):
+  """A tracer for this process.
+  
+  TODO: Connect it somehow to tracers for other processes.  So you can make an
+  HTML report offline.
+
+  https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html#Bash-Variables
+
+  Bare minimum to debug problems:
+    - argv and span ID of the SimpleCommand that corresponds to that
+    - then print line number using arena
+    - set -x doesn't print line numbers!  OH but you can do that with
+      PS4=$LINENO 
+  """
+  def __init__(self, exec_opts, mem, word_ev):
+    """
+    Args:
+      exec_opts: For xtrace setting
+      mem: for retrieving PS4
+      word_ev: for evaluating PS4
+    """
+    self.exec_opts = exec_opts
+    self.mem = mem
+    self.word_ev = word_ev
+
+    self.arena = alloc.PluginArena()
+    self.parse_cache = {}  # PS4 value -> CompoundWord.  PS4 is scoped.
+
+  def OnSimpleCommand(self, argv):
+    """For set -x."""
+
+    # NOTE: I think tracing should be on by default?  For post-mortem viewing.
+    if not self.exec_opts.xtrace:
+      return
+
+    val = self.mem.GetVar('PS4')
+    assert val.tag == value_e.Str
+
+    s = val.s
+    if s:
+      first_char, ps4 = s[0], s[1:]
+    else:
+      first_char, ps4 = '+', ' '  # default
+
+    try:
+      ps4_word = self.parse_cache[ps4]
+    except KeyError:
+      # We have to parse this at runtime.  PS4 should usually remain constant.
+      w_parser = parse_lib.MakeWordParserForPlugin(ps4, self.arena)
+
+      # NOTE: Reading PS4 is just like reading a here doc line.  "\n" is
+      # allowed too.  The OUTER mode would stop at spaces, and ReadWord
+      # doesn't allow lex_mode_e.DQ.
+      ps4_word = w_parser.ReadHereDocBody()
+
+      if not ps4_word:
+        error_str = '<ERROR: cannot parse PS4>'
+        t = ast.token(Id.Lit_Chars, error_str, const.NO_INTEGER)
+        ps4_word = ast.CompoundWord([ast.LiteralPart(t)])
+      self.parse_cache[ps4] = ps4_word
+
+    #print(ps4_word)
+
+    # TODO: Repeat first character according process stack depth.  Where is
+    # that stored?  In the executor itself?  It should be stored along with
+    # the PID.  Need some kind of ShellProcessState or something.
+    #
+    # We should come up with a better mechanism.  Something like $PROC_INDENT
+    # and $OIL_XTRACE_PREFIX.
+
+    # TODO: Handle runtime errors!  For example, you could PS4='$(( 1 / 0 ))'
+    # <ERROR: cannot evaluate PS4>
+    prefix = self.word_ev.EvalWordToString(ps4_word)
+
+    print('%s%s%s' % (
+        first_char, prefix.s, ' '.join(_PrettyString(a) for a in argv)),
+        file=sys.stderr)
+
+  def Event(self):
+    """
+    Other events:
+
+    - Function call events.  As opposed to external commands.
+    - Process Forks.  Subshell, command sub, pipeline,
+    - Command Completion -- you get the status code.
+    - Assignments
+      - We should desugar to SetVar like mksh
+    """
+    pass
+
+
+# Copied from asdl/format.py.  We're not using it directly because that is
+# debug output, and this is real input.
+# TODO: Is this slow?
+
+# NOTE: bash prints \' for single quote, repr() prints "'".  Gah.  This is also
+# used for printf %q and ${var@q} (bash 4.4).
+
+import re
+_PLAIN_RE = re.compile(r'^[a-zA-Z0-9\-_./]+$')
+
+def _PrettyString(s):
+  if '\n' in s:
+    #return json.dumps(s)  # account for the fact that $ matches the newline
+    return repr(s)
+  if _PLAIN_RE.match(s):
+    return s
+  else:
+    #return json.dumps(s)
+    return repr(s)
diff --git a/core/reader.py b/core/reader.py
@@ -77,12 +77,6 @@ def _GetLine(self):
    if not line:
      return None

-    # TODO: Remove this anachonism.  We no longer need every line to end with a
-    # newline.  See input handling comment at the top of osh/lex.py.  (I tried
-    # and it made a bunch of tests fail.)
-    if not line.endswith('\n'):
-      line += '\n'
-
    return line


diff --git a/core/reader_test.py b/core/reader_test.py
@@ -22,7 +22,7 @@ def testStringLineReader(self):

    r = reader.StringLineReader('one\ntwo', arena)
    self.assertEqual((0, 'one\n'), r.GetLine())
-    self.assertEqual((1, 'two\n'), r.GetLine())
+    self.assertEqual((1, 'two'), r.GetLine())
    self.assertEqual((-1, None), r.GetLine())

  def testLineReadersAreEquivalent(self):
@@ -34,7 +34,7 @@ def testLineReadersAreEquivalent(self):
    r2 = reader.FileLineReader(f, a2)

    a3 = self.pool.NewArena()
-    lines = [(0, 'one\n'), (1, 'two\n')]
+    lines = [(0, 'one\n'), (1, 'two')]
    r3 = reader.VirtualLineReader(lines, a3)

    for a in [a1, a2, a3]:
@@ -44,7 +44,7 @@ def testLineReadersAreEquivalent(self):
      print(r)
      # Lines are added to the arena with a line_id.
      self.assertEqual((0, 'one\n'), r.GetLine())
-      self.assertEqual((1, 'two\n'), r.GetLine())
+      self.assertEqual((1, 'two'), r.GetLine())
      self.assertEqual((-1, None), r.GetLine())


diff --git a/core/shell_test.py b/core/shell_test.py
@@ -33,19 +33,6 @@
  #util.WrapMethods(Lexer, state)


-class LineReaderTest(unittest.TestCase):
-
-  def testGetLine(self):
-    arena = test_lib.MakeArena('<shell_test.py>')
-    r = reader.StringLineReader('foo\nbar', arena)  # no trailing newline
-    self.assertEqual((0, 'foo\n'), r.GetLine())
-    self.assertEqual((1, 'bar\n'), r.GetLine())
-
-    # Keep returning EOF after exhausted
-    self.assertEqual((-1, None), r.GetLine())
-    self.assertEqual((-1, None), r.GetLine())
-
-
 def ParseAndExecute(code_str):
  arena = test_lib.MakeArena('<shell_test.py>')

diff --git a/osh/parse_lib.py b/osh/parse_lib.py
@@ -126,6 +126,13 @@ def MakeWordParserForHereDoc(lines, arena):
  return word_parse.WordParser(lx, line_reader)


+def MakeWordParserForPlugin(code_str, arena):
+  line_reader = reader.StringLineReader(code_str, arena)
+  line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena)
+  lx = lexer.Lexer(line_lexer, line_reader)
+  return word_parse.WordParser(lx, line_reader)
+
+
 def MakeParserForCommandSub(line_reader, lexer):
  """To parse command sub, we want a fresh word parser state.