Implement $SOURCE_NAME and $LINENO to allow tracing with xtrace/PS4.

Fix a bug where physical line numbers are off by one. Also: - Fold core/shell_test.py into core/cmd_exec_test.py and remove it. - Spec test for set -o verbose
oilshell · Dec 29, 2017 · 3d6099aea539a9776d61641722e3e5621a581a84 · 3d6099a
1 parent cf95049
commit 3d6099aea539a9776d61641722e3e5621a581a84
diff --git a/core/alloc.py b/core/alloc.py
@@ -13,6 +13,8 @@

 from asdl import const

+from core import util
+

 class Arena(object):
  """A collection of lines and line spans.
@@ -99,7 +101,11 @@ def AddLineSpan(self, line_span):

  def GetLineSpan(self, span_id):
    assert span_id != const.NO_INTEGER, span_id
-    return self.spans[span_id]  # span IDs start from 1
+    try:
+      return self.spans[span_id]
+    except IndexError:
+      util.log('Span ID out of range: %d', span_id)
+      raise

  def GetDebugInfo(self, line_id):
    """Get the path and physical line number, for parse errors."""
diff --git a/core/cmd_exec.py b/core/cmd_exec.py
@@ -29,6 +29,7 @@
 from core import expr_eval
 from core import reader
 from core import test_builtin
+from core import word
 from core import word_eval
 from core import ui
 from core import util
@@ -177,7 +178,8 @@ def _EvalHelper(self, c_parser, source_name):
    try:
      node = c_parser.ParseWholeFile()
      # NOTE: We could model a parse error as an exception, like Python, so we
-      # get a traceback.  (This won't be applicable for a static module system.)
+      # get a traceback.  (This won't be applicable for a static module
+      # system.)
      if not node:
        util.error('Parse error in %r:', source_name)
        err = c_parser.Error()
@@ -636,6 +638,37 @@ def _Dispatch(self, node, fork_external):
      environ = self.mem.GetExported()
      self._EvalEnv(node.more_env, environ)

+      # This is a very basic implementation for PS4='+$SOURCE_NAME:$LINENO:'
+
+      # TODO:
+      # - It should be a stack eventually.  So if there is an exception we can
+      # print the full stack trace.  Python has a list of frame objects, and
+      # each one has a location?
+      # - The API to get DebugInfo is overly long.
+      # - Maybe just do a simple thing like osh-o line-trace without any PS4?
+
+      # NOTE: osh2oil uses node.more_env, but we don't need that.
+      found = False
+      if node.words:
+        first_word = node.words[0]
+        span_id = word.LeftMostSpanForWord(first_word)
+        if span_id == const.NO_INTEGER:
+          log('Warning: word has no location information: %s', first_word)
+        else:
+          found = True
+
+      if found:
+        # NOTE: This is what we want to expose as variables for PS4.
+        #ui.PrintFilenameAndLine(span_id, self.arena)
+
+        line_span = self.arena.GetLineSpan(span_id)
+        line_id = line_span.line_id
+        line = self.arena.GetLine(line_id)
+        source_name, line_num = self.arena.GetDebugInfo(line_id)
+        self.mem.SetSourceLocation(source_name, line_num)
+      else:
+        self.mem.SetSourceLocation('<unknown>', -1)
+
      self.tracer.OnSimpleCommand(argv)

      status = self._RunSimpleCommand(argv, environ, fork_external)
diff --git a/core/cmd_exec_test.py b/core/cmd_exec_test.py
@@ -38,15 +38,16 @@ def InitCommandParser(code_str):
  return c_parser


-def InitExecutor():
+def InitExecutor(arena=None):
+  if not arena:
+    arena = test_lib.MakeArena('<InitExecutor>')
+
  mem = state.Mem('', [], {}, None)
  status_lines = None  # not needed for what we're testing
  builtins = builtin.BUILTIN_DEF
  funcs = {}
  comp_funcs = {}
  exec_opts = state.ExecOpts(mem)
-  pool = alloc.Pool()
-  arena = pool.NewArena()
  return cmd_exec.Executor(mem, status_lines, funcs, completion, comp_funcs,
                           exec_opts, arena)

@@ -69,7 +70,8 @@ def testBraceExpand(self):
    node = c_parser.ParseCommandLine()
    print(node)

-    ex = InitExecutor()
+    arena = test_lib.MakeArena('<cmd_exec_test.py>')
+    ex = InitExecutor(arena)
    #print(ex.Execute(node))

    #print(ex._ExpandWords(node.words))
@@ -96,5 +98,34 @@ def testVarOps(self):
    print(ev.part_ev._EvalWordPart(set_sub))


+def ParseAndExecute(code_str):
+  arena = test_lib.MakeArena('<shell_test.py>')
+
+  # TODO: Unify with InitCommandParser above.
+  from osh.word_parse import WordParser
+  from osh.cmd_parse import CommandParser
+
+  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
+  w_parser = WordParser(lexer, line_reader)
+  c_parser = CommandParser(w_parser, lexer, line_reader, arena)
+
+  node = c_parser.ParseWholeFile()
+  if not node:
+    raise AssertionError()
+
+  print(node)
+  ex = InitExecutor(arena)
+  status = ex.Execute(node)
+
+  # TODO: Can we capture output here?
+  return status
+
+
+class ExecutorTest(unittest.TestCase):
+
+  def testBuiltin(self):
+    print(ParseAndExecute('echo hi'))
+
+
 if __name__ == '__main__':
  unittest.main()
diff --git a/core/reader.py b/core/reader.py
@@ -17,7 +17,7 @@
 class _Reader(object):
  def __init__(self, arena):
    self.arena = arena
-    self.line_num = 0  # physical line number
+    self.line_num = 1  # physical line numbers start from 1

  def GetLine(self):
    line = self._GetLine()
diff --git a/core/shell_test.py b/core/shell_test.py
diff --git a/core/state.py b/core/state.py
@@ -283,11 +283,16 @@ def __init__(self, argv0, argv, environ, arena):
    self.var_stack = [top]
    self.argv0 = argv0
    self.argv_stack = [_ArgFrame(argv)]
-    # NOTE: could use deque and appendleft/popleft, but
+    # NOTE: could use deque and appendleft/popleft, but:
    # 1. ASDL type checking of StrArray doesn't allow it (could be fixed)
    # 2. We don't otherwise depend on the collections module
    self.func_name_stack = []

+    # Note: we're reusing these objects because they change on every single
+    # line!  Don't want to allocate more than necsesary.
+    self.source_name = runtime.Str('')
+    self.line_num = runtime.Str('')
+
    self.last_status = 0  # Mutable public variable
    self.last_job_id = -1  # Uninitialized value mutable public variable

@@ -298,6 +303,7 @@ def __init__(self, argv0, argv, environ, arena):
    self._InitVarsFromEnv(environ)
    self.arena = arena

+
  def __repr__(self):
    parts = []
    parts.append('<Mem')
@@ -334,6 +340,10 @@ def _InitVarsFromEnv(self, environ):

    # If it's not in the environment, initialize it.  This makes it easier to
    # update later in ExecOpts.
+
+    # TODO: IFS, PWD, etc. should follow this pattern.  Maybe need a SysCall
+    # interface?  self.syscall.getcwd() etc.
+
    v = self.GetVar('SHELLOPTS')
    if v.tag == value_e.Undef:
      SetGlobalString(self, 'SHELLOPTS', '')
@@ -342,6 +352,11 @@ def _InitVarsFromEnv(self, environ):
        ast.LhsName('SHELLOPTS'), None, (var_flags_e.ReadOnly,),
        scope_e.GlobalOnly)

+  def SetSourceLocation(self, source_name, line_num):
+    # Mutate Str() objects.
+    self.source_name.s = source_name
+    self.line_num.s = str(line_num)
+
  #
  # Stack
  #
@@ -611,8 +626,16 @@ def GetVar(self, name, lookup_mode=scope_e.Dynamic):
      # bash wants it in reverse order.  This is a little inefficient but we're
      # not depending on deque().
      strs = list(reversed(self.func_name_stack))
+      # TODO: Reuse this object too?
      return runtime.StrArray(strs)

+    if name == 'LINENO':
+      return self.line_num
+
+    # Instead of BASH_SOURCE.  Using Oil _ convnetion.
+    if name == 'SOURCE_NAME':
+      return self.source_name
+
    cell, _ = self._FindCellAndNamespace(name, lookup_mode)

    if cell:
diff --git a/core/word.py b/core/word.py
@@ -155,6 +155,9 @@ def LeftMostSpanForPart(part):
    return part.spids[0]
    #return part.op.span_id  # e.g. @( is the left-most token

+  elif part.tag == word_part_e.BracedAltPart:
+    return const.NO_INTEGER
+
  else:
    raise AssertionError(part.__class__.__name__)

@@ -229,16 +232,27 @@ def LeftMostSpanForWord(w):
  if w.tag == word_e.CompoundWord:
    if len(w.parts) == 0:
      return const.NO_INTEGER
-    elif len(w.parts) == 1:
-      return LeftMostSpanForPart(w.parts[0])
    else:
      begin = w.parts[0]
      # TODO: We need to combine LineSpan()?  If they're both on the same line,
      # return them both.  If they're not, then just use "begin"?
      return LeftMostSpanForPart(begin)

-  # It's a TokenWord?
-  return w.token.span_id
+  elif w.tag == word_e.TokenWord:
+    return w.token.span_id
+
+  elif w.tag == word_e.BracedWordTree:
+    if len(w.parts) == 0:
+      return const.NO_INTEGER
+    else:
+      begin = w.parts[0]
+      # TODO: We need to combine LineSpan()?  If they're both on the same line,
+      # return them both.  If they're not, then just use "begin"?
+      return LeftMostSpanForPart(begin)
+
+  elif w.tag == word_e.StringWord:
+    # There is no place to store this now?
+    return const.NO_INTEGER


 # This is needed for DoWord I guess?  IT makes it easier to write the fixer.
diff --git a/spec/xtrace.test.sh b/spec/xtrace.test.sh
@@ -2,15 +2,26 @@
 #
 # xtrace test.  Test PS4 and line numbers, etc.

-### basic xtrace
-echo 1
-set -o xtrace
-echo 2
+### set -o verbose prints unevaluated code
+set -o verbose
+x=foo
+y=bar
+echo $x
+echo $(echo $y)
 ## STDOUT:
-1
-2
+foo
+bar
 ## STDERR:
-+ echo 2
+x=foo
+y=bar
+echo $x
+echo $(echo $y)
+## OK bash STDERR:
+x=foo
+y=bar
+echo $x
+echo $(echo $y)
+echo $y
 ## END

 ### xtrace with whitespace and quotes
@@ -150,6 +161,14 @@ x=1
 PS4='+$(x'
 set -o xtrace
 echo one
-# stdout: one
-# stderr: + echo one
+echo status=$?
+## STDOUT:
+one
+status=0
+## END
+# mksh and dash both fail.  bash prints errors to stderr.
+# OK dash stdout-json: ""
+# OK dash status: 2
+# OK mksh stdout-json: ""
+# OK mksh status: 1

diff --git a/test/spec.sh b/test/spec.sh
@@ -370,7 +370,7 @@ special-vars() {

 # dash/mksh don't implement this.
 introspect() {
-  sh-spec spec/introspect.test.sh \
+  sh-spec spec/introspect.test.sh --osh-failures-allowed 3 \
    $BASH $OSH "$@"
 }