[refactor] span_id -> Token for state.Mem current location

devtools: Update scripts to stock of big parser refactoring

core/runtime.asdl

@@ -132,7 +132,7 @@ module runtime
   # We store name_loc separately because ShFunction has a word as a name, but
   # a Proc has a Token.
   Proc = (
-    str name, loc name_loc, proc_sig sig, command body, List[value] defaults,
+    str name, Token name_tok, proc_sig sig, command body, List[value] defaults,
     bool dynamic_scope
   )
 

core/state.py

@@ -18,7 +18,7 @@
     lvalue, lvalue_e, lvalue_t,
     scope_e, scope_t, HayNode, Cell
 )
-from _devbuild.gen.syntax_asdl import loc, loc_t
+from _devbuild.gen.syntax_asdl import loc, loc_t, Token
 from _devbuild.gen.types_asdl import opt_group_i
 from asdl import runtime
 from core import error
@@ -1119,7 +1119,7 @@ class ctx_Call(object):
 
   def __init__(self, mem, mutable_opts, proc, argv):
     # type: (Mem, MutableOpts, Proc, List[str]) -> None
-    mem.PushCall(proc.name, proc.name_loc, argv)
+    mem.PushCall(proc.name, proc.name_tok, argv)
     mutable_opts.PushDynamicScope(proc.dynamic_scope)
     # It may have been disabled with ctx_ErrExit for 'if echo $(false)', but
     # 'if p' should be allowed.
@@ -1284,6 +1284,7 @@ def __init__(self, dollar0, argv, arena, debug_stack):
 
     self.pwd = None  # type: Optional[str]
 
+    self.current_tok = None  # type: Optional[Token]
     self.current_spid = runtime.NO_SPID
 
     self.last_arg = ''  # $_ is initially empty, NOT unset
@@ -1370,8 +1371,8 @@ def SetLastArgument(self, s):
     """
     self.last_arg = s
 
-  def SetCurrentSpanId(self, span_id):
-    # type: (int) -> None
+  def SetLocationToken(self, tok):
+    # type: (Token) -> None
     """Set the current source location, for BASH_SOURCE, BASH_LINENO, LINENO,
     etc.
 
@@ -1381,18 +1382,20 @@ def SetCurrentSpanId(self, span_id):
     if self.running_debug_trap:
       return
 
-    if span_id == runtime.NO_SPID:
+    if tok.span_id == runtime.NO_SPID:
       # NOTE: This happened in the osh-runtime benchmark for yash.
       log('Warning: span_id undefined in SetCurrentSpanId')
 
       #import traceback
       #traceback.print_stack()
       return
-    self.current_spid = span_id
+
+    self.current_tok = tok
+    self.current_spid = tok.span_id
 
   def CurrentLocation(self):
-    # type: () -> loc_t
-    return loc.Span(self.current_spid)
+    # type: () -> Token
+    return self.current_tok
 
   #
   # Status Variable Stack (for isolating $PS1 and $PS4)
@@ -1440,16 +1443,14 @@ def SetProcessSubStatus(self, x):
   # Call Stack
   #
 
-  def PushCall(self, func_name, def_loc, argv):
-    # type: (str, loc_t, List[str]) -> None
+  def PushCall(self, func_name, def_tok, argv):
+    # type: (str, Token, List[str]) -> None
     """For function calls."""
     self.argv_stack.append(_ArgFrame(argv))
     frame = NewDict()  # type: Dict[str, Cell]
     self.var_stack.append(frame)
 
-    def_spid = location.GetSpanId(def_loc)
-    token = self.arena.GetToken(def_spid)
-    source_str = ui.GetLineSourceString(self.arena, token.line)
+    source_str = ui.GetLineSourceString(self.arena, def_tok.line)
 
     # bash uses this order: top of stack first.
     self._PushDebugStack(source_str, func_name, None)

core/state_test.py

@@ -6,12 +6,14 @@
 import unittest
 import os.path
 
+from _devbuild.gen.id_kind_asdl import Id
 from _devbuild.gen.runtime_asdl import scope_e, lvalue, value, value_e
-from _devbuild.gen.syntax_asdl import loc
+from _devbuild.gen.syntax_asdl import loc, source, SourceLine
 from asdl import runtime
 from core import error
 from core import test_lib
 from core import state  # module under test
+from frontend import lexer
 from frontend import location
 
 
@@ -34,7 +36,11 @@ class MemTest(unittest.TestCase):
 
   def testGet(self):
     mem = _InitMem()
-    mem.PushCall('my-func', loc.Span(0), ['a', 'b'])
+
+    tok_a = lexer.DummyToken(Id.Lit_Chars, 'a')
+    tok_a.line = SourceLine(1, 'a b', source.Interactive)
+
+    mem.PushCall('my-func', tok_a, ['a', 'b'])
     print(mem.GetValue('HOME'))
     mem.PopCall()
     print(mem.GetValue('NONEXISTENT'))
@@ -51,7 +57,7 @@ def testSearchPath(self):
 
     # Set $PATH
     mem.SetValue(location.LName('PATH'), value.Str('/bin:/usr/bin'),
-               scope_e.GlobalOnly)
+                 scope_e.GlobalOnly)
 
     self.assertEqual(None, search_path.Lookup('__nonexistent__'))
     self.assertEqual('bin/osh', search_path.Lookup('bin/osh'))
@@ -97,7 +103,13 @@ def testSetVarClearFlag(self):
     mem = _InitMem()
     print(mem)
 
-    mem.PushCall('my-func', loc.Span(0), ['ONE'])
+    tok_one = lexer.DummyToken(Id.Lit_Chars, 'ONE')
+    tok_one.line = SourceLine(1, 'ONE', source.Interactive)
+
+    tok_two = lexer.DummyToken(Id.Lit_Chars, 'TWO')
+    tok_two.line = SourceLine(1, 'TWO', source.Interactive)
+
+    mem.PushCall('my-func', tok_one, ['ONE'])
     self.assertEqual(2, len(mem.var_stack))  # internal details
 
     # local x=y
@@ -106,7 +118,7 @@ def testSetVarClearFlag(self):
     self.assertEqual('y', mem.var_stack[-1]['x'].val.s)
 
     # New frame
-    mem.PushCall('my-func', loc.Span(0), ['TWO'])
+    mem.PushCall('my-func', tok_two, ['TWO'])
     self.assertEqual(3, len(mem.var_stack))  # internal details
 
     # x=y -- test out dynamic scope
@@ -285,10 +297,18 @@ def testUnset(self):
 
   def testArgv(self):
     mem = _InitMem()
-    mem.PushCall('my-func', loc.Span(0), ['a', 'b'])
+    src = source.Interactive
+
+    tok_a = lexer.DummyToken(Id.Lit_Chars, 'a')
+    tok_a.line = SourceLine(1, 'a b', src)
+
+    mem.PushCall('my-func', tok_a, ['a', 'b'])
     self.assertEqual(['a', 'b'], mem.GetArgv())
 
-    mem.PushCall('my-func', loc.Span(0), ['x', 'y'])
+    tok_x = lexer.DummyToken(Id.Lit_Chars, 'x')
+    tok_x.line = SourceLine(2, 'x y', src)
+
+    mem.PushCall('my-func', tok_x, ['x', 'y'])
     self.assertEqual(['x', 'y'], mem.GetArgv())
 
     status = mem.Shift(1)

devtools/refactor.sh

@@ -189,28 +189,37 @@ tval-eval() {
     grep -n -w tval */*_eval.py
 }
 
-# 2023-04: 518 left, many are in osh2oil
+# 2023-05: 372 left, many are in osh2oil
 spid-all() {
   show-usages _tmp/spid-all \
     egrep -n 'span_id|spid' */*.py
 }
 
-# 2023-04: 14 left
+# 2023-05: 10 left
 spid-sig() {
   show-usages _tmp/spid-sig \
     egrep -n 'def.*(span_id|spid)' */*.py
 }
 
-legacy-asdl-types() {
-  show-usages _tmp/legacy-asdl \
-    egrep -n '[a-zA-Z]__[a-zA-Z]' */*.py
+# 2023-05: 26 instances
+get-token() {
+  # Memory leak
+  show-usages _tmp/get-token \
+    egrep -n 'GetToken' */*.py
+}
+
+# 2023-05: 7 instances
+get-span-id() {
+  show-usages _tmp/get-span-id \
+    egrep -n 'GetSpanId' */*.py
+}
+
+# 2023-05: 7 instances
+get-span-id() {
+  show-usages _tmp/get-span-id \
+    egrep -n 'GetSpanId' */*.py
 }
 
-# We also want to get rid of 2 instances of 'attributes' in frontend/syntax.asdl
-#
-# - Every variant of command_t has a left token
-# - Every variant of word_part_t has a left AND a right -- so we can look up
-#   the right most span for a word
 
 asdl-create() {
   fgrep -n 'CreateNull(alloc' */*.py */*/*.py | egrep -v '_devbuild|_test.py' | tee _tmp/asdl

osh/cmd_eval.py

@@ -395,7 +395,7 @@ def _EvalRedirect(self, r):
         arg_word = cast(CompoundWord, UP_arg)
 
         # note: needed for redirect like 'echo foo > x$LINENO'
-        self.mem.SetCurrentSpanId(r.op.span_id)
+        self.mem.SetLocationToken(r.op)
 
         redir_type = consts.RedirArgType(r.op.id)  # could be static in the LST?
 
@@ -636,15 +636,11 @@ def _Dispatch(self, node, cmd_st):
 
         cmd_st.check_errexit = True
 
-        # Find span_id for a basic implementation of $LINENO, e.g.
-        # PS4='+$SOURCE_NAME:$LINENO:'
-        # Note that for '> $LINENO' the span_id is set in _EvalRedirect.
-        # TODO: Can we avoid setting this so many times?  See issue #567.
-        if len(node.words):
-          span_id = location.OfWordLeft(node.words[0])
-          # Special case for __cat < file: leave it at the redirect.
-          if span_id != runtime.NO_SPID:
-            self.mem.SetCurrentSpanId(span_id)
+        # for $LINENO, e.g.  PS4='+$SOURCE_NAME:$LINENO:'
+        # Note that for '> $LINENO' the location token is set in _EvalRedirect.
+        # TODO: blame_tok should always be set.
+        if node.blame_tok is not None:
+          self.mem.SetLocationToken(node.blame_tok)
 
         self._MaybeRunDebugTrap()
 
@@ -729,7 +725,7 @@ def _Dispatch(self, node, cmd_st):
         # Expanded aliases need redirects and env bindings from the calling
         # context, as well as redirects in the expansion!
 
-        # TODO: SetCurrentSpanId to OUTSIDE?  Don't bother with stuff inside
+        # TODO: SetLocationToken to OUTSIDE?  Don't bother with stuff inside
         # expansion, since aliases are discouraged.
 
         if len(node.more_env):
@@ -786,7 +782,7 @@ def _Dispatch(self, node, cmd_st):
 
       elif case(command_e.DBracket):
         node = cast(command.DBracket, UP_node)
-        self.mem.SetCurrentSpanId(node.left.span_id)
+        self.mem.SetLocationToken(node.left)
         self._MaybeRunDebugTrap()
 
         self.tracer.PrintSourceCode(node.left, node.right, self.arena)
@@ -798,7 +794,7 @@ def _Dispatch(self, node, cmd_st):
 
       elif case(command_e.DParen):
         node = cast(command.DParen, UP_node)
-        self.mem.SetCurrentSpanId(node.left.span_id)
+        self.mem.SetLocationToken(node.left)
         self._MaybeRunDebugTrap()
 
         self.tracer.PrintSourceCode(node.left, node.right, self.arena)
@@ -814,7 +810,7 @@ def _Dispatch(self, node, cmd_st):
         if mylib.PYTHON:
           # x = 'foo' in Hay blocks
           if node.keyword is None or node.keyword.id == Id.KW_Const:
-            self.mem.SetCurrentSpanId(node.lhs[0].name.span_id)  # point to var name
+            self.mem.SetLocationToken(node.lhs[0].name)  # point to var name
 
             # Note: there's only one LHS
             vd_lval = location.LName(node.lhs[0].name.tval)  # type: lvalue_t
@@ -825,7 +821,7 @@ def _Dispatch(self, node, cmd_st):
                               flags=_PackFlags(Id.KW_Const, state.SetReadOnly))
 
           else:
-            self.mem.SetCurrentSpanId(node.keyword.span_id)  # point to var
+            self.mem.SetLocationToken(node.keyword)  # point to var
 
             py_val = self.expr_ev.EvalExpr(node.rhs, loc.Missing)
             vd_lvals = []  # type: List[lvalue_t]
@@ -856,7 +852,7 @@ def _Dispatch(self, node, cmd_st):
 
         if mylib.PYTHON:  # DISABLED because it relies on CPytho now
           node = cast(command.PlaceMutation, UP_node)
-          self.mem.SetCurrentSpanId(node.keyword.span_id)  # point to setvar/set
+          self.mem.SetLocationToken(node.keyword)  # point to setvar/set
 
           with switch(node.keyword.id) as case2:
             if case2(Id.KW_SetVar):
@@ -936,7 +932,7 @@ def _Dispatch(self, node, cmd_st):
         which_scopes = self.mem.ScopesForWriting()
 
         for pair in node.pairs:
-          self.mem.SetCurrentSpanId(pair.left.span_id)
+          self.mem.SetLocationToken(pair.left)
 
           if pair.op == assign_op_e.PlusEqual:
             assert pair.rhs, pair.rhs  # I don't think a+= is valid?
@@ -991,7 +987,7 @@ def _Dispatch(self, node, cmd_st):
         node = cast(command.Expr, UP_node)
 
         if mylib.PYTHON:
-          self.mem.SetCurrentSpanId(node.keyword.span_id)
+          self.mem.SetLocationToken(node.keyword)
           obj = self.expr_ev.EvalExpr(node.e, loc.Missing)
 
           if node.keyword.id == Id.Lit_Equals:
@@ -1143,7 +1139,7 @@ def _Dispatch(self, node, cmd_st):
 
       elif case(command_e.ForEach):
         node = cast(command.ForEach, UP_node)
-        self.mem.SetCurrentSpanId(node.keyword.span_id)  # for x in $LINENO
+        self.mem.SetLocationToken(node.keyword)  # for x in $LINENO
 
         # for the 2 kinds of shell loop
         iter_list = None  # type: List[str]  
@@ -1301,7 +1297,7 @@ def _Dispatch(self, node, cmd_st):
       elif case(command_e.ForExpr):
         node = cast(command.ForExpr, UP_node)
 
-        self.mem.SetCurrentSpanId(node.keyword.span_id)
+        self.mem.SetLocationToken(node.keyword)
         #self._MaybeRunDebugTrap()
 
         status = 0
@@ -1391,7 +1387,7 @@ def _Dispatch(self, node, cmd_st):
         str_val = self.word_ev.EvalWordToString(node.to_match)
         to_match = str_val.s
 
-        self.mem.SetCurrentSpanId(node.case_kw.span_id)
+        self.mem.SetLocationToken(node.case_kw)
         self._MaybeRunDebugTrap()
 
         status = 0  # If there are no arms, it should be zero?
@@ -1762,7 +1758,7 @@ def _MaybeRunDebugTrap(self):
 
       with dev.ctx_Tracer(self.tracer, 'trap DEBUG', None):
         with state.ctx_Registers(self.mem):  # prevent setting $? etc.
-          with state.ctx_DebugTrap(self.mem):  # for SetCurrentSpanId $LINENO
+          with state.ctx_DebugTrap(self.mem):  # for SetLocationToken $LINENO
             # Don't catch util.UserExit, etc.
             self._Execute(node)