[eggex] Improve error from type conversion func

Tickle this with

    test/ysh-runtime-errors.sh test-eggex-convert-func

Output:

  var pat = / <capture d+: evalExpr> /; var m = "10" => search(pat) => group(1)
                                                                            ^
[ -c flag ]:1: Fatal error calling Eggex conversion func 'evalExpr' from this Match accessor

  var pat = / <capture d+: evalExpr> /; var m = "10" => search(pat) => group(1)
                           ^~~~~~~~
[ -c flag ]:1: fatal: Arg 1 should be a Expr, got Str

builtin/func_eggex.py

@@ -4,7 +4,7 @@
 """
 from __future__ import print_function
 
-from _devbuild.gen.syntax_asdl import loc_t
+from _devbuild.gen.syntax_asdl import loc_t, Token
 from _devbuild.gen.value_asdl import (value, value_e, value_t, eggex_ops,
                                       eggex_ops_e, eggex_ops_t, regex_match_e,
                                       RegexMatch)
@@ -50,20 +50,24 @@ def _ReturnValue(self, match, group_index, blame_loc):
                 val = value.Str(match.s[start:end])  # type: value_t
 
                 convert_func = None  # type: Optional[value_t]
+                convert_tok = None  # type: Optional[Token]
                 with tagswitch(match.ops) as case:
                     if case(eggex_ops_e.Yes):
                         ops = cast(eggex_ops.Yes, match.ops)
 
                         # group 0 doesn't have a name or type attached to it
                         if len(ops.convert_funcs) and group_index != 0:
                             convert_func = ops.convert_funcs[group_index - 1]
+                            convert_tok = ops.convert_toks[group_index - 1]
 
-                if convert_func:
+                if convert_func is not None:
+                    assert convert_tok is not None
                     # Blame the group() call?  It would be nicer to blame the
                     # Token re.Capture.func_name, but we lost that in
                     # _EvalEggex()
                     val = self.expr_ev.CallConvertFunc(convert_func, val,
-                                                       blame_loc)
+                                                       convert_tok, blame_loc)
+
                 return val
         else:
             assert num_groups != 0

builtin/method_str.py

@@ -97,7 +97,7 @@ def Call(self, rd):
                 ere = regex_translate.AsPosixEre(eggex_val)
                 cflags = regex_translate.LibcFlags(eggex_val.canonical_flags)
                 capture = eggex_ops.Yes(
-                    eggex_val.convert_funcs, eggex_val.convert_locs,
+                    eggex_val.convert_funcs, eggex_val.convert_toks,
                     eggex_val.capture_names)  # type: eggex_ops_t
 
             elif case(value_e.Str):

core/value.asdl

@@ -48,7 +48,7 @@ module value
     # for BASH_REMATCH or ~ with a string
     No
     # These lists are indexed by group number, and will have None entries
-  | Yes(List[value?] convert_funcs, List[loc] convert_locs,
+  | Yes(List[value?] convert_funcs, List[Token?] convert_toks,
         List[str?] capture_names)
 
   RegexMatch = (str s, List[int] indices, eggex_ops ops)
@@ -89,7 +89,7 @@ module value
     # expr is spliced
     # / d+; ignorecase / -> '[[:digit:]]+' REG_ICASE
   | Eggex(re spliced, str canonical_flags,
-          List[value?] convert_funcs, List[loc] convert_locs,
+          List[value?] convert_funcs, List[Token?] convert_toks,
           # str? is because some groups are not named
           str? as_ere, List[str?] capture_names)
 

frontend/syntax.asdl

@@ -81,11 +81,13 @@ module syntax
 
   SourceLine = (int line_num, str content, source src)
 
-  # Token TODO:
-  # - remove .tval field.  If necessary, the string value should be manually
+  # Two ways to make Token smaller:
+  # - remove .tval field.  If necessary, the string value could be manually
   #   computed and attached to specific LST nodes.
-  # - maybe get rid of span_id, and re-compute length on demand too
-  #   -
+  #   - or is it easier to compute on demand?
+  # - Get rid of span_id, and re-compute length on demand too
+  #   - span_id is only used by tools/ysh_ify.py
+  #   - but length is used when computing .tval
   Token = (id id, int col, int length, int span_id, SourceLine? line, str tval)
 
   # Slight ASDL bug: CompoundWord has to be defined before using it as a shared
@@ -617,8 +619,7 @@ module syntax
 
   | Group(re child)
     # convert_func is filled in on evaluation
-    # TODO: the real type should be value? from core/runtime.asdl, but we don't
-    # have circular deps!
+    # TODO: name and func_name can be expanded to strings
   | Capture(re child, Token? name, Token? func_name)
   | Backtracking(bool negated, Token name, re child)
 

ysh/expr_eval.py

@@ -59,7 +59,7 @@
 from frontend import typed_args
 from osh import braces
 from osh import word_compile
-from mycpp.mylib import log, NewDict, switch, tagswitch
+from mycpp.mylib import log, NewDict, switch, tagswitch, print_stderr
 from ysh import func_proc
 from ysh import val_ops
 
@@ -355,22 +355,23 @@ def PluginCall(self, func_val, pos_args):
 
         return val
 
-    def CallConvertFunc(self, func_val, arg, blame_loc):
-        # type: (value_t, value_t, loc_t) -> value_t
+    def CallConvertFunc(self, func_val, arg, convert_tok, call_loc):
+        # type: (value_t, value_t, Token, loc_t) -> value_t
         """ For Eggex captures """
         with state.ctx_YshExpr(self.mutable_opts):
             pos_args = [arg]
             named_args = {}  # type: Dict[str, value_t]
             arg_list = ArgList.CreateNull()  # There's no call site
             rd = typed_args.Reader(pos_args, named_args, arg_list)
-            rd.SetCallLocation(blame_loc)
+            rd.SetCallLocation(convert_tok)
             try:
                 val = self._CallFunc(func_val, rd)
             except error.FatalRuntime as e:
-                # TODO: it needs a name
-                # This blames the group() call
-                self.errfmt.Print_('Fatal error calling Eggex conversion func',
-                                   blame_loc)
+                func_name = lexer.TokenVal(convert_tok)
+                self.errfmt.Print_(
+                    'Fatal error calling Eggex conversion func %r from this Match accessor'
+                    % func_name, call_loc)
+                print_stderr('')
                 raise
 
         return val
@@ -1191,7 +1192,7 @@ def EvalEggex(self, node):
 
         # as_ere and capture_names filled by ~ operator or Str method
         return value.Eggex(spliced, node.canonical_flags, ev.convert_funcs,
-                           ev.convert_locs, None, [])
+                           ev.convert_toks, None, [])
 
 
 class EggexEvaluator(object):
@@ -1201,7 +1202,7 @@ def __init__(self, mem, canonical_flags):
         self.mem = mem
         self.canonical_flags = canonical_flags
         self.convert_funcs = []  # type: List[Optional[value_t]]
-        self.convert_locs = []  # type: List[loc_t]
+        self.convert_toks = []  # type: List[Optional[Token]]
 
     def _LookupVar(self, name, var_loc):
         # type: (str, loc_t) -> value_t
@@ -1302,28 +1303,28 @@ def EvalE(self, node):
 
                 # placeholder for non-capturing group
                 self.convert_funcs.append(None)
-                self.convert_locs.append(None)
+                self.convert_toks.append(None)
                 return re.Group(self.EvalE(node.child))
 
             elif case(re_e.Capture):  # Identical to Group
                 node = cast(re.Capture, UP_node)
-                convert_func = None  # type: value_t
-                convert_loc = loc.Missing  # type: loc_t
+                convert_func = None  # type: Optional[value_t]
+                convert_tok = None  # type: Optional[Token]
                 if node.func_name:
                     func_name = lexer.TokenVal(node.func_name)
                     func_val = self.mem.GetValue(func_name)
                     with tagswitch(func_val) as case:
                         if case(value_e.Func, value_e.BuiltinFunc):
                             convert_func = func_val
-                            convert_loc = node.func_name
+                            convert_tok = node.func_name
                         else:
                             raise error.TypeErr(
                                 func_val,
                                 "Expected %r to be a func" % func_name,
                                 node.func_name)
 
                 self.convert_funcs.append(convert_func)
-                self.convert_locs.append(convert_loc)
+                self.convert_toks.append(convert_tok)
                 return re.Capture(self.EvalE(node.child), node.name,
                                   node.func_name)
 
@@ -1392,7 +1393,7 @@ def EvalE(self, node):
 
                         # Splicing means we get the conversion funcs too.
                         self.convert_funcs.extend(val.convert_funcs)
-                        self.convert_locs.extend(val.convert_locs)
+                        self.convert_toks.extend(val.convert_toks)
 
                         # Splicing requires flags to match.  This check is
                         # transitive.

ysh/val_ops.py

@@ -455,7 +455,7 @@ def MatchRegex(left, right, mem):
 
             right_s = regex_translate.AsPosixEre(right)
             regex_flags = regex_translate.LibcFlags(right.canonical_flags)
-            capture = eggex_ops.Yes(right.convert_funcs, right.convert_locs,
+            capture = eggex_ops.Yes(right.convert_funcs, right.convert_toks,
                                     right.capture_names)
 
         else: