[ysh parser] Unify parsing of proc and func params

No bchange in behavior yet, but it greatly simplifies the code.

To avoid breakage, the only thing I had to do is make sure 'out Ref' is
handled, where Ref is a type_expr.

doc/ref/toc-ysh.md

@@ -266,9 +266,8 @@ X [Codecs]        quoteUrl()  quoteHtml()  quoteSh()   quoteC()
 X [Serialize]     toJ8()   fromJ8()
                   toJson()   fromJson()
   [Word]          glob()   maybe()
-  [Introspection] shvar_get()   VM.procs()   VM.types()
-                  evalExpr()
-  [Hay Config]    parse_hay()   eval_hay()   block_as_str()   
+  [Introspection] shvar_get()   procs()   evalExpr()
+  [Hay Config]    parse_hay()   eval_hay()
 X [Date Time]     strftime()
 X [Wok]           _field()
 X [Hashing]       sha1dc()   sha256()

frontend/parse_lib.py

@@ -382,7 +382,7 @@ def ParseProc(self, lexer, out):
         """proc f(x, y, @args) {"""
         e_parser = self._YshParser()
         with ctx_PNodeAllocator(e_parser):
-            pnode, last_token = e_parser.Parse(lexer, grammar_nt.oil_proc)
+            pnode, last_token = e_parser.Parse(lexer, grammar_nt.ysh_proc)
 
             if 0:
                 self.p_printer.Print(pnode)

frontend/syntax.asdl

@@ -404,16 +404,16 @@ module syntax
 
   ImportName = (Token name, Token? alias)
 
-  # type is Ref, Expr, or Block
-  ProcParam = (Token name, Token? type, expr? default_val)
-
   # 'open' is for proc p { }; closed is for proc p () { }
   proc_sig =
     Open
-  | Closed(List[ProcParam] pos_params, Token? rest)
+  | Closed(List[Param] words, Token? rest_words,
+           List[Param] typed, Token? rest_typed,
+           List[Param] named, Token? rest_named,
+           Param? block_param)
 
-  # prefix is : for out param, @ for proc splat, ... for func splat
-  # procs only have types Expr, Block (and Str is implicit)
+  # prefix is : for out param, ... for splat
+  # procs only have types Ref, Expr, Block (and Str is implicit)
   Param = (Token? prefix, Token name, type_expr? type, expr? default_val)
 
   #
@@ -450,7 +450,7 @@ module syntax
   # type expressions:   Int   Array[Int]   Dict[Str, Any]
   type_expr =
     Simple(Token tok, str name)
-  | Compound(Token name, List[type_expr] params)
+  | Compound(Token tok, str name, List[type_expr] params)
 
   # LHS binding in loops, list comprehensions, and var/const
   NameType = (Token name, type_expr? typ)

osh/cmd_eval.py

@@ -48,6 +48,7 @@
     for_iter_e,
     pat,
     pat_e,
+    type_expr, type_expr_e,
 )
 from _devbuild.gen.runtime_asdl import (
     lvalue,
@@ -1487,8 +1488,8 @@ def _Dispatch(self, node, cmd_st):
                 if UP_sig.tag() == proc_sig_e.Closed:
                     sig = cast(proc_sig.Closed, UP_sig)
                     no_val = None  # type: value_t
-                    defaults = [no_val] * len(sig.pos_params)
-                    for i, p in enumerate(sig.pos_params):
+                    defaults = [no_val] * len(sig.words)
+                    for i, p in enumerate(sig.words):
                         if p.default_val:
                             val = self.expr_ev.EvalExpr(
                                 p.default_val, loc.Missing)
@@ -2016,8 +2017,15 @@ def RunProc(self, proc, argv, arg0_loc):
 
             if UP_sig.tag() == proc_sig_e.Closed:  # proc is-closed ()
                 sig = cast(proc_sig.Closed, UP_sig)
-                for i, p in enumerate(sig.pos_params):
-                    is_out_param = p.type is not None and p.type.tval == 'Ref'
+                for i, p in enumerate(sig.words):
+
+                    # proc p(out Ref)
+                    is_out_param = False
+                    if p.type is not None:
+                        if p.type.tag() == type_expr_e.Simple:
+                            typ = cast(type_expr.Simple, p.type)
+                            if typ.tok.tval == 'Ref':
+                                is_out_param = True
 
                     param_name = p.name.tval
                     if i < n_args:
@@ -2055,12 +2063,12 @@ def RunProc(self, proc, argv, arg0_loc):
                                       scope_e.LocalOnly,
                                       flags=flags)
 
-                n_params = len(sig.pos_params)
-                if sig.rest:
+                n_params = len(sig.words)
+                if sig.rest_words:
                     items = [value.Str(s)
                              for s in argv[n_params:]]  # type: List[value_t]
                     leftover = value.List(items)
-                    self.mem.SetValue(location.LName(sig.rest.tval), leftover,
+                    self.mem.SetValue(location.LName(sig.rest_words.tval), leftover,
                                       scope_e.LocalOnly)
                 else:
                     if n_args > n_params:

osh/cmd_parse.py

@@ -1997,10 +1997,10 @@ def ParseYshProc(self):
                 sig = cast(proc_sig.Closed, node.sig)
 
                 # Treat params as variables.
-                for param in sig.pos_params:
+                for param in sig.words:
                     self.var_checker.Check(Id.KW_Var, param.name)
-                if sig.rest:
-                    self.var_checker.Check(Id.KW_Var, sig.rest)
+                if sig.rest_words:
+                    self.var_checker.Check(Id.KW_Var, sig.rest_words)
                     # We COULD register __out here but it would require a different API.
                     #if param.prefix and param.prefix.id == Id.Arith_Colon:
                     #  self.var_checker.Check(Id.KW_Var, '__' + param.name)

spec/ysh-builtin-argparse.test.sh

@@ -13,16 +13,19 @@ Args :spec {
   flag -v --verbose (Bool)
   arg src
   arg dst
-}
-var argv = ['-v', 'src/path', 'dst/path']
 
-# Gah we don't have destructuring assignment?
-# Also need to define :spec
+  rest more  # allow more args
+}
+json write (spec)
 
-var arg = parseArgs(spec, argv)
+var argv = ['-v', 'src/path', 'dst/path']
 
+# TODO: need destructuring with var
 # var arg, i = parseArgs(spec, argv)
 
+var result = parseArgs(spec, argv)
+setvar arg, i = result
+
 json write (arg)
 json write (i)
 ## STDOUT:

spec/ysh-proc.test.sh

@@ -150,7 +150,7 @@ f a b
 # With varargs and block
 shopt --set parse_proc
 
-proc g(x, y, ...rest; block = null) {
+proc g(x, y, ...rest; block) {
   echo G
 }
 g a b c d

stdlib/args.ysh

@@ -16,7 +16,21 @@
 # var arg, i = parseArgs(spec, ARGV)
 #
 # echo "Verbose $[arg.verbose]"
+
+
+# TODO: how to register these as Args ?
 #
+# var proc_bindings = {}
+# var var_bindings = {}
+# eval ()
+
+proc _Args_flag {
+  echo 'flag'
+}
+
+proc _Args_arg {
+  echo 'arg'
+}
 
 proc Args(spec Ref ; ; block) {
   # TODO: evaluate block somehow
@@ -25,10 +39,24 @@ proc Args(spec Ref ; ; block) {
   #
   # You have to put 'flag' and 'arg' in scope
 
-  echo hi
+  # Flags have these fields
+  # default type is Bool
+  var flag = {short: '', long: '', type: null, default: '', help: ''}
+
+  # Args have these fields
+  #
+  # I think all args are required -- you can ARGV[i:] if you want something else
+  #
+  # Default type is string
+  var arg = {name: '', type: null, help: ''}
+
+  # May also have program name, etc.
+  var result = {flags: [], args: []}
 
   #= spec
   #= block
+
+  setref spec = result
 }
 
 func parseArgs(spec, argv) {

stdlib/testing.ysh

@@ -8,15 +8,70 @@
 # describe foo {
 #   assert (43 === f(42))
 # }
+#
+# if is-main {
+#   run-tests @ARGV   # --filter
+# }
 
-proc describe (...desc, ) {
-  echo hi
-}
 
 # is 'cond Expr' lazily evaluated?
 # Or all proc arguments are autoomatically lazy, except words?
 
 proc assert (; cond ) {
   echo hi
+
+  # TODO: This is a builtin cond
+  var val = evalExpr(cond) 
+  if (not val) {
+    # TODO: if it's an expr.Binary
+    # Then
+    #
+    # Then print $left != $right
+    #
+    # I think you need to introspect on the source code
+    #
+    # Or print '5 != 3'
+    #
+    # Or you can evaluate left and right separately, and then compare them
+
+    echo
+  }
+}
+
+
+# is this accessible to users?
+# It can contain a global list of things to run
+
+var _describe = []
+
+proc describe (...desc; ; cmd) {
+  echo describe
+  = desc
+
+  # TODO:
+  # - need append
+  # - need ::
+  # _ _describe->append(cmd)
+  #
+  # Need to clean this up
+  # append (_describe, cmd)  # does NOT work!
+
+  _ append(_describe, cmd)
+}
+
+proc run-tests (...argv) {
+  # TODO:
+  # - parse --filter foo, which you can use eggex for!
+
+  for cmd in (_describe) {
+    # TODO: print filename and 'describe' name?
+    try {
+
+      eval (cmd)
+    }
+    if (_status !== 0) {
+      echo 'failed'
+    }
+  }
 }
 

test/ysh-parse-errors.sh

@@ -85,6 +85,12 @@ test-func-var-checker() {
   '
 }
 
+test-proc-sig() {
+  _should-parse 'proc p () { echo hi }'
+  _should-parse 'proc p (a) { echo hi }'
+  _should-parse 'proc p (out Ref) { echo hi }'
+}
+
 soil-run() {
   # This is like run-test-funcs, except errexit is off here
   run-test-funcs