Parse += (append)

- Add assign_op LST field
- Test cases for executing +=
- Plan out the parsing of 'declare a[1]=x', which is different than
  the global 'a[1]=x', with test cases

core/cmd_exec.py

@@ -319,10 +319,6 @@ def _CheckStatus(self, status, node, argv0=None):
  def _EvalLhs(self, node):
    assert isinstance(node, ast.lhs_expr), node

    # NOTE: shares some logic with _EvalLhs in ArithEvaluator.  
    # TODO: Need to get the old value like _EvalLhs, for +=.  Maybe have a flag
    # that says whether you need it?

    if node.tag == lhs_expr_e.LhsName:  # a=x
      return runtime.LhsName(node.name)

@@ -675,7 +671,11 @@ def _Dispatch(self, node, fork_external):
          val = runtime.Str('')

        lval = self._EvalLhs(pair.lhs)
        # TODO: Respect readonly

        # TODO: Respect +=
        # See expr_eval.
        # old_val, lval = expr_eval.EvalLhs(mem, exec_opts, arith_eval)

        #log('ASSIGNING %s -> %s', lval, val)
        self.mem.SetVar(lval, val, flags, lookup_mode)

core/word.py

@@ -3,12 +3,12 @@
"""

import sys

from osh import ast_ as ast
from core.id_kind import Id, Kind, LookupKind

word_e = ast.word_e
word_part_e = ast.word_part_e
assign_op = ast.assign_op


def _LiteralPartId(p):
@@ -371,9 +371,23 @@ def AsArithVarName(w):


def LooksLikeAssignment(w):
  """Tests whether a word looke like FOO=bar.
  """Tests whether a word looks like FOO=bar.

  Returns:
    (string, CompoundWord) if it looks like FOO=bar
    False                  if it doesn't

  s=1
  s+=1
  s[x]=1
  s[x]+=1

  If so, return a (string, CompoundWord) pair.  Otherwise, return False.
  a=()
  a+=()
  a[x]=()
  a[x]+=()  # Not valid because arrays can't be nested.

  NOTE: a[ and s[ might be parsed separately?
  """
  assert w.tag == word_e.CompoundWord
  if len(w.parts) == 0:
@@ -383,8 +397,14 @@ def LooksLikeAssignment(w):
  if _LiteralPartId(part0) != Id.Lit_VarLike:
    return False

  assert part0.token.val.endswith('=')
  name = part0.token.val[:-1]
  s = part0.token.val
  assert s.endswith('=')
  if s[-2] == '+':
    op = assign_op.PlusEqual
    name = s[:-2]
  else:
    op = assign_op.Equal
    name = s[:-1]

  rhs = ast.CompoundWord()
  if len(w.parts) == 1:
@@ -395,7 +415,26 @@ def LooksLikeAssignment(w):
    for p in w.parts[1:]:
      rhs.parts.append(p)

  return name, rhs
  return name, op, rhs


# TODO:
# - local/declare should use this.
# - Doesn't work with 'readonly' or 'export'
# - global is parsed at the top level with LhsIndexedLike.
def LooksLikeLhsIndex(s):
  """Tests if a STRING looks like a[x + 1]=b

  # After EvalStatic, do another around of lexing at runtime.
  # Use osh/lex.py.

  Returns:
    (string, arith_expr) if it looks like a[x + 1]=b
    LhsIndexedName?

    False                  if it doesn't
  """
  # PROBLEM: What arena tokens to use?


def KeywordToken(w):

osh/cmd_parse.py

@@ -23,6 +23,7 @@

log = util.log
command_e = ast.command_e
assign_op = ast.assign_op


def _UnfilledHereDocs(redirects):
@@ -386,15 +387,15 @@ def _SplitSimpleCommandPrefix(self, words):

      left_spid = word.LeftMostSpanForWord(w)

      kv = word.LooksLikeAssignment(w)
      if kv:
        k, v = kv
      kov = word.LooksLikeAssignment(w)
      if kov:
        k, op, v = kov
        t = word.TildeDetect(v)
        if t:
          # t is an unevaluated word with TildeSubPart
          prefix_bindings.append((k, t, left_spid))
          prefix_bindings.append((k, op, t, left_spid))
        else:
          prefix_bindings.append((k, v, left_spid))  # v is unevaluated word
          prefix_bindings.append((k, op, v, left_spid))  # v is unevaluated word
      else:
        done_prefix = True
        suffix_words.append(w)
@@ -403,7 +404,7 @@ def _SplitSimpleCommandPrefix(self, words):

  def _MakeSimpleCommand(self, prefix_bindings, suffix_words, redirects):
    # FOO=(1 2 3) ls is not allowed
    for k, v, _ in prefix_bindings:
    for k, _, v, _ in prefix_bindings:
      if word.HasArrayPart(v):
        self.AddErrorContext(
            'Unexpected array literal in binding: %s', v, word=v)
@@ -413,9 +414,9 @@ def _MakeSimpleCommand(self, prefix_bindings, suffix_words, redirects):
    # NOTE: Other checks can be inserted here.  Can resolve builtins,
    # functions, aliases, static PATH, etc.
    for w in suffix_words:
      kv = word.LooksLikeAssignment(w)
      if kv:
        k, v = kv
      kov = word.LooksLikeAssignment(w)
      if kov:
        _, _, v = kov
        if word.HasArrayPart(v):
          self.AddErrorContext('Unexpected array literal: %s', v, word=v)
          return None
@@ -432,7 +433,12 @@ def _MakeSimpleCommand(self, prefix_bindings, suffix_words, redirects):
    node = ast.SimpleCommand()
    node.words = words3
    node.redirects = redirects
    for name, val, left_spid in prefix_bindings:
    for name, op, val, left_spid in prefix_bindings:
      if op != assign_op.Equal:
        # NOTE: Using spid of RHS for now, since we don't have one for op.
        self.AddErrorContext('Expected = in environment binding, got +=',
            word=val)
        return None
      pair = ast.env_pair(name, val)
      pair.spids.append(left_spid)
      node.more_env.append(pair)
@@ -461,15 +467,15 @@ def _MakeAssignment(self, assign_kw, suffix_words):
    while i < n:
      w = suffix_words[i]
      left_spid = word.LeftMostSpanForWord(w)
      kv = word.LooksLikeAssignment(w)
      if kv:
        k, v = kv
      kov = word.LooksLikeAssignment(w)
      if kov:
        k, op, v = kov
        t = word.TildeDetect(v)
        if t:
          # t is an unevaluated word with TildeSubPart
          pair = (k, t, left_spid)
          pair = (k, op, t, left_spid)
        else:
          pair = (k, v, left_spid)  # v is unevaluated word
          pair = (k, op, v, left_spid)  # v is unevaluated word
      else:
        # In aboriginal in variables/sources: export_if_blank does export "$1".
        # We should allow that.
@@ -494,8 +500,8 @@ def _MakeAssignment(self, assign_kw, suffix_words):

    # TODO: Also make with LhsIndexedName
    pairs = []
    for lhs, rhs, spid in bindings:
      p = ast.assign_pair(ast.LhsName(lhs), rhs)
    for lhs, op, rhs, spid in bindings:
      p = ast.assign_pair(ast.LhsName(lhs), op, rhs)
      p.spids.append(spid)
      pairs.append(p)

@@ -589,8 +595,8 @@ def ParseSimpleCommand(self):
        util.warn('WARNING: Got redirects in assignment: %s', redirects)

      pairs = []
      for lhs, rhs, spid in prefix_bindings:
        p = ast.assign_pair(ast.LhsName(lhs), rhs)
      for lhs, op, rhs, spid in prefix_bindings:
        p = ast.assign_pair(ast.LhsName(lhs), op, rhs)
        p.spids.append(spid)
        pairs.append(p)

@@ -611,7 +617,7 @@ def ParseSimpleCommand(self):
      if prefix_bindings:  # FOO=bar local spam=eggs not allowed
        # Use the location of the first value.  TODO: Use the whole word before
        # splitting.
        _, v0, _ = prefix_bindings[0]
        _, _, v0, _ = prefix_bindings[0]
        self.AddErrorContext(
            'Invalid prefix bindings in assignment: %s', prefix_bindings,
            word=v0)
@@ -1275,8 +1281,8 @@ def ParseCommand(self):

    if self.c_kind == Kind.Word:
      if self.w_parser.LookAhead() == Id.Op_LParen:  # (
        kv = word.LooksLikeAssignment(self.cur_word)
        if kv:
        kov = word.LooksLikeAssignment(self.cur_word)
        if kov:
          return self.ParseSimpleCommand()  # f=(a b c)  # array
        else:
          return self.ParseFunctionDef()  # f() { echo; }  # function

osh/osh.asdl

@@ -139,8 +139,8 @@ module osh
  | HereDoc(id op_id, int fd, word? body, int do_expansion,
            string here_end, bool was_filled)

  -- TODO: Add op_id for = vs +=
  assign_pair = (lhs_expr lhs, word? rhs)
  assign_op = Equal | PlusEqual
  assign_pair = (lhs_expr lhs, assign_op op, word? rhs)
  env_pair = (string name, word val)

  -- Each arm tests one word against multiple words

spec/append.test.sh

@@ -61,3 +61,52 @@ s2=$s1
s1+='d'
echo $s1 $s2
# stdout: abcd abc

### Append to nonexistent string
f() {
  local a+=a
  echo $a

  b+=b
  echo $b

  readonly c+=c
  echo $c

  export d+=d
  echo $d

  # Not declared anywhere
  e[1]+=e
  echo ${e[1]}

  # Declare is the same, but mksh doesn't support it
  #declare e+=e
  #echo $e
}
f
# stdout-json: "a\nb\nc\nd\ne\n"

### Append to nonexistent array
f() {
  # NOTE: mksh doesn't like a=() after keyword.  Doesn't allow local arrays!
  local x+=(a b)
  argv "${x[@]}"

  y+=(c d)
  argv "${y[@]}"

  readonly z+=(e f)
  argv "${z[@]}"
}
f
# stdout-json: "['a', 'b']\n['c', 'd']\n['e', 'f']\n"
# N-I mksh stdout-json: ""
# N-I mksh status: 1

### Append used like env prefix
# This should be an error but it's not.
A=a
A+=a printenv.py A
# BUG bash stdout: aa
# BUG mksh stdout: a

spec/assign.test.sh

@@ -73,7 +73,7 @@ echo "v=$v"
# nixpkgs setup.sh uses this (issue #26)
f() {
  local -a array=(x y z)
  argv "${array[@]}"
  argv.py "${array[@]}"
}
f
# stdout: ['x', 'y', 'z']
@@ -85,9 +85,25 @@ f
### declare -a
# nixpkgs setup.sh uses this (issue #26)
declare -a array=(x y z)
argv "${array[@]}"
argv.py "${array[@]}"
# stdout: ['x', 'y', 'z']
# N-I dash stdout-json: ""
# N-I dash status: 2
# N-I mksh stdout-json: ""
# N-I mksh status: 1

### typeset -a a[1]=a a[3]=c
# declare works the same way in bash, but not mksh.
# spaces are NOT allowed here.
typeset -a a[1*1]=x a[1+2]=z
argv.py "${a[@]}"
# stdout: ['x', 'z']
# N-I dash stdout-json: ""
# N-I dash status: 2

### indexed LHS without spaces is allowed
a[1 * 1]=x a[ 1 + 2 ]=z
argv.py "${a[@]}"
# stdout: ['x', 'z']
# N-I dash stdout-json: ""
# N-I dash status: 2

spec/smoke.test.sh

@@ -44,9 +44,9 @@ EOF
echo "var = $var"
# stdout: var = value

### Redirect
expr 3 > _tmp/expr3.txt
cat _tmp/expr3.txt
### Redirect external command
expr 3 > $TMP/expr3.txt
cat $TMP/expr3.txt
# stdout: 3
# stderr-json: ""

test/spec.sh

@@ -201,7 +201,7 @@ word-eval() {

# 'do' -- detected statically as syntax error?  hm.
assign() {
  sh-spec spec/assign.test.sh --osh-failures-allowed 1 \
  sh-spec spec/assign.test.sh --osh-failures-allowed 3 \
    ${REF_SHELLS[@]} $OSH "$@" 
}