Make '$(x' a parse error.

The Eof_Real token was being treated as a substitute for Eof_RParen,
which was invalid.

Fixes issue #144.

osh/cmd_parse.py

@@ -262,13 +262,16 @@ class CommandParser(object):
    lexer: for lookahead in function def, PushHint of ()
    line_reader: for here doc
  """
  def __init__(self, parse_ctx, w_parser, lexer_, line_reader, arena=None):
  def __init__(self, parse_ctx, w_parser, lexer_, line_reader, arena=None,
               eof_id=Id.Eof_Real):
    self.parse_ctx = parse_ctx
    self.aliases = parse_ctx.aliases  # aliases to expand at parse time

    self.w_parser = w_parser  # for normal parsing
    self.lexer = lexer_  # for pushing hints, lookahead to (
    self.line_reader = line_reader  # for here docs
    self.arena = arena or parse_ctx.arena  # for adding here doc and alias spans
    self.aliases = parse_ctx.aliases  # aliases to expand at parse time
    self.eof_id = eof_id

    self.Reset()

@@ -1542,10 +1545,8 @@ def _ParseCommandTerm(self):
    Returns:
      ast.command
    """
    # Word types that will end the command term.
    END_LIST = (
        Id.Eof_Real, Id.Eof_RParen, Id.Eof_Backtick, Id.Right_Subshell,
        Id.Lit_RBrace, Id.Op_DSemi)
    # Token types that will end the command term.
    END_LIST = (self.eof_id, Id.Right_Subshell, Id.Lit_RBrace, Id.Op_DSemi)

    # NOTE: This is similar to _ParseCommandLine.
    #

osh/parse_lib.py

@@ -41,13 +41,14 @@ def MakeWordParserForHereDoc(self, line_reader):
    lx = lexer.Lexer(line_lexer, line_reader)
    return word_parse.WordParser(self, lx, line_reader)

  def MakeParserForCommandSub(self, line_reader, lexer):
  def MakeParserForCommandSub(self, line_reader, lexer, eof_id):
    """To parse command sub, we want a fresh word parser state.

    It's a new instance based on same lexer and arena.
    """
    w_parser = word_parse.WordParser(self, lexer, line_reader)
    c_parser = cmd_parse.CommandParser(self, w_parser, lexer, line_reader)
    c_parser = cmd_parse.CommandParser(self, w_parser, lexer, line_reader,
                                       eof_id=eof_id)
    return c_parser

  def MakeWordParserForPlugin(self, code_str, arena):
@@ -70,8 +71,8 @@ def MakeParserForCompletion(self, code_str, arena):

    NOTE: Uses its own arena!
    """
    # NOTE: We don't need to use a arena here?  Or we need a "scratch arena" that
    # doesn't interfere with the rest of the program.
    # NOTE: We don't need to use a arena here?  Or we need a "scratch arena"
    # that doesn't interfere with the rest of the program.
    line_reader = reader.StringLineReader(code_str, arena)
    line_lexer = lexer.LineLexer(match.MATCHER, '', arena)  # AtEnd() is true
    lx = lexer.Lexer(line_lexer, line_reader)

osh/word_parse.py

@@ -638,32 +638,38 @@ def _ReadDoubleQuotedPart(self):
    dq_part.spids.append(self.cur_token.span_id)  # Right "
    return dq_part

  def _ReadCommandSubPart(self, token_type):
  def _ReadCommandSubPart(self, left_id):
    """
    NOTE: This is not in the grammar, because word parts aren't in the grammar!

    command_sub = '$(' command_list ')'
                | ` command_list `
                | '<(' command_list ')'
                | '>(' command_list ')'
    """
    left_token = self.cur_token
    left_spid = left_token.span_id

    self._Next(lex_mode_e.OUTER)  # advance past $( or `

    # Set the lexer in a state so ) becomes the EOF token.
    if token_type in (
        Id.Left_CommandSub, Id.Left_ProcSubIn, Id.Left_ProcSubOut):
      self.lexer.PushHint(Id.Op_RParen, Id.Eof_RParen)
    elif token_type == Id.Left_Backtick:
      self.lexer.PushHint(Id.Left_Backtick, Id.Eof_Backtick)
    if left_id in (Id.Left_CommandSub, Id.Left_ProcSubIn, Id.Left_ProcSubOut):
      right_id = Id.Eof_RParen
      self.lexer.PushHint(Id.Op_RParen, right_id)

    elif left_id == Id.Left_Backtick:
      right_id = Id.Eof_Backtick
      self.lexer.PushHint(Id.Left_Backtick, right_id)

    else:
      raise AssertionError(self.token_type)
      raise AssertionError(self.left_id)

    c_parser = self.parse_ctx.MakeParserForCommandSub(self.line_reader,
                                                      self.lexer)
                                                      self.lexer, right_id)

    # NOTE: This doesn't use something like main_loop because we don't want to
    # interleave parsing and execution!  Unlike 'source' and 'eval'.
    node = c_parser.ParseCommandSub()  # `` and $() allowed
    node = c_parser.ParseCommandSub()
    assert node is not None

    # Hm this creates its own word parser, which is thrown away?

test/spec.sh

@@ -388,8 +388,7 @@ explore-parsing() {
}

parse-errors() {
  sh-spec spec/parse-errors.test.sh --osh-failures-allowed 2 \
    ${REF_SHELLS[@]} $OSH_LIST "$@"
  sh-spec spec/parse-errors.test.sh ${REF_SHELLS[@]} $OSH_LIST "$@"
}

here-doc() {