Make style of compile-time choices in osh/match.py more consistent.

osh/lex_test.py

@@ -186,41 +186,41 @@ def assertTokensEqual(self, left, right):
    self.assertTrue(test_lib.TokensEqual(left, right))

  def testReadOuter(self):
    l = LineLexer(match.MakeMatcher(), '\n', self.arena)
    l = LineLexer(match.MATCHER, '\n', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Op_Newline, '\n'), l.Read(lex_mode_e.OUTER))

  def testRead_VS_ARG_UNQ(self):
    l = LineLexer(match.MakeMatcher(), "'hi'", self.arena)
    l = LineLexer(match.MATCHER, "'hi'", self.arena)
    t = l.Read(lex_mode_e.VS_ARG_UNQ)
    self.assertEqual(Id.Left_SingleQuote, t.id)

  def testLookAhead(self):
    # Lines always end with '\n'
    l = LineLexer(match.MakeMatcher(), '', self.arena)
    l = LineLexer(match.MATCHER, '', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Unknown_Tok, ''), l.LookAhead(lex_mode_e.OUTER))

    l = LineLexer(match.MakeMatcher(), 'foo', self.arena)
    l = LineLexer(match.MATCHER, 'foo', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
        ast.token(Id.Unknown_Tok, ''), l.LookAhead(lex_mode_e.OUTER))

    l = LineLexer(match.MakeMatcher(), 'foo  bar', self.arena)
    l = LineLexer(match.MATCHER, 'foo  bar', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'bar'), l.LookAhead(lex_mode_e.OUTER))

    # No lookahead; using the cursor!
    l = LineLexer(match.MakeMatcher(), 'func(', self.arena)
    l = LineLexer(match.MATCHER, 'func(', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'func'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(
        ast.token(Id.Op_LParen, '('), l.LookAhead(lex_mode_e.OUTER))

    l = LineLexer(match.MakeMatcher(), 'func  (', self.arena)
    l = LineLexer(match.MATCHER, 'func  (', self.arena)
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'func'), l.Read(lex_mode_e.OUTER))
    self.assertTokensEqual(

osh/match.py

@@ -48,20 +48,15 @@ def _MatchToken_Fast(lex_mode, line, start_pos):
  return IdInstance(tok_type), end_pos


def MakeMatcher():
  if fastlex:
    return _MatchToken_Fast
  else:
    return _MatchToken_Slow(lex.LEXER_DEF)


if fastlex:
  MATCHER = _MatchToken_Fast
  IsValidVarName = fastlex.IsValidVarName
else:
  import re
  MATCHER = _MatchToken_Slow(lex.LEXER_DEF)

# Used by osh/cmd_parse.py to validate for loop name.  Note it must be
# anchored on the right.
  # Used by osh/cmd_parse.py to validate for loop name.  Note it must be
  # anchored on the right.
  import re
  _VAR_NAME_RE = re.compile(lex.VAR_NAME_RE + '$')

  def IsValidVarName(s):

osh/parse_lib.py

@@ -12,7 +12,7 @@

def InitLexer(s, arena):
  """For tests only."""
  match_func = match.MakeMatcher()
  match_func = match.MATCHER
  line_lexer = lexer.LineLexer(match_func, '', arena)
  line_reader = reader.StringLineReader(s, arena)
  lx = lexer.Lexer(line_lexer, line_reader)
@@ -38,7 +38,7 @@ def InitLexer(s, arena):

def MakeParser(line_reader, arena):
  """Top level parser."""
  line_lexer = lexer.LineLexer(match.MakeMatcher(), '', arena)
  line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
  lx = lexer.Lexer(line_lexer, line_reader)
  w_parser = word_parse.WordParser(lx, line_reader)
  c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena)
@@ -57,7 +57,7 @@ def MakeParserForCompletion(code_str, arena):
  # NOTE: We don't need to use a arena here?  Or we need a "scratch arena" that
  # doesn't interfere with the rest of the program.
  line_reader = reader.StringLineReader(code_str, arena)
  line_lexer = lexer.LineLexer(match.MakeMatcher(), '', arena)  # AtEnd() is true
  line_lexer = lexer.LineLexer(match.MATCHER, '', arena)  # AtEnd() is true
  lx = lexer.Lexer(line_lexer, line_reader)
  w_parser = word_parse.WordParser(lx, line_reader)
  c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena)
@@ -66,14 +66,14 @@ def MakeParserForCompletion(code_str, arena):

def MakeWordParserForHereDoc(lines, arena):
  line_reader = reader.VirtualLineReader(lines, arena)
  line_lexer = lexer.LineLexer(match.MakeMatcher(), '', arena)
  line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
  lx = lexer.Lexer(line_lexer, line_reader)
  return word_parse.WordParser(lx, line_reader)


def MakeWordParserForPlugin(code_str, arena):
  line_reader = reader.StringLineReader(code_str, arena)
  line_lexer = lexer.LineLexer(match.MakeMatcher(), '', arena)
  line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
  lx = lexer.Lexer(line_lexer, line_reader)
  return word_parse.WordParser(lx, line_reader)

scripts/refactor.sh

@@ -48,6 +48,9 @@ replace() {
  local file=$1

  # NOTE: Escaping here is messed up.  sed doesn't have --name like awk?
  # To match literal parentheses I had to double-escape like this
  # (shell-escape, then sed-escape).
  # MakeMatcher\\(\\) MATCHER
  while read pat replace; do
    sed -r -i "s/${pat}/${replace}/g" */*.py
  done < $file