View
116 osh/parse_lib.py
@@ -55,60 +55,68 @@ def InitLexer(s, arena):
# common thing: word parser does not use arena OR aliases. But it needs to
# create a command parser.
class ParseContext(object):
"""Context shared between the mutually recursive Command and Word parsers.
def MakeParser(line_reader, arena, aliases):
"""Top level parser."""
line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(lx, line_reader)
c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena,
aliases=aliases)
return w_parser, c_parser
# TODO: We could reuse w_parser with Reset() each time. That's what the REPL
# does.
# But LineLexer and Lexer are also stateful! So that might not be worth it.
# Hm the REPL only does line_reader.Reset()?
#
# NOTE: It probably needs to take a VirtualLineReader for $PS1, $PS2, ...
# values.
def MakeParserForCompletion(code_str, arena):
"""Parser for partial lines."""
# NOTE: We don't need to use a arena here? Or we need a "scratch arena" that
# doesn't interfere with the rest of the program.
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '', arena) # AtEnd() is true
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(lx, line_reader)
c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena)
return w_parser, c_parser
def MakeWordParserForHereDoc(line_reader, arena):
line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
return word_parse.WordParser(lx, line_reader)
def MakeWordParserForPlugin(code_str, arena):
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
return word_parse.WordParser(lx, line_reader)
def MakeParserForCommandSub(line_reader, lexer):
"""To parse command sub, we want a fresh word parser state.
It's a new instance based on same lexer and arena.
In constrast, STATE is stored in the CommandParser and WordParser instances.
"""
arena = line_reader.arena
w_parser = word_parse.WordParser(lexer, line_reader)
c_parser = cmd_parse.CommandParser(w_parser, lexer, line_reader, arena)
return c_parser
# Another parser instantiation:
# - For Array Literal in word_parse.py WordParser:
# w_parser = WordParser(self.lexer, self.line_reader)
def __init__(self, arena, aliases):
self.arena = arena
self.aliases = aliases
def MakeParser(self, line_reader):
line_lexer = lexer.LineLexer(match.MATCHER, '', self.arena)
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(self, lx, line_reader)
c_parser = cmd_parse.CommandParser(self, w_parser, lx, line_reader, self.arena,
aliases=self.aliases)
return w_parser, c_parser
def MakeWordParserForHereDoc(self, line_reader):
line_lexer = lexer.LineLexer(match.MATCHER, '', self.arena)
lx = lexer.Lexer(line_lexer, line_reader)
return word_parse.WordParser(self, lx, line_reader)
def MakeParserForCommandSub(self, line_reader, lexer):
"""To parse command sub, we want a fresh word parser state.
It's a new instance based on same lexer and arena.
"""
w_parser = word_parse.WordParser(self, lexer, line_reader)
c_parser = cmd_parse.CommandParser(self, w_parser, lexer, line_reader,
self.arena)
return c_parser
def MakeWordParserForPlugin(self, code_str, arena):
"""FOr $PS1, etc.
NOTE: Uses its own arena! I think that does nothing though?
"""
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
return word_parse.WordParser(self, lx, line_reader)
# TODO: We could reuse w_parser with ResetInputObjects() each time. That's
# what the REPL does.
#
# NOTE: It probably needs to take a VirtualLineReader for $PS1, $PS2, ...
# values.
def MakeParserForCompletion(self, code_str, arena):
"""Parser for partial lines.
NOTE: Uses its own arena!
"""
# NOTE: We don't need to use a arena here? Or we need a "scratch arena" that
# doesn't interfere with the rest of the program.
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '', arena) # AtEnd() is true
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(self, lx, line_reader)
c_parser = cmd_parse.CommandParser(self, w_parser, lx, line_reader, arena)
return w_parser, c_parser
# Another parser instantiation:
# - For Array Literal in word_parse.py WordParser:
# w_parser = WordParser(self.lexer, self.line_reader)
View
8 osh/word_parse.py
@@ -66,7 +66,8 @@
class WordParser(object):
def __init__(self, lexer, line_reader, lex_mode=lex_mode_e.OUTER):
def __init__(self, parse_ctx, lexer, line_reader, lex_mode=lex_mode_e.OUTER):
self.parse_ctx = parse_ctx
self.lexer = lexer
self.line_reader = line_reader
self.Reset(lex_mode=lex_mode)
@@ -662,8 +663,7 @@ def _ReadCommandSubPart(self, token_type):
else:
raise AssertionError(self.token_type)
from osh import parse_lib
c_parser = parse_lib.MakeParserForCommandSub(self.line_reader, self.lexer)
c_parser = self.parse_ctx.MakeParserForCommandSub(self.line_reader, self.lexer)
# NOTE: This doesn't use something like main_loop because we don't want to
# interleave parsing and execution! Unlike 'source' and 'eval'.
@@ -837,7 +837,7 @@ def _ReadArrayLiteralPart(self):
token=self.cur_token)
# MUST use a new word parser (with same lexer).
w_parser = WordParser(self.lexer, self.line_reader)
w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.OUTER)
View
5 osh/word_parse_test.py
@@ -21,7 +21,6 @@
from osh.meta import ast, Id, types
from osh import ast_lib
from osh import parse_lib
from osh.word_parse import WordParser # module under test
arith_expr_e = ast.arith_expr_e
lex_mode_e = types.lex_mode_e
@@ -31,8 +30,10 @@ def _InitWordParserWithArena(s):
pool = alloc.Pool()
arena = pool.NewArena()
arena.PushSource('word_parse_test.py')
parse_ctx = parse_lib.ParseContext(arena, {})
line_reader, lexer = parse_lib.InitLexer(s, arena)
w_parser = WordParser(lexer, line_reader)
w_parser, _ = parse_ctx.MakeParser(line_reader)
return arena, w_parser