View
92 osh/lex_test.py
@@ -13,7 +13,7 @@
from osh import parse_lib
from osh import ast_ as ast
from osh.lex import LEXER_DEF, LexMode
from osh.lex import LEXER_DEF
from osh import ast_ as ast
lex_mode_e = ast.lex_mode_e
@@ -46,97 +46,97 @@ def assertTokensEqual(self, left, right):
def testRead(self):
lexer = _InitLexer(CMD)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'ls'), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.WS_Space, ' '), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Lit_Chars, '/'), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Op_Newline, '\n'), t)
# Line two
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'ls'), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.WS_Space, ' '), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Lit_Chars, '/home/'), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Op_Newline, '\n'), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Eof_Real, ''), t)
# Another EOF gives EOF
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Eof_Real, ''), t)
def testRead_VS_ARG_UNQ(self):
# Another EOF gives EOF
lexer = _InitLexer("'hi'")
t = lexer.Read(LexMode.VS_ARG_UNQ)
t = lexer.Read(lex_mode_e.VS_ARG_UNQ)
#self.assertTokensEqual(ast.token(Id.Eof_Real, ''), t)
#t = l.Read(LexMode.VS_ARG_UNQ)
#t = l.Read(lex_mode_e.VS_ARG_UNQ)
print(t)
def testExtGlob(self):
lexer = _InitLexer('@(foo|bar)')
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.ExtGlob_At, '@('), t)
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'foo'), t)
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.Op_Pipe, '|'), t)
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'bar'), t)
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.Op_RParen, ')'), t)
# Individual cases
lexer = _InitLexer('@(')
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.ExtGlob_At, '@('), t)
lexer = _InitLexer('*(')
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.ExtGlob_Star, '*('), t)
lexer = _InitLexer('?(')
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.ExtGlob_QMark, '?('), t)
lexer = _InitLexer('$')
t = lexer.Read(LexMode.EXTGLOB)
t = lexer.Read(lex_mode_e.EXTGLOB)
self.assertTokensEqual(ast.token(Id.Lit_Other, '$'), t)
def testBashRegexState(self):
lexer = _InitLexer('(foo|bar)')
t = lexer.Read(LexMode.BASH_REGEX)
t = lexer.Read(lex_mode_e.BASH_REGEX)
self.assertTokensEqual(ast.token(Id.Lit_Chars, '('), t)
t = lexer.Read(LexMode.BASH_REGEX)
t = lexer.Read(lex_mode_e.BASH_REGEX)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'foo'), t)
t = lexer.Read(LexMode.BASH_REGEX)
t = lexer.Read(lex_mode_e.BASH_REGEX)
self.assertTokensEqual(ast.token(Id.Lit_Chars, '|'), t)
def testDBracketState(self):
lexer = _InitLexer('-z foo')
t = lexer.Read(LexMode.DBRACKET)
t = lexer.Read(lex_mode_e.DBRACKET)
self.assertTokensEqual(ast.token(Id.BoolUnary_z, '-z'), t)
self.assertEqual(Kind.BoolUnary, LookupKind(t.id))
@@ -145,27 +145,27 @@ def testLookAhead(self):
# the function; then Peek() the next token. Then Lookahead in that state.
lexer = _InitLexer('func()')
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'func'), t)
#self.assertEqual(Id.Op_LParen, lexer.LookAhead())
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Op_LParen, '('), t)
self.assertTokensEqual(
ast.token(Id.Op_RParen, ')'), lexer.LookAhead(LexMode.OUTER))
ast.token(Id.Op_RParen, ')'), lexer.LookAhead(lex_mode_e.OUTER))
lexer = _InitLexer('func ()')
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.Lit_Chars, 'func'), t)
t = lexer.Read(LexMode.OUTER)
t = lexer.Read(lex_mode_e.OUTER)
self.assertTokensEqual(ast.token(Id.WS_Space, ' '), t)
self.assertTokensEqual(
ast.token(Id.Op_LParen, '('), lexer.LookAhead(LexMode.OUTER))
ast.token(Id.Op_LParen, '('), lexer.LookAhead(lex_mode_e.OUTER))
class LineLexerTest(unittest.TestCase):
@@ -177,55 +177,55 @@ def testReadOuter(self):
# Lines always end with '\n'
l = LineLexer(LEXER_DEF, '')
try:
l.Read(LexMode.OUTER)
l.Read(lex_mode_e.OUTER)
except AssertionError as e:
print(e)
else:
raise AssertionError('Expected error')
l = LineLexer(LEXER_DEF, '\n')
self.assertTokensEqual(
ast.token(Id.Op_Newline, '\n'), l.Read(LexMode.OUTER))
ast.token(Id.Op_Newline, '\n'), l.Read(lex_mode_e.OUTER))
def testRead_VS_ARG_UNQ(self):
l = LineLexer(LEXER_DEF, "'hi'")
t = l.Read(LexMode.VS_ARG_UNQ)
t = l.Read(lex_mode_e.VS_ARG_UNQ)
self.assertEqual(Id.Left_SingleQuote, t.id)
def testLookAhead(self):
# Lines always end with '\n'
l = LineLexer(LEXER_DEF, '')
self.assertTokensEqual(
ast.token(Id.Unknown_Tok, ''), l.LookAhead(LexMode.OUTER))
ast.token(Id.Unknown_Tok, ''), l.LookAhead(lex_mode_e.OUTER))
l = LineLexer(LEXER_DEF, 'foo')
self.assertTokensEqual(
ast.token(Id.Lit_Chars, 'foo'), l.Read(LexMode.OUTER))
ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
self.assertTokensEqual(
ast.token(Id.Unknown_Tok, ''), l.LookAhead(LexMode.OUTER))
ast.token(Id.Unknown_Tok, ''), l.LookAhead(lex_mode_e.OUTER))
l = LineLexer(LEXER_DEF, 'foo bar')
self.assertTokensEqual(
ast.token(Id.Lit_Chars, 'foo'), l.Read(LexMode.OUTER))
ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
self.assertEqual(
ast.token(Id.Lit_Chars, 'bar'), l.LookAhead(LexMode.OUTER))
ast.token(Id.Lit_Chars, 'bar'), l.LookAhead(lex_mode_e.OUTER))
# No lookahead; using the cursor!
l = LineLexer(LEXER_DEF, 'func(')
self.assertTokensEqual(
ast.token(Id.Lit_Chars, 'func'), l.Read(LexMode.OUTER))
ast.token(Id.Lit_Chars, 'func'), l.Read(lex_mode_e.OUTER))
self.assertTokensEqual(
ast.token(Id.Op_LParen, '('), l.LookAhead(LexMode.OUTER))
ast.token(Id.Op_LParen, '('), l.LookAhead(lex_mode_e.OUTER))
l = LineLexer(LEXER_DEF, 'func (')
self.assertTokensEqual(
ast.token(Id.Lit_Chars, 'func'), l.Read(LexMode.OUTER))
ast.token(Id.Lit_Chars, 'func'), l.Read(lex_mode_e.OUTER))
self.assertTokensEqual(
ast.token(Id.Op_LParen, '('), l.LookAhead(LexMode.OUTER))
ast.token(Id.Op_LParen, '('), l.LookAhead(lex_mode_e.OUTER))
OUTER_RE = CompileAll(LEXER_DEF[LexMode.OUTER])
DOUBLE_QUOTED_RE = CompileAll(LEXER_DEF[LexMode.DQ])
OUTER_RE = CompileAll(LEXER_DEF[lex_mode_e.OUTER])
DOUBLE_QUOTED_RE = CompileAll(LEXER_DEF[lex_mode_e.DQ])
class FunctionTest(unittest.TestCase):
View
4 osh/osh.asdl
@@ -232,6 +232,10 @@ module osh
--
-- APIs
--
-- Fifteen lexer modes for osh.
-- Possible additional modes:
-- nested backticks: echo `echo \`echo foo\` bar`
lex_mode =
NONE
| COMMENT
View
112 osh/word_parse.py

Large diffs are not rendered by default.

Oops, something went wrong.
View
46 osh/word_parse_test.py
@@ -18,10 +18,10 @@
from osh import ast_ as ast
from osh import parse_lib
from osh.lex import LexMode
from osh.word_parse import WordParser # module under test
arith_expr_e = ast.arith_expr_e
lex_mode_e = ast.lex_mode_e
def _InitWordParserWithArena(s):
@@ -41,15 +41,15 @@ def InitWordParser(s):
def _assertReadWordWithArena(test, word_str):
print('\n---', word_str)
arena, w_parser = _InitWordParserWithArena(word_str)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w:
ast.PrettyPrint(w)
else:
err = w_parser.Error()
test.fail("Couldn't parse %r: %s" % (word_str, err))
# Next word must be \n
w2 = w_parser.ReadWord(LexMode.OUTER)
w2 = w_parser.ReadWord(lex_mode_e.OUTER)
test.assertTrue(
TokenWordsEqual(
ast.TokenWord(ast.token(Id.Op_Newline, '\n')),
@@ -78,7 +78,7 @@ def _assertSpanForWord(test, code_str):
def _assertReadWordFailure(test, word_str):
print('\n---', word_str)
w_parser = InitWordParser(word_str)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w:
ast.PrettyPrint(w)
test.fail('Expected a parser error, got %r' % w)
@@ -109,7 +109,7 @@ class WordParserTest(unittest.TestCase):
def testStaticEvalWord(self):
expr = r'\EOF' # Quoted here doc delimiter
w_parser = InitWordParser(expr)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
ok, s, quoted = word.StaticEval(w)
self.assertEqual(True, ok)
self.assertEqual('EOF', s)
@@ -340,7 +340,7 @@ def testRead(self):
w_parser = InitWordParser(expr)
while True:
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w is None:
e = w_parser.Error()
print('Error in word parser: %s' % e)
@@ -355,33 +355,33 @@ def testReadComment(self):
# Test that we get Id.Op_Newline
code = 'foo # comment\nbar #comment\n'
w_parser = InitWordParser(code)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w
self.assertEqual('foo', w.parts[0].token.val)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w
self.assertEqual(Id.Op_Newline, w.token.id)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w
self.assertEqual('bar', w.parts[0].token.val)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w
self.assertEqual(Id.Op_Newline, w.token.id)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w
self.assertEqual(Id.Eof_Real, w.token.id)
def testReadRegex(self):
# Test that we get Id.Op_Newline
code = '(foo|bar)'
w_parser = InitWordParser(code)
w_parser.next_lex_mode = LexMode.BASH_REGEX # needed at beginning
w_parser.next_lex_mode = lex_mode_e.BASH_REGEX # needed at beginning
w = w_parser.ReadWord(LexMode.BASH_REGEX)
w = w_parser.ReadWord(lex_mode_e.BASH_REGEX)
assert w
self.assertEqual('(', w.parts[0].token.val)
self.assertEqual('foo', w.parts[1].token.val)
@@ -390,7 +390,7 @@ def testReadRegex(self):
self.assertEqual(')', w.parts[4].token.val)
self.assertEqual(5, len(w.parts))
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w
self.assertEqual(Id.Op_Newline, w.token.id)
@@ -439,10 +439,10 @@ def testReadArith(self):
print()
w_parser = InitWordParser(expr)
w_parser._Next(LexMode.ARITH) # Can we remove this requirement?
w_parser._Next(lex_mode_e.ARITH) # Can we remove this requirement?
while True:
w = w_parser.ReadWord(LexMode.ARITH)
w = w_parser.ReadWord(lex_mode_e.ARITH)
if not w:
err = w_parser.Error()
print('ERROR', err)
@@ -461,31 +461,31 @@ def testMultiLine(self):
ls bar
""")
print('--MULTI')
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls'))]
self.assertEqual(ast.CompoundWord(parts), w)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'foo'))]
self.assertEqual(ast.CompoundWord(parts), w)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
t = ast.token(Id.Op_Newline, '\n')
self.assertEqual(ast.TokenWord(t), w)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls'))]
self.assertEqual(ast.CompoundWord(parts), w)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'bar'))]
self.assertEqual(ast.CompoundWord(parts), w)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
t = ast.token(Id.Op_Newline, '\n')
self.assertEqual(ast.TokenWord(t), w)
w = w_parser.ReadWord(LexMode.OUTER)
w = w_parser.ReadWord(lex_mode_e.OUTER)
t = ast.token(Id.Eof_Real, '')
self.assertEqual(ast.TokenWord(t), w)