From 3576e40670b2b781a5442e61717fa4c94d571810 Mon Sep 17 00:00:00 2001 From: Mike Bjorge Date: Mon, 24 Apr 2017 18:17:42 -0700 Subject: [PATCH] Split the lexer from the parser and put them in classes. In anticipation of the parser getting more complex with logic for improved error messaging, split out the lexer to keep parser.py cleaner. Move the lexer and the parser into their own classes. This makes it easier and cleaner to keep a references to lexer / parser instances which are needed to do better error messaging when a syntax error comes up. Issue: #2 Test: stl/parser_test.py --- stl/lexer.py | 142 ++++++++++++++++++ stl/parser.py | 354 +++++++++++++++++---------------------------- stl/parser_test.py | 6 +- 3 files changed, 278 insertions(+), 224 deletions(-) create mode 100644 stl/lexer.py diff --git a/stl/lexer.py b/stl/lexer.py new file mode 100644 index 0000000..cd04c35 --- /dev/null +++ b/stl/lexer.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lexing (tokenizing) a state transition (STL) spec.""" + +# pylint: disable=g-doc-args +# pylint: disable=g-docstring-missing-newline +# pylint: disable=g-docstring-quotes +# pylint: disable=g-no-space-after-docstring-summary +# pylint: disable=g-short-docstring-punctuation +# pylint: disable=g-short-docstring-space +# pylint: disable=invalid-name +# pylint: disable=unused-variable + +import logging +import ply.lex # pylint: disable=g-bad-import-order + + +class StlSyntaxError(SyntaxError): + """Error for incorrect STL syntax.""" + + +class StlLexer(object): + + def __init__(self, filename=None, **kwargs): + """Create a Lex lexer. + + To pass this into a Ply Yacc parser, pass it in using the .lexer propert + of an StlLexer instance: + my_lexer = StlLexer() + my_parser = ply.yacc.parser(lexer=my_lexer.lexer) + + Args: + filename: The filename string to use in any error messaging. + kwargs: Forwarded to ply.lex.lex. + """ + self.filename = filename + self.lexer = ply.lex.lex(module=self, **kwargs) + + RESERVED = { + 'bool': 'BOOL', + 'const': 'CONST', + 'encode': 'ENCODE', + 'error_states': 'ERROR_STATES', + 'event': 'EVENT', + 'events': 'EVENTS', + 'external': 'EXTERNAL', + 'int': 'INT', + 'message': 'MESSAGE', + 'module': 'MODULE', + 'optional': 'OPTIONAL', + 'post_states': 'POST_STATES', + 'pre_states': 'PRE_STATES', + 'qualifier': 'QUALIFIER', + 'repeated': 'REPEATED', + 'required': 'REQUIRED', + 'role': 'ROLE', + 'state': 'STATE', + 'string': 'STRING', + 'transition': 'TRANSITION', + } + + # |literals| is a special field for ply.lex. Each of these + # characters is interpreted as a separate token + literals = ':;{}()[]=,.&' + + # |tokens| is a special field for ply.lex. This must contain a list + # of all possible token types. + tokens = [ + 'ARROW', # -> + 'BOOLEAN', + 'NAME', + 'NULL', + 'NUMBER', + 'STRING_LITERAL', + ] + list(RESERVED.values()) + + # A string containing ignored characters (spaces and tabs) + t_ignore = ' \t' + + def t_ARROW(self, t): + r'->' + return t + + def t_BOOLEAN(self, t): + r'(true|false)' + t.value = (t.value == 'true') + return t + + def t_NULL(self, t): + r'null' + t.value = None + return t + + def t_NAME(self, t): + r'[a-zA-Z_]\w*' + t.type = self.RESERVED.get(t.value, 'NAME') + return t + + def t_NUMBER(self, t): + r'-?\d+' + t.value = int(t.value) + return t + + def t_STRING_LITERAL(self, t): + r'"([^\\"]|\\"|\\\\)*"' + t.value = t.value[1:-1].replace('\\"', '"').replace('\\\\', '\\') + return t + + def t_COMMENT(self, t): + r'//.*' + del t # unused argument + + # Define a rule so we can track line numbers. + def t_newline(self, t): + r'\n+' + t.lexer.lineno += len(t.value) + + # Error handling rule. + def t_error(self, t): + logging.error('[%s:%d] Illegal character: %s', + self.filename, + t.lexer.lineno, + t.value[0]) + t.lexer.skip(1) + + def debug(data): + """Print out all the tokens in |data|.""" + for token in self.lexer.tokens(): + print token diff --git a/stl/parser.py b/stl/parser.py index c94234f..a055a65 100644 --- a/stl/parser.py +++ b/stl/parser.py @@ -24,13 +24,13 @@ # pylint: disable=unused-variable import logging -import ply.lex # pylint: disable=g-bad-import-order import ply.yacc # pylint: disable=g-bad-import-order import pprint import sys import stl.base import stl.event +import stl.lexer import stl.message import stl.module import stl.qualifier @@ -41,129 +41,50 @@ class StlSyntaxError(SyntaxError): """Error for incorrect STL syntax.""" -################################################### -# Lexer - -reserved = { - 'bool': 'BOOL', - 'const': 'CONST', - 'encode': 'ENCODE', - 'error_states': 'ERROR_STATES', - 'event': 'EVENT', - 'events': 'EVENTS', - 'external': 'EXTERNAL', - 'int': 'INT', - 'message': 'MESSAGE', - 'module': 'MODULE', - 'optional': 'OPTIONAL', - 'post_states': 'POST_STATES', - 'pre_states': 'PRE_STATES', - 'qualifier': 'QUALIFIER', - 'repeated': 'REPEATED', - 'required': 'REQUIRED', - 'role': 'ROLE', - 'state': 'STATE', - 'string': 'STRING', - 'transition': 'TRANSITION', -} - -literals = ':;{}()[]=,.&' - -tokens = [ - 'ARROW', # -> - 'BOOLEAN', - 'NAME', - 'NULL', - 'NUMBER', - 'STRING_LITERAL', -] + list(reserved.values()) - - -def _GetLexer(filename): - """Returns a lexer for STL.""" - - t_ARROW = r'->' - - def t_BOOLEAN(t): - r'(true|false)' - t.value = (t.value == 'true') - return t - - def t_NULL(t): - r'null' - t.value = None - return t - - def t_NAME(t): - r'[a-zA-Z_]\w*' - t.type = reserved.get(t.value, 'NAME') # reserved? - return t - - def t_NUMBER(t): - r'-?\d+' - t.value = int(t.value) - return t - - def t_STRING_LITERAL(t): - r'"([^\\"]|\\"|\\\\)*"' - t.value = t.value[1:-1].replace('\\"', '"').replace('\\\\', '\\') - return t - - def t_COMMENT(t): - r'//.*' - del t # unused argument - - # Define a rule so we can track line numbers - def t_newline(t): - r'\n+' - t.lexer.lineno += len(t.value) - - # A string containing ignored characters (spaces and tabs) - t_ignore = ' \t' - - # Error handling rule - def t_error(t): - logging.error('[%s:%d] Illegal character: %s', filename, t.lexer.lineno, - t.value[0]) - t.lexer.skip(1) - - return ply.lex.lex() - - -def _DebugLexer(lexer, data): - lexer.input(data) - tok = lexer.token() - while tok: - logging.debug(tok) - tok = lexer.token() - - -################################################### -# Parser - - -def _GetParser(filename, global_env): - """Return a parser for STL.""" - local_env = {'_curr_module': None} - - def p_module(p): + +class StlParser(object): + """A parser for Sprockets STL files.""" + + tokens = stl.lexer.StlLexer.tokens + + def __init__(self, filename=None, global_env=None): + """Creates an StlParser with an StlLexer. + + Args: + filename: The string to use in debug output. The parser does not attempt + to read from |filename|. To pass data to parse, use StlParser.parse + global_env: The global environment dict to be updated during parsing. + """ + self.filename = filename + self.global_env = global_env + self.local_env = {'_curr_module': None} + self.lexer = stl.lexer.StlLexer(self.filename) + self.parser = ply.yacc.yacc(module=self) + + def parse(self, data): + """Parses the |data| string and returns an updated global_env.""" + self.parser.parse(data, lexer=self.lexer.lexer) + return self.global_env + + + def p_module(self, p): """module : module_def defs""" del p # unused argument - def p_module_def(p): + def p_module_def(self, p): """module_def : MODULE NAME ';' """ - if p[2] in global_env['modules']: - local_env['_curr_module'] = global_env['modules'][p[2]] + if p[2] in self.global_env['modules']: + self.local_env['_curr_module'] = self.global_env['modules'][p[2]] else: - local_env['_curr_module'] = stl.module.Module(p[2]) - global_env['modules'][p[2]] = local_env['_curr_module'] + self.local_env['_curr_module'] = stl.module.Module(p[2]) + self.global_env['modules'][p[2]] = self.local_env['_curr_module'] - def p_defs(p): + def p_defs(self, p): """defs : defs def | def""" del p # unused argument - def p_def(p): + def p_def(self, p): """def : const_def | role_def | state_def @@ -173,31 +94,31 @@ def p_def(p): | transition_def""" del p # unused argument - def p_const_def(p): + def p_const_def(self, p): """const_def : CONST type NAME ';' | CONST type NAME '=' value ';' """ - if local_env['_curr_module'].HasDefinition(p[3]): - logging.error('[%s:%d] Duplicated const: %s', filename, p.lineno(3), p[3]) + if self.local_env['_curr_module'].HasDefinition(p[3]): + logging.error('[%s:%d] Duplicated const: %s', self.filename, p.lineno(3), p[3]) return # TODO(byungchul): Type checking if len(p) == 5: - local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2]) + self.local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2]) else: - local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2], p[5]) + self.local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2], p[5]) - def p_role_def(p): + def p_role_def(self, p): """role_def : ROLE NAME '{' '}' | ROLE NAME '{' role_fields '}' """ - if local_env['_curr_module'].HasDefinition(p[2]): - logging.error('[%s:%d] Duplicated role: %s', filename, p.lineno(2), p[2]) + if self.local_env['_curr_module'].HasDefinition(p[2]): + logging.error('[%s:%d] Duplicated role: %s', self.filename, p.lineno(2), p[2]) return role = stl.base.Role(p[2]) if len(p) >= 6: for f in p[4]: role.fields[f.name] = f - local_env['_curr_module'].roles[role.name] = role + self.local_env['_curr_module'].roles[role.name] = role - def p_role_fields(p): + def p_role_fields(self, p): """role_fields : role_fields role_field | role_field""" if len(p) == 2: # first def @@ -206,29 +127,29 @@ def p_role_fields(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[2].name: - logging.error('[%s:%d] Duplicated field: %s', filename, + logging.error('[%s:%d] Duplicated field: %s', self.filename, p.lineno(2), p[2]) return p[1].append(p[2]) p[0] = p[1] - def p_role_field(p): + def p_role_field(self, p): """role_field : type NAME ';' """ p[0] = stl.base.Field(p[2], p[1]) p.set_lineno(0, p.lineno(2)) - def p_state_def(p): + def p_state_def(self, p): """state_def : STATE NAME params '{' names '}' | STATE NAME params '{' names ',' '}' """ - if local_env['_curr_module'].HasDefinition(p[2]): - logging.error('[%s:%d] Duplicated state: %s', filename, p.lineno(2), p[2]) + if self.local_env['_curr_module'].HasDefinition(p[2]): + logging.error('[%s:%d] Duplicated state: %s', self.filename, p.lineno(2), p[2]) return state_ = stl.state.State(p[2]) state_.params = p[3] state_.values = p[5] - local_env['_curr_module'].states[state_.name] = state_ + self.local_env['_curr_module'].states[state_.name] = state_ - def p_names(p): + def p_names(self, p): """names : names ',' NAME | NAME""" if len(p) == 2: # first state value @@ -237,16 +158,16 @@ def p_names(p): assert isinstance(p[1], list) for n in p[1]: if n == p[3]: - logging.error('[%s:%d] Duplicated state value: %s', filename, + logging.error('[%s:%d] Duplicated state value: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_message_def(p): + def p_message_def(self, p): """message_def : message_or_array NAME '{' encode_decl message_body_or_external '}'""" # pylint: disable=line-too-long - if local_env['_curr_module'].HasDefinition(p[2]): - logging.error('[%s:%d] Duplicated message: %s', filename, + if self.local_env['_curr_module'].HasDefinition(p[2]): + logging.error('[%s:%d] Duplicated message: %s', self.filename, p.lineno(2), p[2]) return encode_name = p[4] @@ -258,20 +179,20 @@ def p_message_def(p): msg = stl.message.MessageFromExternal(p[2], encode_name, p[1], p[5]) except Exception as e: logging.exception('Could not import message: %s', p[5]) - global_env['error'] = True + self.global_env['error'] = True raise e - local_env['_curr_module'].messages[msg.name] = msg + self.local_env['_curr_module'].messages[msg.name] = msg - def p_message_or_array(p): + def p_message_or_array(self, p): """message_or_array : MESSAGE | MESSAGE '[' ']' """ p[0] = (len(p) == 4) # True if it's a message array. - def p_encode_decl(p): + def p_encode_decl(self, p): """encode_decl : ENCODE STRING_LITERAL ';' """ p[0] = p[2] - def p_message_body_or_external(p): + def p_message_body_or_external(self, p): """message_body_or_external : message_body | EXTERNAL STRING_LITERAL ';' """ if len(p) == 2: # message_body @@ -279,7 +200,7 @@ def p_message_body_or_external(p): else: # EXTERNAL STRING_LITERAL ';' p[0] = p[2] - def p_message_body(p): + def p_message_body(self, p): """message_body : message_body message_field | message_body sub_message | message_field @@ -293,18 +214,18 @@ def p_message_body(p): if isinstance(p[index], stl.base.Field): for f in p[0][0]: if f.name == p[index].name: - logging.error('[%s:%d] Duplicated field: %s', filename, + logging.error('[%s:%d] Duplicated field: %s', self.filename, p.lineno(index), p[index].name) return p[0][0].append(p[index]) else: if p[index].name in p[0][1]: - logging.error('[%s:%d] Duplicated message: %s', filename, + logging.error('[%s:%d] Duplicated message: %s', self.filename, p.lineno(index), p[index].name) return p[0][1][p[index].name] = p[index] - def p_message_field(p): + def p_message_field(self, p): """message_field : field_rule type NAME ';' | field_rule type NAME ':' field_property_list ';' """ p[0] = stl.base.Field(p[3], p[2], p[1] == 'optional', p[1] == 'repeated') @@ -312,13 +233,13 @@ def p_message_field(p): p[0].encoding_props = p[5] p.set_lineno(0, p.lineno(3)) - def p_field_rule(p): + def p_field_rule(self, p): """field_rule : REQUIRED | OPTIONAL | REPEATED""" p[0] = p[1] - def p_field_property_list(p): + def p_field_property_list(self, p): """field_property_list : field_property_list ',' field_property | field_property """ if len(p) == 2: # first key-value pair @@ -328,34 +249,34 @@ def p_field_property_list(p): assert isinstance(p[1], dict) key, val = p[3] if key in p[1]: - logging.error('[%s:%d] Duplicated key: %s', filename, p.lineno(3), key) + logging.error('[%s:%d] Duplicated key: %s', self.filename, p.lineno(3), key) return p[1][key] = val p[0] = p[1] - def p_field_property(p): + def p_field_property(self, p): """field_property : STRING_LITERAL '=' constant """ p[0] = (p[1], p[3].value) - def p_sub_message(p): + def p_sub_message(self, p): """sub_message : MESSAGE NAME '{' message_body '}' """ msg = stl.message.Message(p[2], None, False) msg.fields, msg.messages = p[4] p[0] = msg p.set_lineno(0, p.lineno(2)) - def p_qualifier_def(p): + def p_qualifier_def(self, p): """qualifier_def : QUALIFIER type NAME params '=' EXTERNAL STRING_LITERAL ';'""" # pylint: disable=line-too-long try: qual = stl.qualifier.QualifierFromExternal(p[3], p[2], p[7]) except Exception as e: logging.exception('Could not import qualifier: %s', p[7]) - global_env['error'] = True + self.global_env['error'] = True raise e qual.params = p[4] - local_env['_curr_module'].qualifiers[qual.name] = qual + self.local_env['_curr_module'].qualifiers[qual.name] = qual - def p_event_def(p): + def p_event_def(self, p): """event_def : EVENT NAME params ';' | EVENT NAME params '=' EXTERNAL STRING_LITERAL ';' | EVENT NAME params '=' NAME param_values ';' """ @@ -365,7 +286,7 @@ def p_event_def(p): evt = stl.event.EventFromExternal(p[2], p[6]) except Exception as e: logging.exception('Could not import event: %s', p[6]) - global_env['error'] = True + self.global_env['error'] = True raise e elif len(p) == 8 and isinstance(p[6], list): # NAME params = NAME param_values ; @@ -378,9 +299,9 @@ def p_event_def(p): evt = stl.event.Event(p[2]) evt.params = p[3] - local_env['_curr_module'].events[evt.name] = evt + self.local_env['_curr_module'].events[evt.name] = evt - def p_transition_def(p): + def p_transition_def(self, p): """transition_def : TRANSITION NAME params '{' transition_body '}' | TRANSITION NAME params '=' NAME param_values ';' """ trans = stl.state.Transition(p[2]) @@ -391,13 +312,13 @@ def p_transition_def(p): else: (trans.local_vars, trans.pre_states, trans.events, trans.post_states, trans.error_states) = p[5] - local_env['_curr_module'].transitions[trans.name] = trans + self.local_env['_curr_module'].transitions[trans.name] = trans - def p_transition_body(p): + def p_transition_body(self, p): """transition_body : local_vars pre_states events post_states error_states""" # pylint: disable=line-too-long p[0] = (p[1], p[2], p[3], p[4], p[5]) - def p_local_vars(p): + def p_local_vars(self, p): """local_vars : local_vars local_var | empty""" if len(p) == 2: # empty @@ -406,20 +327,20 @@ def p_local_vars(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[2].name: - logging.error('[%s:%d] Duplicated local var: %s', filename, + logging.error('[%s:%d] Duplicated local var: %s', self.filename, p.lineno(2), p[2].name) p[1].append(p[2]) p[0] = p[1] - def p_local_var(p): + def p_local_var(self, p): """local_var : type NAME ';' """ p[0] = stl.base.LocalVar(p[2], p[1]) - def p_pre_states(p): + def p_pre_states(self, p): """pre_states : PRE_STATES '=' '[' pre_state_values ']' """ p[0] = p[4] - def p_post_states(p): + def p_post_states(self, p): """post_states : POST_STATES '=' '[' state_values ']' | POST_STATES '=' '[' ']' """ if len(p) == 6: @@ -427,7 +348,7 @@ def p_post_states(p): else: p[0] = [] - def p_error_states(p): + def p_error_states(self, p): """error_states : ERROR_STATES '=' '[' state_values ']' | ERROR_STATES '=' '[' ']' | empty""" @@ -436,7 +357,7 @@ def p_error_states(p): else: p[0] = [] - def p_pre_state_values(p): + def p_pre_state_values(self, p): """pre_state_values : pre_state_values ',' pre_state_value | pre_state_value""" if len(p) == 2: # first state_value @@ -445,20 +366,20 @@ def p_pre_state_values(p): assert isinstance(p[1], list) for s in p[1]: if str(s) == str(p[3]): - logging.error('[%s:%d] Duplicated state: %s', filename, + logging.error('[%s:%d] Duplicated state: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_pre_state_value(p): + def p_pre_state_value(self, p): """pre_state_value : NAME param_values '.' pre_state_value_options""" assert isinstance(p[4], list) p[0] = [stl.state.StateValueInTransition(p[1], s) for s in p[4]] for s in p[0]: s.param_values = p[2] - def p_pre_state_value_options(p): + def p_pre_state_value_options(self, p): """pre_state_value_options : NAME | '{' names '}' """ if len(p) == 2: # Only a single pre_state value @@ -467,7 +388,7 @@ def p_pre_state_value_options(p): assert isinstance(p[2], list) p[0] = p[2] - def p_state_values(p): + def p_state_values(self, p): """state_values : state_values ',' state_value | state_value""" if len(p) == 2: # first state_value @@ -476,22 +397,22 @@ def p_state_values(p): assert isinstance(p[1], list) for s in p[1]: if str(s) == str(p[3]): - logging.error('[%s:%d] Duplicated state: %s', filename, + logging.error('[%s:%d] Duplicated state: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_state_value(p): + def p_state_value(self, p): """state_value : NAME param_values '.' NAME""" p[0] = stl.state.StateValueInTransition(p[1], p[4]) p[0].param_values = p[2] - def p_events(p): + def p_events(self, p): """events : EVENTS '{' role_events '}' """ p[0] = p[3] - def p_role_events(p): + def p_role_events(self, p): """role_events : role_events role_event | role_event""" if len(p) == 2: # first role_event @@ -501,12 +422,12 @@ def p_role_events(p): p[1].append(p[2]) p[0] = p[1] - def p_role_event(p): + def p_role_event(self, p): """role_event : NAME ARROW NAME param_values ARROW NAME ';' """ p[0] = stl.event.EventInTransition(p[3], p[1], p[6]) p[0].param_values = p[4] - def p_params(p): + def p_params(self, p): """params : empty | '(' ')' | '(' params_without_paren ')' """ @@ -515,7 +436,7 @@ def p_params(p): else: p[0] = [] - def p_params_without_paren(p): + def p_params_without_paren(self, p): """params_without_paren : params_without_paren ',' param | param""" if len(p) == 2: # first param @@ -524,13 +445,13 @@ def p_params_without_paren(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[3].name: - logging.error('[%s:%d] Duplicated param: %s', filename, + logging.error('[%s:%d] Duplicated param: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_param(p): + def p_param(self, p): """param : type_or_role NAME | type_or_role '&' NAME""" if len(p) == 3: @@ -538,7 +459,7 @@ def p_param(p): else: p[0] = stl.base.Param(p[3], p[1]) # TODO(byungchul): Handle out param (&) - def p_param_values(p): + def p_param_values(self, p): """param_values : empty | '(' ')' | '(' param_values_without_paren ')' """ @@ -547,7 +468,7 @@ def p_param_values(p): else: p[0] = [] - def p_param_values_without_paren(p): + def p_param_values_without_paren(self, p): """param_values_without_paren : param_values_without_paren ',' param_value | param_value""" if len(p) == 2: # first param_value @@ -557,26 +478,26 @@ def p_param_values_without_paren(p): p[1].append(p[3]) p[0] = p[1] - def p_param_value(p): + def p_param_value(self, p): """param_value : value | message_value | message_array""" p[0] = p[1] - def p_value(p): + def p_value(self, p): """value : constant | reference_maybe_with_ampersand""" p[0] = p[1] - def p_constant(p): + def p_constant(self, p): """constant : BOOLEAN | NULL | NUMBER | STRING_LITERAL""" p[0] = stl.base.Value(p[1]) - def p_reference_maybe_with_ampersand(p): + def p_reference_maybe_with_ampersand(self, p): """reference_maybe_with_ampersand : reference | '&' reference""" if len(p) == 2: @@ -584,7 +505,7 @@ def p_reference_maybe_with_ampersand(p): else: p[0] = stl.base.Value('&' + p[2]) # FuncSet - def p_reference(p): + def p_reference(self, p): """reference : NAME | reference '.' NAME""" # TODO(byungchul): Support other module's names. @@ -594,18 +515,18 @@ def p_reference(p): else: p[0] = p[1] + '.' + p[3] - def p_message_array(p): + def p_message_array(self, p): """message_array : NAME array """ assert isinstance(p[2].value, list) p[0] = stl.base.Expand(p[1]) p[0].values = [p[2]] - def p_message_value(p): + def p_message_value(self, p): """message_value : NAME '{' field_values '}' """ p[0] = stl.base.Expand(p[1]) p[0].values = p[3] - def p_field_values(p): + def p_field_values(self, p): """field_values : field_values field_value | empty""" if len(p) == 2: # empty @@ -614,19 +535,19 @@ def p_field_values(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[2].name: - logging.error('[%s:%d] Cannot set field again: %s', filename, + logging.error('[%s:%d] Cannot set field again: %s', self.filename, p.lineno(2), p[2].name) p[1].append(p[2]) p[0] = p[1] - def p_field_value(p): + def p_field_value(self, p): """field_value : NAME '=' rvalue""" assert isinstance(p[3], stl.base.Value) p[0] = p[3] p[0].name = p[1] p.set_lineno(0, p.lineno(1)) - def p_rvalue(p): + def p_rvalue(self, p): """rvalue : value ';' | qualifier_value ';' | array ';' @@ -637,7 +558,7 @@ def p_rvalue(p): | message_array_value""" p[0] = p[1] - def p_array(p): + def p_array(self, p): """array : '[' ']' | '[' array_elements ']' | '[' array_elements ',' ']' """ @@ -646,7 +567,7 @@ def p_array(p): else: p[0] = stl.base.Value(p[2]) - def p_array_elements(p): + def p_array_elements(self, p): """array_elements : array_elements ',' array_element | array_element""" if len(p) == 2: # first element @@ -656,58 +577,56 @@ def p_array_elements(p): p[1].append(p[3]) p[0] = p[1] - def p_array_element(p): + def p_array_element(self, p): """array_element : value | array | struct""" p[0] = p[1] - def p_struct(p): + def p_struct(self, p): """struct : '{' field_values '}' """ p[0] = stl.base.Value(p[2]) - def p_message_array_value(p): + def p_message_array_value(self, p): """message_array_value : message_array | message_value""" p[0] = stl.base.Value(p[1]) - def p_qualifier_value(p): + def p_qualifier_value(self, p): """qualifier_value : NAME param_values ARROW reference | NAME param_values""" - qual = local_env['_curr_module'].qualifiers[p[1]] + qual = self.local_env['_curr_module'].qualifiers[p[1]] assert qual if len(p) == 5: p[0] = stl.base.QualifierValue(qual, p[2], stl.base.Value('&' + p[4])) else: p[0] = stl.base.QualifierValue(qual, p[2]) - def p_type(p): + def p_type(self, p): """type : BOOL | INT | NAME | STRING""" p[0] = p[1] - def p_type_or_role(p): + def p_type_or_role(self, p): """type_or_role : type | ROLE""" p[0] = p[1] - def p_empty(p): + def p_empty(self, p): """empty : """ p[0] = None - def p_error(p): - global_env['error'] = True + def p_error(self, p): + self.global_env['error'] = True if p is None: raise StlSyntaxError( '[{}] Syntax error: ' - 'Reached end of file unexpectantly.'.format(filename)) - else: - raise StlSyntaxError('[{}:{}] Syntax error at: {}'.format( - filename, p.lexer.lineno, p.value)) + 'Reached end of file unexpectantly.'.format(self.filename)) - return ply.yacc.yacc() + raise StlSyntaxError('[{}:{}] Syntax error at: {}'.format( + self.filename, p.lexer.lineno, p.value)) def Parse(filename, global_env): @@ -718,23 +637,16 @@ def Parse(filename, global_env): global_env: Dictionary to store global STL state. It has one field: global_env['modules']: Dictionary of stl.module.Module by name. """ - data = open(filename).read() - lexer = _GetLexer(filename) - parser = _GetParser(filename, global_env) - - parser.parse(data, lexer=lexer) - - -def _DebugParser(stl_filename): - global_env = {'modules': {}} - Parse(stl_filename, global_env) - return global_env + parser = StlParser(filename=filename, global_env=global_env) + with open(filename) as data: + return parser.parse(data.read()) def main(): logging.basicConfig(level=logging.DEBUG) - filename = sys.argv[1] if len(sys.argv) == 2 else 'model2/cast_v2.stl' - pprint.pprint(_DebugParser(filename), indent=2) + filename = sys.argv[1] + global_env = {'modules': {}} + pprint.pprint(Parse(filename, global_env), indent=2) if __name__ == '__main__': diff --git a/stl/parser_test.py b/stl/parser_test.py index 3bd264a..14e2cef 100644 --- a/stl/parser_test.py +++ b/stl/parser_test.py @@ -35,8 +35,8 @@ class StlParserTest(unittest.TestCase): def setUp(self): self.actual_module = stl.module.Module('foo') self.global_env = {'modules': {'foo': self.actual_module}} - self.lexer = stl.parser._GetLexer(self.TEST_FILENAME) - self.parser = stl.parser._GetParser(self.TEST_FILENAME, self.global_env) + self.parser = stl.parser.StlParser(self.TEST_FILENAME, self.global_env) + self.lexer = self.parser.lexer.lexer self.expected_module = stl.module.Module('foo') def tearDown(self): @@ -49,7 +49,7 @@ def Parse(self, text): Args: text: The text to parse. """ - self.parser.parse(text, lexer=self.lexer) + self.parser.parse(text) def testLineNumber(self): input_text = ('module foo;\n'