diff --git a/stl/lexer.py b/stl/lexer.py new file mode 100644 index 0000000..cd04c35 --- /dev/null +++ b/stl/lexer.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lexing (tokenizing) a state transition (STL) spec.""" + +# pylint: disable=g-doc-args +# pylint: disable=g-docstring-missing-newline +# pylint: disable=g-docstring-quotes +# pylint: disable=g-no-space-after-docstring-summary +# pylint: disable=g-short-docstring-punctuation +# pylint: disable=g-short-docstring-space +# pylint: disable=invalid-name +# pylint: disable=unused-variable + +import logging +import ply.lex # pylint: disable=g-bad-import-order + + +class StlSyntaxError(SyntaxError): + """Error for incorrect STL syntax.""" + + +class StlLexer(object): + + def __init__(self, filename=None, **kwargs): + """Create a Lex lexer. + + To pass this into a Ply Yacc parser, pass it in using the .lexer propert + of an StlLexer instance: + my_lexer = StlLexer() + my_parser = ply.yacc.parser(lexer=my_lexer.lexer) + + Args: + filename: The filename string to use in any error messaging. + kwargs: Forwarded to ply.lex.lex. + """ + self.filename = filename + self.lexer = ply.lex.lex(module=self, **kwargs) + + RESERVED = { + 'bool': 'BOOL', + 'const': 'CONST', + 'encode': 'ENCODE', + 'error_states': 'ERROR_STATES', + 'event': 'EVENT', + 'events': 'EVENTS', + 'external': 'EXTERNAL', + 'int': 'INT', + 'message': 'MESSAGE', + 'module': 'MODULE', + 'optional': 'OPTIONAL', + 'post_states': 'POST_STATES', + 'pre_states': 'PRE_STATES', + 'qualifier': 'QUALIFIER', + 'repeated': 'REPEATED', + 'required': 'REQUIRED', + 'role': 'ROLE', + 'state': 'STATE', + 'string': 'STRING', + 'transition': 'TRANSITION', + } + + # |literals| is a special field for ply.lex. Each of these + # characters is interpreted as a separate token + literals = ':;{}()[]=,.&' + + # |tokens| is a special field for ply.lex. This must contain a list + # of all possible token types. + tokens = [ + 'ARROW', # -> + 'BOOLEAN', + 'NAME', + 'NULL', + 'NUMBER', + 'STRING_LITERAL', + ] + list(RESERVED.values()) + + # A string containing ignored characters (spaces and tabs) + t_ignore = ' \t' + + def t_ARROW(self, t): + r'->' + return t + + def t_BOOLEAN(self, t): + r'(true|false)' + t.value = (t.value == 'true') + return t + + def t_NULL(self, t): + r'null' + t.value = None + return t + + def t_NAME(self, t): + r'[a-zA-Z_]\w*' + t.type = self.RESERVED.get(t.value, 'NAME') + return t + + def t_NUMBER(self, t): + r'-?\d+' + t.value = int(t.value) + return t + + def t_STRING_LITERAL(self, t): + r'"([^\\"]|\\"|\\\\)*"' + t.value = t.value[1:-1].replace('\\"', '"').replace('\\\\', '\\') + return t + + def t_COMMENT(self, t): + r'//.*' + del t # unused argument + + # Define a rule so we can track line numbers. + def t_newline(self, t): + r'\n+' + t.lexer.lineno += len(t.value) + + # Error handling rule. + def t_error(self, t): + logging.error('[%s:%d] Illegal character: %s', + self.filename, + t.lexer.lineno, + t.value[0]) + t.lexer.skip(1) + + def debug(data): + """Print out all the tokens in |data|.""" + for token in self.lexer.tokens(): + print token diff --git a/stl/parser.py b/stl/parser.py index c94234f..a055a65 100644 --- a/stl/parser.py +++ b/stl/parser.py @@ -24,13 +24,13 @@ # pylint: disable=unused-variable import logging -import ply.lex # pylint: disable=g-bad-import-order import ply.yacc # pylint: disable=g-bad-import-order import pprint import sys import stl.base import stl.event +import stl.lexer import stl.message import stl.module import stl.qualifier @@ -41,129 +41,50 @@ class StlSyntaxError(SyntaxError): """Error for incorrect STL syntax.""" -################################################### -# Lexer - -reserved = { - 'bool': 'BOOL', - 'const': 'CONST', - 'encode': 'ENCODE', - 'error_states': 'ERROR_STATES', - 'event': 'EVENT', - 'events': 'EVENTS', - 'external': 'EXTERNAL', - 'int': 'INT', - 'message': 'MESSAGE', - 'module': 'MODULE', - 'optional': 'OPTIONAL', - 'post_states': 'POST_STATES', - 'pre_states': 'PRE_STATES', - 'qualifier': 'QUALIFIER', - 'repeated': 'REPEATED', - 'required': 'REQUIRED', - 'role': 'ROLE', - 'state': 'STATE', - 'string': 'STRING', - 'transition': 'TRANSITION', -} - -literals = ':;{}()[]=,.&' - -tokens = [ - 'ARROW', # -> - 'BOOLEAN', - 'NAME', - 'NULL', - 'NUMBER', - 'STRING_LITERAL', -] + list(reserved.values()) - - -def _GetLexer(filename): - """Returns a lexer for STL.""" - - t_ARROW = r'->' - - def t_BOOLEAN(t): - r'(true|false)' - t.value = (t.value == 'true') - return t - - def t_NULL(t): - r'null' - t.value = None - return t - - def t_NAME(t): - r'[a-zA-Z_]\w*' - t.type = reserved.get(t.value, 'NAME') # reserved? - return t - - def t_NUMBER(t): - r'-?\d+' - t.value = int(t.value) - return t - - def t_STRING_LITERAL(t): - r'"([^\\"]|\\"|\\\\)*"' - t.value = t.value[1:-1].replace('\\"', '"').replace('\\\\', '\\') - return t - - def t_COMMENT(t): - r'//.*' - del t # unused argument - - # Define a rule so we can track line numbers - def t_newline(t): - r'\n+' - t.lexer.lineno += len(t.value) - - # A string containing ignored characters (spaces and tabs) - t_ignore = ' \t' - - # Error handling rule - def t_error(t): - logging.error('[%s:%d] Illegal character: %s', filename, t.lexer.lineno, - t.value[0]) - t.lexer.skip(1) - - return ply.lex.lex() - - -def _DebugLexer(lexer, data): - lexer.input(data) - tok = lexer.token() - while tok: - logging.debug(tok) - tok = lexer.token() - - -################################################### -# Parser - - -def _GetParser(filename, global_env): - """Return a parser for STL.""" - local_env = {'_curr_module': None} - - def p_module(p): + +class StlParser(object): + """A parser for Sprockets STL files.""" + + tokens = stl.lexer.StlLexer.tokens + + def __init__(self, filename=None, global_env=None): + """Creates an StlParser with an StlLexer. + + Args: + filename: The string to use in debug output. The parser does not attempt + to read from |filename|. To pass data to parse, use StlParser.parse + global_env: The global environment dict to be updated during parsing. + """ + self.filename = filename + self.global_env = global_env + self.local_env = {'_curr_module': None} + self.lexer = stl.lexer.StlLexer(self.filename) + self.parser = ply.yacc.yacc(module=self) + + def parse(self, data): + """Parses the |data| string and returns an updated global_env.""" + self.parser.parse(data, lexer=self.lexer.lexer) + return self.global_env + + + def p_module(self, p): """module : module_def defs""" del p # unused argument - def p_module_def(p): + def p_module_def(self, p): """module_def : MODULE NAME ';' """ - if p[2] in global_env['modules']: - local_env['_curr_module'] = global_env['modules'][p[2]] + if p[2] in self.global_env['modules']: + self.local_env['_curr_module'] = self.global_env['modules'][p[2]] else: - local_env['_curr_module'] = stl.module.Module(p[2]) - global_env['modules'][p[2]] = local_env['_curr_module'] + self.local_env['_curr_module'] = stl.module.Module(p[2]) + self.global_env['modules'][p[2]] = self.local_env['_curr_module'] - def p_defs(p): + def p_defs(self, p): """defs : defs def | def""" del p # unused argument - def p_def(p): + def p_def(self, p): """def : const_def | role_def | state_def @@ -173,31 +94,31 @@ def p_def(p): | transition_def""" del p # unused argument - def p_const_def(p): + def p_const_def(self, p): """const_def : CONST type NAME ';' | CONST type NAME '=' value ';' """ - if local_env['_curr_module'].HasDefinition(p[3]): - logging.error('[%s:%d] Duplicated const: %s', filename, p.lineno(3), p[3]) + if self.local_env['_curr_module'].HasDefinition(p[3]): + logging.error('[%s:%d] Duplicated const: %s', self.filename, p.lineno(3), p[3]) return # TODO(byungchul): Type checking if len(p) == 5: - local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2]) + self.local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2]) else: - local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2], p[5]) + self.local_env['_curr_module'].consts[p[3]] = stl.base.Const(p[3], p[2], p[5]) - def p_role_def(p): + def p_role_def(self, p): """role_def : ROLE NAME '{' '}' | ROLE NAME '{' role_fields '}' """ - if local_env['_curr_module'].HasDefinition(p[2]): - logging.error('[%s:%d] Duplicated role: %s', filename, p.lineno(2), p[2]) + if self.local_env['_curr_module'].HasDefinition(p[2]): + logging.error('[%s:%d] Duplicated role: %s', self.filename, p.lineno(2), p[2]) return role = stl.base.Role(p[2]) if len(p) >= 6: for f in p[4]: role.fields[f.name] = f - local_env['_curr_module'].roles[role.name] = role + self.local_env['_curr_module'].roles[role.name] = role - def p_role_fields(p): + def p_role_fields(self, p): """role_fields : role_fields role_field | role_field""" if len(p) == 2: # first def @@ -206,29 +127,29 @@ def p_role_fields(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[2].name: - logging.error('[%s:%d] Duplicated field: %s', filename, + logging.error('[%s:%d] Duplicated field: %s', self.filename, p.lineno(2), p[2]) return p[1].append(p[2]) p[0] = p[1] - def p_role_field(p): + def p_role_field(self, p): """role_field : type NAME ';' """ p[0] = stl.base.Field(p[2], p[1]) p.set_lineno(0, p.lineno(2)) - def p_state_def(p): + def p_state_def(self, p): """state_def : STATE NAME params '{' names '}' | STATE NAME params '{' names ',' '}' """ - if local_env['_curr_module'].HasDefinition(p[2]): - logging.error('[%s:%d] Duplicated state: %s', filename, p.lineno(2), p[2]) + if self.local_env['_curr_module'].HasDefinition(p[2]): + logging.error('[%s:%d] Duplicated state: %s', self.filename, p.lineno(2), p[2]) return state_ = stl.state.State(p[2]) state_.params = p[3] state_.values = p[5] - local_env['_curr_module'].states[state_.name] = state_ + self.local_env['_curr_module'].states[state_.name] = state_ - def p_names(p): + def p_names(self, p): """names : names ',' NAME | NAME""" if len(p) == 2: # first state value @@ -237,16 +158,16 @@ def p_names(p): assert isinstance(p[1], list) for n in p[1]: if n == p[3]: - logging.error('[%s:%d] Duplicated state value: %s', filename, + logging.error('[%s:%d] Duplicated state value: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_message_def(p): + def p_message_def(self, p): """message_def : message_or_array NAME '{' encode_decl message_body_or_external '}'""" # pylint: disable=line-too-long - if local_env['_curr_module'].HasDefinition(p[2]): - logging.error('[%s:%d] Duplicated message: %s', filename, + if self.local_env['_curr_module'].HasDefinition(p[2]): + logging.error('[%s:%d] Duplicated message: %s', self.filename, p.lineno(2), p[2]) return encode_name = p[4] @@ -258,20 +179,20 @@ def p_message_def(p): msg = stl.message.MessageFromExternal(p[2], encode_name, p[1], p[5]) except Exception as e: logging.exception('Could not import message: %s', p[5]) - global_env['error'] = True + self.global_env['error'] = True raise e - local_env['_curr_module'].messages[msg.name] = msg + self.local_env['_curr_module'].messages[msg.name] = msg - def p_message_or_array(p): + def p_message_or_array(self, p): """message_or_array : MESSAGE | MESSAGE '[' ']' """ p[0] = (len(p) == 4) # True if it's a message array. - def p_encode_decl(p): + def p_encode_decl(self, p): """encode_decl : ENCODE STRING_LITERAL ';' """ p[0] = p[2] - def p_message_body_or_external(p): + def p_message_body_or_external(self, p): """message_body_or_external : message_body | EXTERNAL STRING_LITERAL ';' """ if len(p) == 2: # message_body @@ -279,7 +200,7 @@ def p_message_body_or_external(p): else: # EXTERNAL STRING_LITERAL ';' p[0] = p[2] - def p_message_body(p): + def p_message_body(self, p): """message_body : message_body message_field | message_body sub_message | message_field @@ -293,18 +214,18 @@ def p_message_body(p): if isinstance(p[index], stl.base.Field): for f in p[0][0]: if f.name == p[index].name: - logging.error('[%s:%d] Duplicated field: %s', filename, + logging.error('[%s:%d] Duplicated field: %s', self.filename, p.lineno(index), p[index].name) return p[0][0].append(p[index]) else: if p[index].name in p[0][1]: - logging.error('[%s:%d] Duplicated message: %s', filename, + logging.error('[%s:%d] Duplicated message: %s', self.filename, p.lineno(index), p[index].name) return p[0][1][p[index].name] = p[index] - def p_message_field(p): + def p_message_field(self, p): """message_field : field_rule type NAME ';' | field_rule type NAME ':' field_property_list ';' """ p[0] = stl.base.Field(p[3], p[2], p[1] == 'optional', p[1] == 'repeated') @@ -312,13 +233,13 @@ def p_message_field(p): p[0].encoding_props = p[5] p.set_lineno(0, p.lineno(3)) - def p_field_rule(p): + def p_field_rule(self, p): """field_rule : REQUIRED | OPTIONAL | REPEATED""" p[0] = p[1] - def p_field_property_list(p): + def p_field_property_list(self, p): """field_property_list : field_property_list ',' field_property | field_property """ if len(p) == 2: # first key-value pair @@ -328,34 +249,34 @@ def p_field_property_list(p): assert isinstance(p[1], dict) key, val = p[3] if key in p[1]: - logging.error('[%s:%d] Duplicated key: %s', filename, p.lineno(3), key) + logging.error('[%s:%d] Duplicated key: %s', self.filename, p.lineno(3), key) return p[1][key] = val p[0] = p[1] - def p_field_property(p): + def p_field_property(self, p): """field_property : STRING_LITERAL '=' constant """ p[0] = (p[1], p[3].value) - def p_sub_message(p): + def p_sub_message(self, p): """sub_message : MESSAGE NAME '{' message_body '}' """ msg = stl.message.Message(p[2], None, False) msg.fields, msg.messages = p[4] p[0] = msg p.set_lineno(0, p.lineno(2)) - def p_qualifier_def(p): + def p_qualifier_def(self, p): """qualifier_def : QUALIFIER type NAME params '=' EXTERNAL STRING_LITERAL ';'""" # pylint: disable=line-too-long try: qual = stl.qualifier.QualifierFromExternal(p[3], p[2], p[7]) except Exception as e: logging.exception('Could not import qualifier: %s', p[7]) - global_env['error'] = True + self.global_env['error'] = True raise e qual.params = p[4] - local_env['_curr_module'].qualifiers[qual.name] = qual + self.local_env['_curr_module'].qualifiers[qual.name] = qual - def p_event_def(p): + def p_event_def(self, p): """event_def : EVENT NAME params ';' | EVENT NAME params '=' EXTERNAL STRING_LITERAL ';' | EVENT NAME params '=' NAME param_values ';' """ @@ -365,7 +286,7 @@ def p_event_def(p): evt = stl.event.EventFromExternal(p[2], p[6]) except Exception as e: logging.exception('Could not import event: %s', p[6]) - global_env['error'] = True + self.global_env['error'] = True raise e elif len(p) == 8 and isinstance(p[6], list): # NAME params = NAME param_values ; @@ -378,9 +299,9 @@ def p_event_def(p): evt = stl.event.Event(p[2]) evt.params = p[3] - local_env['_curr_module'].events[evt.name] = evt + self.local_env['_curr_module'].events[evt.name] = evt - def p_transition_def(p): + def p_transition_def(self, p): """transition_def : TRANSITION NAME params '{' transition_body '}' | TRANSITION NAME params '=' NAME param_values ';' """ trans = stl.state.Transition(p[2]) @@ -391,13 +312,13 @@ def p_transition_def(p): else: (trans.local_vars, trans.pre_states, trans.events, trans.post_states, trans.error_states) = p[5] - local_env['_curr_module'].transitions[trans.name] = trans + self.local_env['_curr_module'].transitions[trans.name] = trans - def p_transition_body(p): + def p_transition_body(self, p): """transition_body : local_vars pre_states events post_states error_states""" # pylint: disable=line-too-long p[0] = (p[1], p[2], p[3], p[4], p[5]) - def p_local_vars(p): + def p_local_vars(self, p): """local_vars : local_vars local_var | empty""" if len(p) == 2: # empty @@ -406,20 +327,20 @@ def p_local_vars(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[2].name: - logging.error('[%s:%d] Duplicated local var: %s', filename, + logging.error('[%s:%d] Duplicated local var: %s', self.filename, p.lineno(2), p[2].name) p[1].append(p[2]) p[0] = p[1] - def p_local_var(p): + def p_local_var(self, p): """local_var : type NAME ';' """ p[0] = stl.base.LocalVar(p[2], p[1]) - def p_pre_states(p): + def p_pre_states(self, p): """pre_states : PRE_STATES '=' '[' pre_state_values ']' """ p[0] = p[4] - def p_post_states(p): + def p_post_states(self, p): """post_states : POST_STATES '=' '[' state_values ']' | POST_STATES '=' '[' ']' """ if len(p) == 6: @@ -427,7 +348,7 @@ def p_post_states(p): else: p[0] = [] - def p_error_states(p): + def p_error_states(self, p): """error_states : ERROR_STATES '=' '[' state_values ']' | ERROR_STATES '=' '[' ']' | empty""" @@ -436,7 +357,7 @@ def p_error_states(p): else: p[0] = [] - def p_pre_state_values(p): + def p_pre_state_values(self, p): """pre_state_values : pre_state_values ',' pre_state_value | pre_state_value""" if len(p) == 2: # first state_value @@ -445,20 +366,20 @@ def p_pre_state_values(p): assert isinstance(p[1], list) for s in p[1]: if str(s) == str(p[3]): - logging.error('[%s:%d] Duplicated state: %s', filename, + logging.error('[%s:%d] Duplicated state: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_pre_state_value(p): + def p_pre_state_value(self, p): """pre_state_value : NAME param_values '.' pre_state_value_options""" assert isinstance(p[4], list) p[0] = [stl.state.StateValueInTransition(p[1], s) for s in p[4]] for s in p[0]: s.param_values = p[2] - def p_pre_state_value_options(p): + def p_pre_state_value_options(self, p): """pre_state_value_options : NAME | '{' names '}' """ if len(p) == 2: # Only a single pre_state value @@ -467,7 +388,7 @@ def p_pre_state_value_options(p): assert isinstance(p[2], list) p[0] = p[2] - def p_state_values(p): + def p_state_values(self, p): """state_values : state_values ',' state_value | state_value""" if len(p) == 2: # first state_value @@ -476,22 +397,22 @@ def p_state_values(p): assert isinstance(p[1], list) for s in p[1]: if str(s) == str(p[3]): - logging.error('[%s:%d] Duplicated state: %s', filename, + logging.error('[%s:%d] Duplicated state: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_state_value(p): + def p_state_value(self, p): """state_value : NAME param_values '.' NAME""" p[0] = stl.state.StateValueInTransition(p[1], p[4]) p[0].param_values = p[2] - def p_events(p): + def p_events(self, p): """events : EVENTS '{' role_events '}' """ p[0] = p[3] - def p_role_events(p): + def p_role_events(self, p): """role_events : role_events role_event | role_event""" if len(p) == 2: # first role_event @@ -501,12 +422,12 @@ def p_role_events(p): p[1].append(p[2]) p[0] = p[1] - def p_role_event(p): + def p_role_event(self, p): """role_event : NAME ARROW NAME param_values ARROW NAME ';' """ p[0] = stl.event.EventInTransition(p[3], p[1], p[6]) p[0].param_values = p[4] - def p_params(p): + def p_params(self, p): """params : empty | '(' ')' | '(' params_without_paren ')' """ @@ -515,7 +436,7 @@ def p_params(p): else: p[0] = [] - def p_params_without_paren(p): + def p_params_without_paren(self, p): """params_without_paren : params_without_paren ',' param | param""" if len(p) == 2: # first param @@ -524,13 +445,13 @@ def p_params_without_paren(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[3].name: - logging.error('[%s:%d] Duplicated param: %s', filename, + logging.error('[%s:%d] Duplicated param: %s', self.filename, p.lineno(3), p[3]) return p[1].append(p[3]) p[0] = p[1] - def p_param(p): + def p_param(self, p): """param : type_or_role NAME | type_or_role '&' NAME""" if len(p) == 3: @@ -538,7 +459,7 @@ def p_param(p): else: p[0] = stl.base.Param(p[3], p[1]) # TODO(byungchul): Handle out param (&) - def p_param_values(p): + def p_param_values(self, p): """param_values : empty | '(' ')' | '(' param_values_without_paren ')' """ @@ -547,7 +468,7 @@ def p_param_values(p): else: p[0] = [] - def p_param_values_without_paren(p): + def p_param_values_without_paren(self, p): """param_values_without_paren : param_values_without_paren ',' param_value | param_value""" if len(p) == 2: # first param_value @@ -557,26 +478,26 @@ def p_param_values_without_paren(p): p[1].append(p[3]) p[0] = p[1] - def p_param_value(p): + def p_param_value(self, p): """param_value : value | message_value | message_array""" p[0] = p[1] - def p_value(p): + def p_value(self, p): """value : constant | reference_maybe_with_ampersand""" p[0] = p[1] - def p_constant(p): + def p_constant(self, p): """constant : BOOLEAN | NULL | NUMBER | STRING_LITERAL""" p[0] = stl.base.Value(p[1]) - def p_reference_maybe_with_ampersand(p): + def p_reference_maybe_with_ampersand(self, p): """reference_maybe_with_ampersand : reference | '&' reference""" if len(p) == 2: @@ -584,7 +505,7 @@ def p_reference_maybe_with_ampersand(p): else: p[0] = stl.base.Value('&' + p[2]) # FuncSet - def p_reference(p): + def p_reference(self, p): """reference : NAME | reference '.' NAME""" # TODO(byungchul): Support other module's names. @@ -594,18 +515,18 @@ def p_reference(p): else: p[0] = p[1] + '.' + p[3] - def p_message_array(p): + def p_message_array(self, p): """message_array : NAME array """ assert isinstance(p[2].value, list) p[0] = stl.base.Expand(p[1]) p[0].values = [p[2]] - def p_message_value(p): + def p_message_value(self, p): """message_value : NAME '{' field_values '}' """ p[0] = stl.base.Expand(p[1]) p[0].values = p[3] - def p_field_values(p): + def p_field_values(self, p): """field_values : field_values field_value | empty""" if len(p) == 2: # empty @@ -614,19 +535,19 @@ def p_field_values(p): assert isinstance(p[1], list) for f in p[1]: if f.name == p[2].name: - logging.error('[%s:%d] Cannot set field again: %s', filename, + logging.error('[%s:%d] Cannot set field again: %s', self.filename, p.lineno(2), p[2].name) p[1].append(p[2]) p[0] = p[1] - def p_field_value(p): + def p_field_value(self, p): """field_value : NAME '=' rvalue""" assert isinstance(p[3], stl.base.Value) p[0] = p[3] p[0].name = p[1] p.set_lineno(0, p.lineno(1)) - def p_rvalue(p): + def p_rvalue(self, p): """rvalue : value ';' | qualifier_value ';' | array ';' @@ -637,7 +558,7 @@ def p_rvalue(p): | message_array_value""" p[0] = p[1] - def p_array(p): + def p_array(self, p): """array : '[' ']' | '[' array_elements ']' | '[' array_elements ',' ']' """ @@ -646,7 +567,7 @@ def p_array(p): else: p[0] = stl.base.Value(p[2]) - def p_array_elements(p): + def p_array_elements(self, p): """array_elements : array_elements ',' array_element | array_element""" if len(p) == 2: # first element @@ -656,58 +577,56 @@ def p_array_elements(p): p[1].append(p[3]) p[0] = p[1] - def p_array_element(p): + def p_array_element(self, p): """array_element : value | array | struct""" p[0] = p[1] - def p_struct(p): + def p_struct(self, p): """struct : '{' field_values '}' """ p[0] = stl.base.Value(p[2]) - def p_message_array_value(p): + def p_message_array_value(self, p): """message_array_value : message_array | message_value""" p[0] = stl.base.Value(p[1]) - def p_qualifier_value(p): + def p_qualifier_value(self, p): """qualifier_value : NAME param_values ARROW reference | NAME param_values""" - qual = local_env['_curr_module'].qualifiers[p[1]] + qual = self.local_env['_curr_module'].qualifiers[p[1]] assert qual if len(p) == 5: p[0] = stl.base.QualifierValue(qual, p[2], stl.base.Value('&' + p[4])) else: p[0] = stl.base.QualifierValue(qual, p[2]) - def p_type(p): + def p_type(self, p): """type : BOOL | INT | NAME | STRING""" p[0] = p[1] - def p_type_or_role(p): + def p_type_or_role(self, p): """type_or_role : type | ROLE""" p[0] = p[1] - def p_empty(p): + def p_empty(self, p): """empty : """ p[0] = None - def p_error(p): - global_env['error'] = True + def p_error(self, p): + self.global_env['error'] = True if p is None: raise StlSyntaxError( '[{}] Syntax error: ' - 'Reached end of file unexpectantly.'.format(filename)) - else: - raise StlSyntaxError('[{}:{}] Syntax error at: {}'.format( - filename, p.lexer.lineno, p.value)) + 'Reached end of file unexpectantly.'.format(self.filename)) - return ply.yacc.yacc() + raise StlSyntaxError('[{}:{}] Syntax error at: {}'.format( + self.filename, p.lexer.lineno, p.value)) def Parse(filename, global_env): @@ -718,23 +637,16 @@ def Parse(filename, global_env): global_env: Dictionary to store global STL state. It has one field: global_env['modules']: Dictionary of stl.module.Module by name. """ - data = open(filename).read() - lexer = _GetLexer(filename) - parser = _GetParser(filename, global_env) - - parser.parse(data, lexer=lexer) - - -def _DebugParser(stl_filename): - global_env = {'modules': {}} - Parse(stl_filename, global_env) - return global_env + parser = StlParser(filename=filename, global_env=global_env) + with open(filename) as data: + return parser.parse(data.read()) def main(): logging.basicConfig(level=logging.DEBUG) - filename = sys.argv[1] if len(sys.argv) == 2 else 'model2/cast_v2.stl' - pprint.pprint(_DebugParser(filename), indent=2) + filename = sys.argv[1] + global_env = {'modules': {}} + pprint.pprint(Parse(filename, global_env), indent=2) if __name__ == '__main__': diff --git a/stl/parser_test.py b/stl/parser_test.py index 3bd264a..14e2cef 100644 --- a/stl/parser_test.py +++ b/stl/parser_test.py @@ -35,8 +35,8 @@ class StlParserTest(unittest.TestCase): def setUp(self): self.actual_module = stl.module.Module('foo') self.global_env = {'modules': {'foo': self.actual_module}} - self.lexer = stl.parser._GetLexer(self.TEST_FILENAME) - self.parser = stl.parser._GetParser(self.TEST_FILENAME, self.global_env) + self.parser = stl.parser.StlParser(self.TEST_FILENAME, self.global_env) + self.lexer = self.parser.lexer.lexer self.expected_module = stl.module.Module('foo') def tearDown(self): @@ -49,7 +49,7 @@ def Parse(self, text): Args: text: The text to parse. """ - self.parser.parse(text, lexer=self.lexer) + self.parser.parse(text) def testLineNumber(self): input_text = ('module foo;\n'