diff --git a/README.rst b/README.rst index c020884..f1b8d08 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,8 @@ for `Robot Framework `_ test data in plain text format. It is available as a separate plugin and *included in Pygments 1.6 and newer*. +This version will be available not before + What is Pygments ---------------- @@ -57,7 +59,21 @@ script for an example of the programmatic usage. For general information about using Pygments, consult `its documentation `_ and/or the documentation of the tool you are using it with. +Alternative Usage +----------------- + +You can use ``robotframeworklexer.py`` as a custom lexer in ``pygments``. See the documentation +of ``pygments`` for more information. Below, are two examples applied to the test suite `example.robot `_. + +.. code:: bash + + python -m pygments -l ./robotframeworklexer.py:RobotFrameworkLexer -x -o example.png -f png ./example.robot + # result: ``_ + + python -m pygments -l ./robotframeworklexer.py:RobotFrameworkLexer -x -o example.txt -f raw ./example.robot + # result: ``_ + License ------- -`Apache License, Version 2.0 `_. \ No newline at end of file +`Apache License, Version 2.0 `_. diff --git a/example.png b/example.png new file mode 100644 index 0000000..0010d1e Binary files /dev/null and b/example.png differ diff --git a/example.robot b/example.robot index b27d205..f233ab3 100644 --- a/example.robot +++ b/example.robot @@ -1,12 +1,12 @@ *** Settings *** -Documentation Simple example demonstrating syntax highlighting. -Library ExampleLibrary -Test Setup Keyword argument argument with ${VARIABLE} +Documentation Simple example demonstrating syntax highlighting. +Test Setup Keyword argument argument with ${VARIABLE} +Library Process *** Variables *** -${VARIABLE} Variable value -@{LIST} List variable here -&{DICT} Key1=Value1 Key2=Value2 +${VARIABLE} Variable value +@{LIST} List variable here +&{DICT} Key1=Value1 Key2=Value2 *** Test Cases *** Keyword-driven example @@ -17,7 +17,7 @@ Keyword-driven example Data-driven example [Template] Keyword - argument1 argument2 + argument1 argument2 argument ${VARIABLE} @{LIST} @@ -30,12 +30,57 @@ Gherkin | | [Documentation] | Also pipe separated format is supported. | | | Log | As this example demonstrates. | +*** Comments *** +This is a section of comments. +We can have many lines without any comment marker. + *** Keywords *** Result Should Be [Arguments] ${expected} - [Tags] whatever - ${actual} = Get Value + [Tags] whatever + ${actual} = Get Value ${expected} Should be Equal ${actual} ${expected} Then result should be "${expected}" Result Should Be ${expected} + +System is initialized + # This is a single line comment + Initialize System + +something is done + Do Something + +Keyword + [Arguments] ${arg1} ${arg2} ${arg3}=${EMPTY} + IF "${arg1}" == "${arg2}" + Log Equal Arguments + ELSE IF "${arg1}" == "argument1" + Log arg1 is equal to argument1 + ELSE + FOR ${idx} IN RANGE 4 + IF ${idx} == 3 BREAK + Log ${\n}arg${idx+1} = ${arg${idx+1}} console=True + END + END + +Initialize System + Log System initialized + +Do Something + Run Keywords Log Done + ... AND No Operation + +Cleanup System + TRY + Log %{HOME} + EXCEPT + Log To Console \%{HOME} does not exist + ELSE + Log To Console \%{HOME} exists + END + Log System cleaned + +Get Value + [Arguments] ${arg1}=42 + RETURN ${arg1} diff --git a/example.txt b/example.txt new file mode 100644 index 0000000..2153804 --- /dev/null +++ b/example.txt @@ -0,0 +1,403 @@ +Token.Generic.Heading '*** Settings ***' +Token.Punctuation '\n' +Token.Keyword.Namespace 'Documentation' +Token.Punctuation ' ' +Token.Literal.String 'Simple example demonstrating syntax highlighting.' +Token.Punctuation '\n' +Token.Keyword.Namespace 'Test Setup' +Token.Punctuation ' ' +Token.Name.Function 'Keyword' +Token.Punctuation ' ' +Token.Literal.String 'argument' +Token.Punctuation ' ' +Token.Literal.String 'argument with ' +Token.Punctuation '${' +Token.Name.Variable 'VARIABLE' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Keyword.Namespace 'Library' +Token.Punctuation ' ' +Token.Name.Namespace 'Process' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Heading '*** Variables ***' +Token.Punctuation '\n' +Token.Punctuation '${' +Token.Name.Variable 'VARIABLE' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Literal.String 'Variable value' +Token.Punctuation '\n' +Token.Punctuation '@{' +Token.Name.Variable 'LIST' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Literal.String 'List' +Token.Punctuation ' ' +Token.Literal.String 'variable' +Token.Punctuation ' ' +Token.Literal.String 'here' +Token.Punctuation '\n' +Token.Punctuation '&{' +Token.Name.Variable 'DICT' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Literal.String 'Key1=Value1' +Token.Punctuation ' ' +Token.Literal.String 'Key2=Value2' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Heading '*** Test Cases ***' +Token.Punctuation '\n' +Token.Generic.Subheading 'Keyword-driven example' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Initialize System' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Do Something' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Result Should Be' +Token.Punctuation ' ' +Token.Literal.String '42' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Teardown' +Token.Punctuation ']' +Token.Punctuation ' ' +Token.Name.Function 'Cleanup System' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Data-driven example' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Template' +Token.Punctuation ']' +Token.Punctuation ' ' +Token.Name.Function 'Keyword' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'argument1' +Token.Punctuation ' ' +Token.Literal.String 'argument2' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'argument' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'VARIABLE' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '@{' +Token.Name.Variable 'LIST' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Gherkin' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Generic.Emph 'Given ' +Token.Name.Function 'system is initialized' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Generic.Emph 'When ' +Token.Name.Function 'something is done' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Generic.Emph 'Then ' +Token.Name.Function 'result should be "42"' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Punctuation '| ' +Token.Generic.Subheading 'Pipes' +Token.Punctuation ' |' +Token.Punctuation '\n' +Token.Punctuation '| ' +Token.Punctuation '| ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Documentation' +Token.Punctuation ']' +Token.Punctuation ' | ' +Token.Literal.String 'Also pipe separated format is supported.' +Token.Punctuation ' |' +Token.Punctuation '\n' +Token.Punctuation '| ' +Token.Punctuation '| ' +Token.Name.Function 'Log' +Token.Punctuation ' | ' +Token.Literal.String 'As this example demonstrates.' +Token.Punctuation ' |' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Heading '*** Comments ***' +Token.Punctuation '\n' +Token.Comment 'This is a section of comments.' +Token.Punctuation '\n' +Token.Comment 'We can have many lines without any comment marker.' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Heading '*** Keywords ***' +Token.Punctuation '\n' +Token.Generic.Subheading 'Result Should Be' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Arguments' +Token.Punctuation ']' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'expected' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Tags' +Token.Punctuation ']' +Token.Punctuation ' ' +Token.Literal.String 'whatever' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'actual' +Token.Punctuation '}' +Token.Punctuation ' =' +Token.Punctuation ' ' +Token.Name.Function 'Get Value' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'expected' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Should be Equal' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'actual' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'expected' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Emph 'Then ' +Token.Generic.Subheading 'result should be "' +Token.Punctuation '${' +Token.Name.Variable 'expected' +Token.Punctuation '}' +Token.Generic.Subheading '"' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Result Should Be' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'expected' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'System is initialized' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Comment '# This is a single line comment' +Token.Comment '\n' +Token.Punctuation ' ' +Token.Name.Function 'Initialize System' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'something is done' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Do Something' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Keyword' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Arguments' +Token.Punctuation ']' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'arg1' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'arg2' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'arg3' +Token.Punctuation '}' +Token.Literal.String '=' +Token.Punctuation '${' +Token.Name.Variable 'EMPTY' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'IF' +Token.Punctuation ' ' +Token.Name.Function '"' +Token.Punctuation '${' +Token.Name.Variable 'arg1' +Token.Punctuation '}' +Token.Name.Function '" == "' +Token.Punctuation '${' +Token.Name.Variable 'arg2' +Token.Punctuation '}' +Token.Name.Function '"' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log' +Token.Punctuation ' ' +Token.Literal.String 'Equal Arguments' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'ELSE IF' +Token.Punctuation ' ' +Token.Name.Function '"' +Token.Punctuation '${' +Token.Name.Variable 'arg1' +Token.Punctuation '}' +Token.Name.Function '" == "argument1"' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log' +Token.Punctuation ' ' +Token.Literal.String 'arg1 is equal to argument1' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'ELSE' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'FOR' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'idx' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Name.Function.Magic 'IN RANGE' +Token.Punctuation ' ' +Token.Literal.String '4' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'IF' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'idx' +Token.Punctuation '}' +Token.Name.Function ' == 3' +Token.Punctuation ' ' +Token.Name.Function.Magic 'BREAK' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable '\\n' +Token.Punctuation '}' +Token.Literal.String 'arg' +Token.Punctuation '${' +Token.Name.Variable 'idx+1' +Token.Punctuation '}' +Token.Literal.String ' = ' +Token.Punctuation '${' +Token.Name.Variable 'arg' +Token.Punctuation '${' +Token.Name.Variable 'idx+1' +Token.Punctuation '}' +Token.Punctuation '}' +Token.Punctuation ' ' +Token.Literal.String 'console=True' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'END' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'END' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Initialize System' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log' +Token.Punctuation ' ' +Token.Literal.String 'System initialized' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Do Something' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Run Keywords' +Token.Punctuation ' ' +Token.Literal.String 'Log' +Token.Punctuation ' ' +Token.Literal.String 'Done' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '...' +Token.Punctuation ' ' +Token.Name.Function.Magic 'AND' +Token.Punctuation ' ' +Token.Literal.String 'No Operation' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Cleanup System' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'TRY' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log' +Token.Punctuation ' ' +Token.Punctuation '%{' +Token.Name.Variable 'HOME' +Token.Punctuation '}' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'EXCEPT' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log To Console' +Token.Punctuation ' ' +Token.Literal.String '\\%{HOME} does not exist' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'ELSE' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log To Console' +Token.Punctuation ' ' +Token.Literal.String '\\%{HOME} exists' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'END' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function 'Log' +Token.Punctuation ' ' +Token.Literal.String 'System cleaned' +Token.Punctuation '\n' +Token.Punctuation '\n' +Token.Generic.Subheading 'Get Value' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Punctuation '[' +Token.Keyword.Namespace 'Arguments' +Token.Punctuation ']' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'arg1' +Token.Punctuation '}' +Token.Literal.String '=42' +Token.Punctuation '\n' +Token.Punctuation ' ' +Token.Name.Function.Magic 'RETURN' +Token.Punctuation ' ' +Token.Punctuation '${' +Token.Name.Variable 'arg1' +Token.Punctuation '}' +Token.Punctuation '\n' diff --git a/robotframeworklexer.py b/robotframeworklexer.py index 7874934..1939f09 100644 --- a/robotframeworklexer.py +++ b/robotframeworklexer.py @@ -1,4 +1,6 @@ # Copyright 2012 Nokia Siemens Networks Oyj +# Copyright 2012-2015 Nokia Networks +# Copyright 2016-present Robot Framework Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,13 +20,14 @@ from pygments.token import Token -__version__ = '1.1.1.dev1' +__version__ = '1.2.dev1' HEADING = Token.Generic.Heading SETTING = Token.Keyword.Namespace IMPORT = Token.Name.Namespace TC_KW_NAME = Token.Generic.Subheading KEYWORD = Token.Name.Function +CONTROL = Token.Name.Function.Magic ARGUMENT = Token.String VARIABLE = Token.Name.Variable COMMENT = Token.Comment @@ -33,14 +36,20 @@ GHERKIN = Token.Generic.Emph ERROR = Token.Error +FORSEP = ('IN', 'IN ENUMERATE', 'IN RANGE', 'IN ZIP') +CONTROLS = ('AND', 'BREAK', 'CONTINUE', 'ELSE', 'ELSE IF', 'END', 'EXCEPT', + 'FINALLY', 'GROUP', 'IF', FORSEP[:], 'RETURN', 'TRY', 'VAR', 'WHILE') + def normalize(string, remove='', strip=True): string = string.lower() - for char in remove: + for char in remove + ' ': if char in string: string = string.replace(char, '') return string if not strip else string.strip() +def is_control(value): + return value in CONTROLS and value not in FORSEP class RobotFrameworkLexer(Lexer): """ @@ -49,8 +58,9 @@ class RobotFrameworkLexer(Lexer): Supports both space and pipe separated plain text formats. """ name = 'RobotFramework' - aliases = ['RobotFramework', 'robotframework'] - filenames = ['*.robot'] + url = 'http://robotframework.org' + aliases = ['robotframework'] + filenames = ['*.robot', '*.resource'] mimetypes = ['text/x-robotframework'] def __init__(self, **options): @@ -66,13 +76,11 @@ def get_tokens_unprocessed(self, text): for value, token in row_tokenizer.tokenize(row): for value, token in var_tokenizer.tokenize(value, token): if value: - if isinstance(value, bytes): - value = value.decode('UTF-8') - yield index, token, value + yield index, token, str(value) index += len(value) -class VariableTokenizer(object): +class VariableTokenizer: def tokenize(self, string, token): var = VariableSplitter(string, identifiers='$@%&') @@ -87,34 +95,32 @@ def _tokenize(self, var, string, orig_token): before = string[:var.start] yield before, orig_token yield var.identifier + '{', SYNTAX - for value, token in self.tokenize(var.base, VARIABLE): - yield value, token + yield from self.tokenize(var.base, VARIABLE) yield '}', SYNTAX for item in var.items: yield '[', SYNTAX - for value, token in self.tokenize(item, VARIABLE): - yield value, token + yield from self.tokenize(item, VARIABLE) yield ']', SYNTAX - for value, token in self.tokenize(string[var.end:], orig_token): - yield value, token + yield from self.tokenize(string[var.end:], orig_token) -class RowTokenizer(object): +class RowTokenizer: def __init__(self): + self._table = UnknownTable() + self._splitter = RowSplitter() testcases = TestCaseTable() settings = SettingTable(testcases.set_default_template) variables = VariableTable() keywords = KeywordTable() - comments = CommentTable() - self._table = comments - self._tables = {'settings': settings, 'setting': settings, - 'variables': variables, 'variable': variables, - 'test cases': testcases, 'test case': testcases, - 'tasks': testcases, 'task': testcases, - 'keywords': keywords, 'keyword': keywords, - 'comments': comments, 'comment': comments} - self._splitter = RowSplitter() + comments = CommentsTable() + self._tables = {'settings': settings, + 'metadata': settings, + 'variables': variables, + 'testcases': testcases, + 'tasks': testcases, + 'keywords': keywords, + 'comments': comments} def tokenize(self, row): commented = False @@ -127,9 +133,8 @@ def tokenize(self, row): elif index == 0 and value.startswith('*'): self._table = self._start_table(value) heading = True - for value, token in self._tokenize(value, index, commented, - separator, heading): - yield value, token + yield from self._tokenize(value, index, commented, + separator, heading) self._table.end_row() def _start_table(self, header): @@ -144,29 +149,27 @@ def _tokenize(self, value, index, commented, separator, heading): elif heading: token = HEADING if self._in_valid_table() else ERROR yield value, token + elif is_control(value): + yield value, CONTROL else: - for value, token in self._table.tokenize(value, index): - yield value, token + yield from self._table.tokenize(value, index) def _in_valid_table(self): return not isinstance(self._table, UnknownTable) - -class RowSplitter(object): +class RowSplitter: _space_splitter = re.compile('( {2,})') - _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))') + _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') def split(self, row): - splitter = self._split_from_spaces \ - if not row.startswith('| ') else self._split_from_pipes - for value in splitter(row): - yield value + splitter = (row.startswith('| ') and self._split_from_pipes + or self._split_from_spaces) + yield from splitter(row) yield '\n' def _split_from_spaces(self, row): yield '' # Start with (pseudo)separator similarly as with pipes - for value in self._space_splitter.split(row): - yield value + yield from self._space_splitter.split(row) def _split_from_pipes(self, row): _, separator, rest = self._pipe_splitter.split(row, 1) @@ -178,7 +181,7 @@ def _split_from_pipes(self, row): yield rest -class Tokenizer(object): +class Tokenizer: _tokens = None def __init__(self): @@ -208,12 +211,12 @@ class Comment(Tokenizer): class Setting(Tokenizer): _tokens = (SETTING, ARGUMENT) - _keyword_settings = ('suite setup', 'suite teardown', - 'test setup', 'test teardown', 'test template', - 'task setup', 'task teardown', 'task template') + _keyword_settings = ('suitesetup', 'suiteteardown', + 'arguments', 'teardown', 'testsetup', 'tasksetup', + 'testteardown','taskteardown', 'testtemplate', 'tasktemplate', 'setup', 'template') _import_settings = ('library', 'resource', 'variables') - _other_settings = ('documentation', 'metadata', 'force tags', 'default tags', - 'test timeout', 'task timeout') + _other_settings = ('documentation', 'metadata', 'testtags', 'tasktags', 'tags', 'forcetags', 'defaulttags', + 'testtimeout','tasktimeout', 'timeout') _custom_tokenizer = None def __init__(self, template_setter=None): @@ -246,14 +249,19 @@ class TestCaseSetting(Setting): _other_settings = ('documentation', 'tags', 'timeout') def _tokenize(self, value, index): + normalized = normalize(value) + if normalized in self._keyword_settings: + self._custom_tokenizer = KeywordCall(support_assign=False) if index == 0: - token = Setting._tokenize(self, value[1:-1], index) - return [('[', SYNTAX), (value[1:-1], token), (']', SYNTAX)] + stype = Setting._tokenize(self, value[1:-1], index) + return [('[', SYNTAX), (value[1:-1], stype), (']', SYNTAX)] + elif self._custom_tokenizer: + return self._custom_tokenizer.tokenize(value) return Setting._tokenize(self, value, index) class KeywordSetting(TestCaseSetting): - _keyword_settings = ('teardown',) + _keyword_settings = ('setup', 'testsetup', 'teardown', 'template') _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags') @@ -272,19 +280,27 @@ class KeywordCall(Tokenizer): def __init__(self, support_assign=True): Tokenizer.__init__(self) self._keyword_found = not support_assign + self._control_found = False self._assigns = 0 def _tokenize(self, value, index): if not self._keyword_found and self._is_assign(value): self._assigns += 1 return SYNTAX # VariableTokenizer tokenizes this later. - if self._keyword_found: + if self._keyword_found or self._control_found: + if not is_control(value): + self._tokens = (KEYWORD, ARGUMENT) + else: + self._tokens = (CONTROL, ARGUMENT) return Tokenizer._tokenize(self, value, index - self._assigns) - self._keyword_found = True - return GherkinTokenizer().tokenize(value, KEYWORD) + self._control_found = is_control(value) or value in FORSEP + if self._control_found: + self._tokens = (CONTROL, ARGUMENT) + self._keyword_found = not self._control_found + return GherkinTokenizer().tokenize(value, self._tokens[0]) -class GherkinTokenizer(object): +class GherkinTokenizer: _gherkin_prefix = re.compile('^(Given|When|Then|And|But) ', re.IGNORECASE) def tokenize(self, value, token): @@ -303,30 +319,29 @@ class ForLoop(Tokenizer): def __init__(self): Tokenizer.__init__(self) - self._started = False self._in_arguments = False def _tokenize(self, value, index): - if not self._started: - self._started = True - return SYNTAX - if self._in_arguments: - return ARGUMENT # Possible variables tokenized later - if self._is_separator(value): + token = self._in_arguments and ARGUMENT or SYNTAX + if value in ('FOR', 'IN', 'IN ENUMERATE', 'IN RANGE', 'IN ZIP'): # value must be in all caps + self._in_arguments = value != 'FOR' + token = CONTROL + elif (index == 0 and value in (': FOR', 'for') or + index > 1 and value in ('in', 'in enumerate', 'in range', 'in zip')): self._in_arguments = True - return SYNTAX - if self._is_variale(value): - return SYNTAX # Tokenized later - return ERROR - - def _is_separator(self, value): - return value in ('IN', 'IN RANGE', 'IN ENUMERATE', 'IN ZIP') - - def _is_variale(self, value): - return value[:2] == '${' or value[-1:] == '}' + token = ERROR + elif index >= 1 and not self._in_arguments: + var = list(VariableTokenizer().tokenize(value, ARGUMENT)) + if len(var) > 1 and var[1][1] == VARIABLE: + token = SYNTAX + elif var[0][1] == ARGUMENT: + token = ERROR + else: + token = ARGUMENT + return token -class _Table(object): +class _Table: _tokenizer_class = None def __init__(self, prev_tokenizer=None): @@ -339,8 +354,7 @@ def tokenize(self, value, index): self._tokenizer = self._prev_tokenizer yield value, SYNTAX else: - for value_and_token in self._tokenize(value, index): - yield value_and_token + yield from self._tokenize(value, index) self._prev_values_on_row.append(value) def _continues(self, value, index): @@ -357,14 +371,14 @@ def end_row(self): self.__init__(prev_tokenizer=self._tokenizer) -class CommentTable(_Table): +class CommentsTable(_Table): _tokenizer_class = Comment def _continues(self, value, index): return False -class UnknownTable(CommentTable): +class UnknownTable(CommentsTable): pass @@ -380,7 +394,7 @@ def __init__(self, template_setter, prev_tokenizer=None): self._template_setter = template_setter def _tokenize(self, value, index): - if index == 0 and normalize(value) == 'test template': + if index == 0 and normalize(value) == 'testtemplate': self._tokenizer = Setting(self._template_setter) return _Table._tokenize(self, value, index) @@ -414,9 +428,9 @@ def _tokenize(self, value, index): self._tokenizer = self._setting_class(self.set_test_template) else: self._tokenizer = self._setting_class() - if index == 1 and self._is_for_loop(value): + if self._is_for_loop(value): self._tokenizer = ForLoop() - if index == 1 and (value == 'END' or self._is_empty(value)): + if index == 1 and self._is_empty(value): return [(value, SYNTAX)] return _Table._tokenize(self, value, index) @@ -424,11 +438,10 @@ def _is_setting(self, value): return value.startswith('[') and value.endswith(']') def _is_template(self, value): - return normalize(value[1:-1]) == 'template' + return normalize(value) == '[template]' def _is_for_loop(self, value): - return (value == 'FOR' or - value.startswith(':') and normalize(value, remove=': ') == 'for') + return (value.startswith(':') and normalize(value, remove=':') == 'for') or value == 'FOR' def set_test_template(self, template): self._test_template = self._is_template_set(template) @@ -450,7 +463,7 @@ def _is_template(self, value): # Following code copied from Robot Framework 3.1.1. -class VariableSplitter(object): +class VariableSplitter: def __init__(self, string, identifiers='$@%&*'): self.identifier = None diff --git a/test/test_robotframeworklexer.py b/test/test_robotframeworklexer.py index e2851d3..7635011 100644 --- a/test/test_robotframeworklexer.py +++ b/test/test_robotframeworklexer.py @@ -138,61 +138,61 @@ def _tokenize(self, string): def test_in(self): SEP = (SYNTAX, ' ') self._verify('FOR ${x} IN foo bar', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, - (SYNTAX, 'IN'), SEP, + (CONTROL, 'IN'), SEP, (ARGUMENT, 'foo'), SEP, (ARGUMENT, 'bar')) def test_in_range(self): SEP = (SYNTAX, ' ') self._verify('FOR ${index} IN RANGE 1 ${10}', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'index'), (SYNTAX, '}'), SEP, - (SYNTAX, 'IN RANGE'), SEP, + (CONTROL, 'IN RANGE'), SEP, (ARGUMENT, '1'), SEP, (SYNTAX, '${'), (VARIABLE, '10'), (SYNTAX, '}')) def test_in_enumerate(self): SEP = (SYNTAX, ' ') self._verify('FOR ${index} ${item} IN ENUMERATE foo bar', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'index'), (SYNTAX, '}'), SEP, (SYNTAX, '${'), (VARIABLE, 'item'), (SYNTAX, '}'), SEP, - (SYNTAX, 'IN ENUMERATE'), SEP, + (CONTROL, 'IN ENUMERATE'), SEP, (ARGUMENT, 'foo'), SEP, (ARGUMENT, 'bar')) def test_in_zip(self): SEP = (SYNTAX, ' ') self._verify('FOR ${x} ${y} IN ZIP ${XXX} ${YYY}', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, (SYNTAX, '${'), (VARIABLE, 'y'), (SYNTAX, '}'), SEP, - (SYNTAX, 'IN ZIP'), SEP, + (CONTROL, 'IN ZIP'), SEP, (SYNTAX, '${'), (VARIABLE, 'XXX'), (SYNTAX, '}'), SEP, (SYNTAX, '${'), (VARIABLE, 'YYY'), (SYNTAX, '}')) def test_old_for(self): SEP = (SYNTAX, ' ') self._verify(': FOR ${x} IN foo bar', - (SYNTAX, ': FOR'), SEP, + (ERROR, ': FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, - (SYNTAX, 'IN'), SEP, + (CONTROL, 'IN'), SEP, (ARGUMENT, 'foo'), SEP, (ARGUMENT, 'bar')) def test_case_sensitive(self): SEP = (SYNTAX, ' ') self._verify('FOR ${x} in foo bar', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, (ERROR, 'in'), SEP, - (ERROR, 'foo'), SEP, (ERROR, 'bar')) + (ARGUMENT, 'foo'), SEP, (ARGUMENT, 'bar')) def test_invalid_variable(self): SEP = (SYNTAX, ' ') self._verify('FOR x IN foo bar', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (ERROR, 'x'), SEP, - (SYNTAX, 'IN'), SEP, + (CONTROL, 'IN'), SEP, (ARGUMENT, 'foo'), SEP, (ARGUMENT, 'bar')) def test_with_body_and_end(self): @@ -200,13 +200,56 @@ def test_with_body_and_end(self): SEP2 = (SYNTAX, ' ') NEWLINE = (SYNTAX, '\n') self._verify('FOR ${x} IN foo bar\n Log ${x}\nEND', - (SYNTAX, 'FOR'), SEP, + (CONTROL, 'FOR'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, - (SYNTAX, 'IN'), SEP, + (CONTROL, 'IN'), SEP, (ARGUMENT, 'foo'), SEP, (ARGUMENT, 'bar'), NEWLINE, SEP2, (KEYWORD, 'Log'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), NEWLINE, - SEP, (SYNTAX, 'END')) + SEP, (CONTROL, 'END')) + + +class TestControlMarkers(unittest.TestCase): + + def _verify(self, block, *expected): + block = '\n'.join(' '+ line for line in block.splitlines()) + test = '*** Test Cases ***\nMiscellaneous\n' + block + actual = list(RobotFrameworkLexer().get_tokens(test)) + expected = [(HEADING, '*** Test Cases ***'), + (SYNTAX, '\n'), + (TC_KW_NAME, 'Miscellaneous'), + (SYNTAX, '\n'), + (SYNTAX, ' ')] + list(expected) + [(SYNTAX, '\n')] + self.assertEqual(len(actual), len(expected)) + for act, exp in zip(actual, expected): + self.assertEqual(act, exp) + + def _tokenize(self, string): + tokenizer = RowTokenizer() # KeywordCall() + for item in string.split(): + yield tokenizer.tokenize(item) + + def test_while_in(self): + SEP = (SYNTAX, ' ') + self._verify('WHILE ${x} IN @{values}', + (CONTROL, 'WHILE'), SEP, + (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, + (CONTROL, 'IN'), SEP, + (SYNTAX, '@{'), (VARIABLE, 'values'), (SYNTAX, '}')) + + def test_while_continue(self): + SEP = (SYNTAX, ' ') + SEP2 = (SYNTAX, ' ') + NEWLINE = (SYNTAX, '\n') + self._verify('''WHILE ${x} % 3 != 0\n IF ${x} < 11\n VAR ${x} ${x+1}\n''' + ''' ELSE\n CONTINUE\nEND\nEND''', + (CONTROL, 'WHILE'), SEP, + (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), (KEYWORD, ' % 3 != 0'), NEWLINE, + SEP2, (CONTROL, 'IF'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), (KEYWORD, ' < 11'), + NEWLINE, SEP2, (CONTROL, 'VAR'), SEP, (SYNTAX, '${'), (VARIABLE, 'x'), (SYNTAX, '}'), SEP, + (SYNTAX, '${'), (VARIABLE, 'x+1'), (SYNTAX, '}'), NEWLINE, SEP2, (CONTROL, 'ELSE'), + NEWLINE, SEP2, (CONTROL, 'CONTINUE'), NEWLINE, SEP, (CONTROL, 'END'), NEWLINE, SEP, + (CONTROL, 'END')) class TestTrailingSpaces(unittest.TestCase):