From 6de0aaac19f5079823ce065b7535ad7ecf2ae8ba Mon Sep 17 00:00:00 2001 From: Donne Martin Date: Sat, 19 Dec 2015 07:30:55 -0500 Subject: [PATCH 1/4] Implements #27: Add lexer/syntax highlighting. --- awsshell/app.py | 6 ++- awsshell/index/completion.py | 50 +++++++++++++++++++++- awsshell/lexer.py | 64 +++++++++++++++++++++++++++++ awsshell/ui.py | 12 ++++++ tests/unit/test_load_completions.py | 46 +++++++++++++++++++++ 5 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 awsshell/lexer.py create mode 100644 tests/unit/test_load_completions.py diff --git a/awsshell/app.py b/awsshell/app.py index 8340dfe..218e2b0 100644 --- a/awsshell/app.py +++ b/awsshell/app.py @@ -239,8 +239,12 @@ def stop_input_and_refresh_cli(self): raise InputInterrupt def create_layout(self, display_completions_in_columns, toolbar): + from awsshell.lexer import ShellLexer + lexer = ShellLexer + if self.config_section['theme'] == 'none': + lexer = None return create_default_layout( - self, u'aws> ', reserve_space_for_menu=True, + self, u'aws> ', lexer=lexer, reserve_space_for_menu=True, display_completions_in_columns=display_completions_in_columns, get_bottom_toolbar_tokens=toolbar.handler) diff --git a/awsshell/index/completion.py b/awsshell/index/completion.py index eaa7374..2306602 100644 --- a/awsshell/index/completion.py +++ b/awsshell/index/completion.py @@ -9,8 +9,10 @@ """ import os +import json from awsshell.utils import FSLayer, FileReadError, build_config_file_path +from awsshell import utils class IndexLoadError(Exception): @@ -18,7 +20,20 @@ class IndexLoadError(Exception): class CompletionIndex(object): - """Handles working with the local commmand completion index.""" + """Handles working with the local commmand completion index. + + :type commands: list + :param commands: ec2, s3, elb... + + :type subcommands: list + :param subcommands: start-instances, stop-instances, terminate-instances... + + :type global_opts: list + :param global_opts: --profile, --region, --output... + + :type args_opts: set, to filter out duplicates + :param args_opts: ec2 start-instances: --instance-ids, --dry-run... + """ # The completion index can read/write to a cache dir # so that it doesn't have to recompute the completion cache @@ -30,6 +45,11 @@ def __init__(self, cache_dir=DEFAULT_CACHE_DIR, fslayer=None): if fslayer is None: fslayer = FSLayer() self._fslayer = fslayer + self.commands = [] + self.subcommands = [] + self.global_opts = [] + self.args_opts = set() + self.load_completions() def load_index(self, version_string): """Load the completion index for a given CLI version. @@ -48,3 +68,31 @@ def load_index(self, version_string): def _filename_for_version(self, version_string): return os.path.join( self._cache_dir, 'completions-%s.json' % version_string) + + def load_completions(self): + """Loads completions from the completion index. + + Updates the following attributes: + * commands + * subcommands + * global_opts + * args_opts + """ + index_str = self.load_index(utils.AWSCLI_VERSION) + index_data = json.loads(index_str) + index_root = index_data['aws'] + # ec2, s3, elb... + self.commands = index_root['commands'] + # --profile, --region, --output... + self.global_opts = index_root['arguments'] + for command in self.commands: + # ec2: start-instances, stop-instances, terminate-instances... + subcommands_current = index_root['children'] \ + .get(command)['commands'] + self.subcommands.extend(subcommands_current) + for subcommand_current in subcommands_current: + # start-instances: --instance-ids, --dry-run... + args_opts_current = index_root['children'] \ + .get(command)['children'] \ + .get(subcommand_current)['arguments'] + self.args_opts.update(args_opts_current) diff --git a/awsshell/lexer.py b/awsshell/lexer.py new file mode 100644 index 0000000..3b80cad --- /dev/null +++ b/awsshell/lexer.py @@ -0,0 +1,64 @@ +# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import ast +import os + +from pygments.lexer import RegexLexer +from pygments.lexer import words +from pygments.token import Keyword, Literal, Name, Operator, Text + +from awsshell.index.completion import CompletionIndex + + +class ShellLexer(RegexLexer): + """Provides highlighting for commands, subcommands, arguments, and options. + + :type completion_index: :class:`CompletionIndex` + :param completion_index: Completion index used to determine commands, + subcommands, arguments, and options for highlighting. + + :type tokens: dict + :param tokens: A dict of (`pygments.lexer`, `pygments.token`) used for + pygments highlighting. + """ + completion_index = CompletionIndex() + tokens = { + 'root': [ + # ec2, s3, elb... + (words( + tuple(completion_index.commands), + prefix=r'\b', + suffix=r'\b'), + Literal.String), + # describe-instances + (words( + tuple(completion_index.subcommands), + prefix=r'\b', + suffix=r'\b'), + Name.Class), + # --instance-ids + (words( + tuple(list(completion_index.args_opts)), + prefix=r'', + suffix=r'\b'), + Keyword.Declaration), + # --profile + (words( + tuple(completion_index.global_opts), + prefix=r'', + suffix=r'\b'), + Operator.Word), + # Everything else + (r'.*\n', Text), + ] + } diff --git a/awsshell/ui.py b/awsshell/ui.py index c491ed9..657c9d3 100644 --- a/awsshell/ui.py +++ b/awsshell/ui.py @@ -16,7 +16,9 @@ from prompt_toolkit.layout.toolbars import ValidationToolbar, \ SystemToolbar, ArgToolbar, SearchToolbar from prompt_toolkit.layout.utils import explode_tokens +from prompt_toolkit.layout.lexers import PygmentsLexer from pygments.token import Token +from pygments.lexer import Lexer from awsshell.compat import text_type @@ -65,6 +67,16 @@ def create_default_layout(app, message='', get_prompt_tokens_1, get_prompt_tokens_2 = _split_multiline_prompt( get_prompt_tokens) + + # `lexer` is supposed to be a `Lexer` instance. But if a Pygments lexer + # class is given, turn it into a PygmentsLexer. (Important for + # backwards-compatibility.) + try: + if issubclass(lexer, Lexer): + lexer = PygmentsLexer(lexer) + except TypeError: # Happens when lexer is `None` or an instance of something else. + pass + # Create processors list. # (DefaultPrompt should always be at the end.) input_processors = [ diff --git a/tests/unit/test_load_completions.py b/tests/unit/test_load_completions.py new file mode 100644 index 0000000..291ef92 --- /dev/null +++ b/tests/unit/test_load_completions.py @@ -0,0 +1,46 @@ +# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import unittest + +from awsshell.index.completion import CompletionIndex + + +class LoadCompletionsTest(unittest.TestCase): + + def setUp(self): + self.completion_index = CompletionIndex() + # This would probably be cleaner with a pytest.fixture like + # test_completions.index_data + DATA = ( + '{"aws": ' + '{"commands": ["devicefarm", "foo"], ' + '"arguments": ["--debug", "--endpoint-url"], ' + '"children": {"devicefarm": ' + '{"commands": ["create-device-pool"], ' + '"children": {"create-device-pool": ' + '{"commands": [], ' + '"arguments": ["--project-arn", "--name"]}}}, ' + '"foo": ' + '{"commands": ["bar"], ' + '"children": {"bar": ' + '{"commands": [], "arguments": ["--baz"]}}}}}}' + ) + self.completion_index.load_index = lambda x: DATA + + def test_load_completions(self): + commands, subcommands, args_opts, global_opts = \ + self.completion_index.load_completions() + assert commands == ['devicefarm', 'foo'] + assert subcommands == ['create-device-pool', 'bar'] + assert args_opts == set(['--project-arn', '--name', '--baz']) + assert global_opts == ['--debug', '--endpoint-url'] From 23f3b58424816e4de9eab797a27ba21e336cb402 Mon Sep 17 00:00:00 2001 From: Donne Martin Date: Sat, 19 Dec 2015 08:50:44 -0500 Subject: [PATCH 2/4] Update load_completions handling of IndexLoadError. load_completions calls load_index, which can throw IndexLoadError if the index file doesn't exist. load_completions now simply does nothing if the index file does not exist, resulting in no change to the attributes commands, subcommands, global_opts, and args_opts, each of which is a collection of strings. If no completions exist because there's no index, it seems reasonable to return no completions. Not throwing an exception with load_completions also simplifies integration tests. --- awsshell/index/completion.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/awsshell/index/completion.py b/awsshell/index/completion.py index 2306602..77f6507 100644 --- a/awsshell/index/completion.py +++ b/awsshell/index/completion.py @@ -57,6 +57,7 @@ def load_index(self, version_string): :type version_string: str :param version_string: The AWS CLI version, e.g "1.9.2". + :raises: :class:`IndexLoadError ` """ filename = self._filename_for_version(version_string) try: @@ -78,6 +79,10 @@ def load_completions(self): * global_opts * args_opts """ + try: + index_str = self.load_index(utils.AWSCLI_VERSION) + except IndexLoadError: + return index_str = self.load_index(utils.AWSCLI_VERSION) index_data = json.loads(index_str) index_root = index_data['aws'] From 5adca9ee58f0de8a43eac5598276ec930299f91c Mon Sep 17 00:00:00 2001 From: Donne Martin Date: Sat, 19 Dec 2015 08:58:38 -0500 Subject: [PATCH 3/4] Update load_completions test to use new CompletionIndex attributes. CompletionIndex was recently refactored locally (prior to the initial commit for the lexer feature) to use attributes for commands, subcommands, args_opts, global_opts rather than returning the equivalent 4 element tuple in load_completions. This change updates the tests accordingly. --- tests/unit/test_load_completions.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_load_completions.py b/tests/unit/test_load_completions.py index 291ef92..4a7c66e 100644 --- a/tests/unit/test_load_completions.py +++ b/tests/unit/test_load_completions.py @@ -38,9 +38,11 @@ def setUp(self): self.completion_index.load_index = lambda x: DATA def test_load_completions(self): - commands, subcommands, args_opts, global_opts = \ - self.completion_index.load_completions() - assert commands == ['devicefarm', 'foo'] - assert subcommands == ['create-device-pool', 'bar'] - assert args_opts == set(['--project-arn', '--name', '--baz']) - assert global_opts == ['--debug', '--endpoint-url'] + assert self.completion_index.commands == [ + 'devicefarm', 'foo'] + assert self.completion_index.subcommands == [ + 'create-device-pool', 'bar'] + assert self.completion_index.global_opts == [ + '--debug', '--endpoint-url'] + assert self.completion_index.args_opts == set([ + '--project-arn', '--name', '--baz']) From fd22e54c913361e13c6324076a254720c58a2726 Mon Sep 17 00:00:00 2001 From: Donne Martin Date: Sat, 19 Dec 2015 09:15:40 -0500 Subject: [PATCH 4/4] Moved CompletionIndex.load_completions call from init. Moving this out is more modular, as CompletionIndex() is created in two places in the source: main and lexer (possible refactor TODO item). It's also called once in the tests. This change also simplifies testing as we can more easily substitute a mock function, although this could also probably be achieved with a mock side effect. --- awsshell/index/completion.py | 1 - awsshell/lexer.py | 1 + tests/unit/test_load_completions.py | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/awsshell/index/completion.py b/awsshell/index/completion.py index 77f6507..5737a09 100644 --- a/awsshell/index/completion.py +++ b/awsshell/index/completion.py @@ -49,7 +49,6 @@ def __init__(self, cache_dir=DEFAULT_CACHE_DIR, fslayer=None): self.subcommands = [] self.global_opts = [] self.args_opts = set() - self.load_completions() def load_index(self, version_string): """Load the completion index for a given CLI version. diff --git a/awsshell/lexer.py b/awsshell/lexer.py index 3b80cad..287092a 100644 --- a/awsshell/lexer.py +++ b/awsshell/lexer.py @@ -32,6 +32,7 @@ class ShellLexer(RegexLexer): pygments highlighting. """ completion_index = CompletionIndex() + completion_index.load_completions() tokens = { 'root': [ # ec2, s3, elb... diff --git a/tests/unit/test_load_completions.py b/tests/unit/test_load_completions.py index 4a7c66e..ae5086b 100644 --- a/tests/unit/test_load_completions.py +++ b/tests/unit/test_load_completions.py @@ -36,6 +36,7 @@ def setUp(self): '{"commands": [], "arguments": ["--baz"]}}}}}}' ) self.completion_index.load_index = lambda x: DATA + self.completion_index.load_completions() def test_load_completions(self): assert self.completion_index.commands == [