From 3caa73d243c8f42dbb72024b9f514aa2fd8026e4 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 29 Jul 2020 09:26:06 +1000 Subject: [PATCH 01/40] feat(pyfzf): remove echo in execute_fzf --- fzfaws/utils/pyfzf.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fzfaws/utils/pyfzf.py b/fzfaws/utils/pyfzf.py index 62bafe3..ab821b2 100644 --- a/fzfaws/utils/pyfzf.py +++ b/fzfaws/utils/pyfzf.py @@ -113,7 +113,6 @@ def execute_fzf( """ # remove trailing spaces/lines self.fzf_string = str(self.fzf_string).rstrip() - fzf_input = subprocess.Popen(("echo", self.fzf_string), stdout=subprocess.PIPE) cmd_list: list = self._construct_fzf_cmd() selection: bytes = b"" selection_str: str = "" @@ -130,7 +129,16 @@ def execute_fzf( cmd_list.extend(["--preview", preview]) try: - selection = subprocess.check_output(cmd_list, stdin=fzf_input.stdout) + proc = subprocess.Popen( + cmd_list, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None + ) + stdin = proc.stdin + stdin.write(self.fzf_string.encode("utf-8")) + stdin.flush() + stdin.close() + stdout = proc.stdout + selection = stdout.read() + selection_str = str(selection, "utf-8") if not selection and not empty_allow: @@ -139,16 +147,10 @@ def execute_fzf( # if first line contains ctrl-c, exit self._check_ctrl_c(selection_str) - except subprocess.CalledProcessError: - # this exception may happend if user didn't make a selection in fzf - # thus ending with non zero exit code + except (subprocess.CalledProcessError, IOError): if not empty_allow: raise NoSelectionMade - elif empty_allow: - if multi_select: - return [] - else: - return "" + return [] if multi_select else "" if multi_select: return_list: List[str] = [] From 9161aeeb738cd6aa3e6eaa90f4e03af9cdf82754 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 29 Jul 2020 09:46:58 +1000 Subject: [PATCH 02/40] test(pyfzf): reflect new changes --- fzfaws/s3/s3.py | 2 +- fzfaws/utils/pyfzf.py | 2 +- tests/utils/test_pyfzf.py | 28 +++++++++++++--------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/fzfaws/s3/s3.py b/fzfaws/s3/s3.py index f1341a5..b2edc44 100644 --- a/fzfaws/s3/s3.py +++ b/fzfaws/s3/s3.py @@ -432,7 +432,7 @@ def _get_path_option(self, download: bool = False) -> str: selected_option = str( fzf.execute_fzf( print_col=1, - header="Please select which level of the bucket would you like to operate in", + header="select which level of the bucket would you like to operate in", delimiter=": ", ) ) diff --git a/fzfaws/utils/pyfzf.py b/fzfaws/utils/pyfzf.py index ab821b2..6759c09 100644 --- a/fzfaws/utils/pyfzf.py +++ b/fzfaws/utils/pyfzf.py @@ -203,7 +203,7 @@ def get_local_file( home_path = os.path.expanduser("~") os.chdir(home_path) if not header and directory: - header = r"Selecting ./ will use current directory" + header = r"select ./ will use current directory" cmd: str = "" diff --git a/tests/utils/test_pyfzf.py b/tests/utils/test_pyfzf.py index 656e715..ba62e17 100644 --- a/tests/utils/test_pyfzf.py +++ b/tests/utils/test_pyfzf.py @@ -76,41 +76,39 @@ def test_construct_fzf_command(self): ], ) - @patch.object(subprocess, "Popen") - @patch.object(subprocess, "check_output") - def test_execute_fzf(self, mocked_output, mocked_popen): - mocked_output.return_value = b"hello" + @patch("fzfaws.utils.pyfzf.subprocess.Popen") + def test_execute_fzf(self, MockedPopen): + proc = MockedPopen() + proc.stdout.read.return_value = b"hello" result = self.fzf.execute_fzf(print_col=1) self.assertEqual(result, "hello") - mocked_output.assert_called_once() - mocked_output.return_value = b"" + proc.stdout.read.return_value = b"" self.assertRaises(NoSelectionMade, self.fzf.execute_fzf) - mocked_output.return_value = b"" result = self.fzf.execute_fzf(empty_allow=True) self.assertEqual("", result) - mocked_output.return_value = b"hello" + proc.stdout.read.return_value = b"hello" result = self.fzf.execute_fzf(multi_select=True, print_col=1) self.assertEqual(result, ["hello"]) - mocked_output.return_value = b"hello\nworld" + proc.stdout.read.return_value = b"hello\nworld" result = self.fzf.execute_fzf( multi_select=True, print_col=1, preview="hello", header="foo boo" ) self.assertEqual(result, ["hello", "world"]) - mocked_output.return_value = b"hello world\nfoo boo" + proc.stdout.read.return_value = b"hello world\nfoo boo" result = self.fzf.execute_fzf(multi_select=True, print_col=0) self.assertEqual(result, ["hello world", "foo boo"]) - @patch.object(subprocess, "Popen") - @patch.object(subprocess, "check_output") - def test_check_ctrl_c(self, mocked_output, mocked_popen): - mocked_output.return_value = b"ctrl-c" + @patch("fzfaws.utils.pyfzf.subprocess.Popen") + def test_check_ctrl_c(self, MockedPopen): + proc = MockedPopen() + proc.stdout.read.return_value = b"ctrl-c" self.assertRaises(KeyboardInterrupt, self.fzf.execute_fzf) - mocked_output.return_value = b"hello world" + proc.stdout.read.return_value = b"hello world" try: result = self.fzf.execute_fzf() self.assertEqual(result, "world") From b9decc53cd1c8910633cc93884a1b23ffa26e195 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 29 Jul 2020 10:26:30 +1000 Subject: [PATCH 03/40] feat(lambda): init lambda --- fzfaws/cli.py | 16 +++++--- fzfaws/cloudformation/cloudformation.py | 9 ++++- fzfaws/lambdaf/__init__.py | 1 + fzfaws/lambdaf/lambdaf.py | 54 +++++++++++++++++++++++++ fzfaws/lambdaf/main.py | 18 +++++++++ 5 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 fzfaws/lambdaf/__init__.py create mode 100644 fzfaws/lambdaf/lambdaf.py create mode 100644 fzfaws/lambdaf/main.py diff --git a/fzfaws/cli.py b/fzfaws/cli.py index 01772f8..78cb39e 100644 --- a/fzfaws/cli.py +++ b/fzfaws/cli.py @@ -16,6 +16,7 @@ from fzfaws.cloudformation.main import cloudformation from fzfaws.ec2.main import ec2 from fzfaws.s3.main import s3 +from fzfaws.lambdaf.main import lambdaf from fzfaws.utils import FileLoader, get_default_args from fzfaws.utils.exceptions import InvalidFileType, NoSelectionMade @@ -45,6 +46,7 @@ def main() -> None: subparsers.add_parser("cloudformation") subparsers.add_parser("ec2") subparsers.add_parser("s3") + subparsers.add_parser("lambda") if len(sys.argv) < 2: parser.print_help() @@ -65,12 +67,14 @@ def main() -> None: argument_list = get_default_args(args.subparser_name, sys.argv[2:]) - if args.subparser_name == "cloudformation": - cloudformation(argument_list) - elif args.subparser_name == "ec2": - ec2(argument_list) - elif args.subparser_name == "s3": - s3(argument_list) + actions = { + "cloudformation": cloudformation, + "ec2": ec2, + "s3": s3, + "lambda": lambdaf, + } + + actions[args.subparser_name](argument_list) except InvalidFileType: print("Selected file is not a valid file type") diff --git a/fzfaws/cloudformation/cloudformation.py b/fzfaws/cloudformation/cloudformation.py index 59c92c2..6d4b010 100644 --- a/fzfaws/cloudformation/cloudformation.py +++ b/fzfaws/cloudformation/cloudformation.py @@ -5,8 +5,13 @@ import sys from typing import Any, Callable, Dict, Generator, List, Tuple, Union -from fzfaws.utils import BaseSession, Pyfzf, Spinner, get_confirmation -from fzfaws.utils.util import search_dict_in_list +from fzfaws.utils import ( + BaseSession, + Pyfzf, + Spinner, + get_confirmation, + search_dict_in_list, +) class Cloudformation(BaseSession): diff --git a/fzfaws/lambdaf/__init__.py b/fzfaws/lambdaf/__init__.py new file mode 100644 index 0000000..2b54bc5 --- /dev/null +++ b/fzfaws/lambdaf/__init__.py @@ -0,0 +1 @@ +from .lambdaf import Lambdaf diff --git a/fzfaws/lambdaf/lambdaf.py b/fzfaws/lambdaf/lambdaf.py new file mode 100644 index 0000000..8bd7d59 --- /dev/null +++ b/fzfaws/lambdaf/lambdaf.py @@ -0,0 +1,54 @@ +"""Module contains lambda wrapper class.""" +from typing import Union, Optional +from fzfaws.utils import BaseSession, Spinner, Pyfzf + + +class Lambdaf(BaseSession): + """Lambda wrapper class to interact with boto3.client('lambda'). + + Handles the initialisation of lambda client, selection of lambda + functions. + + :param profile: use a different profile for this opeartion + :type profile: Union[str, bool], optional + :param region: use a different region for this operation + :type region: Union[str, bool], optional + """ + + def __init__( + self, profile: Union[str, bool] = None, region: Union[str, bool] = None + ): + """Construct the instance.""" + super().__init__(profile=profile, region=region, service_name="lambda") + self.function_name: str = "" + + def set_lambdaf( + self, no_progress: bool = False, header: str = "", function_name: str = "" + ) -> None: + """Set the function name for lambda operation. + + :param no_progress: don't show spinner, useful for ls commands + :type no_progress: bool + :param header: header to display in fzf header + :type header: str, optional + :param function_name: the function_name to operate, skip fzf selection + :type function_name: str + """ + if function_name: + self.function_name = function_name + return + + with Spinner.spin( + message="Fetching lambda functions ...", no_progress=no_progress + ): + fzf = Pyfzf() + paginator = self.client.get_paginator("list_functions") + for result in paginator.paginate(): + fzf.process_list( + result.get("Functions", {}), + "FunctionName", + "Runtime", + "Version", + "Description", + ) + self.function_name = str(fzf.execute_fzf(header=header)) diff --git a/fzfaws/lambdaf/main.py b/fzfaws/lambdaf/main.py new file mode 100644 index 0000000..ae545f7 --- /dev/null +++ b/fzfaws/lambdaf/main.py @@ -0,0 +1,18 @@ +"""Contains the entry point for all lambda operations.""" +from fzfaws.lambdaf import Lambdaf +from typing import List, Any + + +def lambdaf(raw_args: List[Any]) -> None: + """Parse arguments and direct traffic to lambda handler, internal use only. + + The raw_args are the processed args through cli.py main function. + It also already contains the user default args so no need to process + default args anymore. + + :param raw_args: list of args to be parsed + :type raw_args: list + """ + + lambdaf = Lambdaf() + lambdaf.set_lambdaf() From a2207b3c1577ec94fa1a42b9b937a3bf25b2a788 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 29 Jul 2020 12:16:19 +1000 Subject: [PATCH 04/40] feat(lambda): invoke synchronously --- fzfaws/lambdaf/invoke_function.py | 65 +++++++++++++++++++++++++++++++ fzfaws/lambdaf/lambdaf.py | 20 +++++----- fzfaws/lambdaf/main.py | 59 ++++++++++++++++++++++++++-- fzfaws/utils/pyfzf.py | 4 +- 4 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 fzfaws/lambdaf/invoke_function.py diff --git a/fzfaws/lambdaf/invoke_function.py b/fzfaws/lambdaf/invoke_function.py new file mode 100644 index 0000000..0ff89c3 --- /dev/null +++ b/fzfaws/lambdaf/invoke_function.py @@ -0,0 +1,65 @@ +"""Module contains functions to handle lambda invokations.""" +from typing import Any, Union, Dict +from fzfaws.lambdaf import Lambdaf +from fzfaws.utils import Spinner +import json +import base64 +from pprint import pprint + + +def invoke_function( + profile: Union[str, bool] = False, + region: Union[str, bool] = False, + asynk: bool = False, + all_version: bool = False, +): + """Invoke the selected lambda function. + + :param profile: use a different profile for this operation + :type profile: Union[str, bool], optional + :param region: use a different region for this operation + :type region: Union[str, bool], optional + :param asynk: invoke function asynchronously, asynk for no conflict with async + :type asynk: bool, optional + :param all_version: list all versions of lambda functions + :type all_version: bool, optional + """ + lambdaf = Lambdaf(profile, region) + lambdaf.set_lambdaf(header="select function to invoke", all_version=all_version) + if not asynk: + invoke_function_sync(lambdaf) + + +def invoke_function_sync(lambdaf: Lambdaf) -> None: + """Invoke the lambda synchronously. + + :param lambdaf: the instance of Lambdaf + :type lambdaf: Lambdaf + """ + function_args: Dict[str, str] = get_function_name(lambdaf.function_detail) + function_args["InvocationType"] = "RequestResponse" + + with Spinner.spin(message="Invoking lambda function ..."): + response = lambdaf.client.invoke(**function_args) + response.pop("ResponseMetadata", None) + response["Payload"] = json.loads(response["Payload"].read().decode("utf-8")) + response["LogResult"] = base64.b64decode(response["LogResult"]).decode("utf-8") + pprint(response) + + +def get_function_name(details: Dict[str, str]) -> Dict[str, str]: + """Get the argument for lambda invoke function. + + :param details: the selected lambda details + :type details: Dict[str, str] + """ + function_args: Dict[str, str] = {} + if details.get("Version", "$LATEST") == "$LATEST": + function_args["FunctionName"] = details["FunctionName"] + else: + function_args["FunctionName"] = "%s:%s" % ( + details["FunctionName"], + details["Version"], + ) + function_args["LogType"] = "Tail" + return function_args diff --git a/fzfaws/lambdaf/lambdaf.py b/fzfaws/lambdaf/lambdaf.py index 8bd7d59..f1d3c21 100644 --- a/fzfaws/lambdaf/lambdaf.py +++ b/fzfaws/lambdaf/lambdaf.py @@ -1,5 +1,5 @@ """Module contains lambda wrapper class.""" -from typing import Union, Optional +from typing import Any, Dict, List, Union, Optional from fzfaws.utils import BaseSession, Spinner, Pyfzf @@ -21,9 +21,10 @@ def __init__( """Construct the instance.""" super().__init__(profile=profile, region=region, service_name="lambda") self.function_name: str = "" + self.function_detail: Dict[str, str] = {} def set_lambdaf( - self, no_progress: bool = False, header: str = "", function_name: str = "" + self, no_progress: bool = False, header: str = "", all_version: bool = False ) -> None: """Set the function name for lambda operation. @@ -31,19 +32,16 @@ def set_lambdaf( :type no_progress: bool :param header: header to display in fzf header :type header: str, optional - :param function_name: the function_name to operate, skip fzf selection - :type function_name: str + :param all_version: list all versions of all functions + :type all_version: bool, optional """ - if function_name: - self.function_name = function_name - return - with Spinner.spin( message="Fetching lambda functions ...", no_progress=no_progress ): fzf = Pyfzf() paginator = self.client.get_paginator("list_functions") - for result in paginator.paginate(): + arguments = {"FunctionVersion": "ALL"} if all_version else {} + for result in paginator.paginate(**arguments): fzf.process_list( result.get("Functions", {}), "FunctionName", @@ -51,4 +49,6 @@ def set_lambdaf( "Version", "Description", ) - self.function_name = str(fzf.execute_fzf(header=header)) + selected_function = fzf.execute_fzf(header=header, print_col=0) + self.function_detail = fzf.format_selected_to_dict(str(selected_function)) + self.function_name = self.function_detail.get("FunctionName", "") diff --git a/fzfaws/lambdaf/main.py b/fzfaws/lambdaf/main.py index ae545f7..405d89d 100644 --- a/fzfaws/lambdaf/main.py +++ b/fzfaws/lambdaf/main.py @@ -1,6 +1,8 @@ """Contains the entry point for all lambda operations.""" -from fzfaws.lambdaf import Lambdaf from typing import List, Any +import argparse +import sys +from fzfaws.lambdaf.invoke_function import invoke_function def lambdaf(raw_args: List[Any]) -> None: @@ -13,6 +15,57 @@ def lambdaf(raw_args: List[Any]) -> None: :param raw_args: list of args to be parsed :type raw_args: list """ + parser = argparse.ArgumentParser( + prog="fzfaws lambda", + description="Perform operations and interact with aws Lambda.", + ) + subparsers = parser.add_subparsers(dest="subparser_name") - lambdaf = Lambdaf() - lambdaf.set_lambdaf() + invoke_cmd = subparsers.add_parser( + "invoke", description="Invoke lambda function synchronously." + ) + invoke_cmd.add_argument( + "-A", + "--all", + action="store_true", + default=False, + help="list all versions and functions", + ) + invoke_cmd.add_argument( + "-a", + "--async", + action="store_true", + default=False, + dest="asynk", + help="invoke function asynchronously", + ) + invoke_cmd.add_argument( + "-R", + "--region", + nargs="?", + action="store", + default=False, + help="choose/specify a region for the operation", + ) + invoke_cmd.add_argument( + "-P", + "--profile", + nargs="?", + action="store", + default=False, + help="choose/specify a profile for the operation", + ) + + args = parser.parse_args(raw_args) + + if not raw_args: + parser.print_help() + sys.exit(0) + + if args.profile == None: + args.profile = True + if args.region == None: + args.region = True + + if args.subparser_name == "invoke": + invoke_function(args.profile, args.region, args.asynk, args.all) diff --git a/fzfaws/utils/pyfzf.py b/fzfaws/utils/pyfzf.py index 6759c09..9255a83 100644 --- a/fzfaws/utils/pyfzf.py +++ b/fzfaws/utils/pyfzf.py @@ -333,7 +333,9 @@ def process_list( self.append_fzf("%s: %s" % (key_name, item.get(key_name))) for arg in arg_keys: self.append_fzf(" | ") - self.append_fzf("%s: %s" % (arg, item.get(arg))) + self.append_fzf( + "%s: %s" % (arg, item.get(arg) if item.get(arg) else None) + ) self.append_fzf("\n") if not self.fzf_string and not empty_allow: raise EmptyList("Result list was empty") From 2f0b971dff8dc6ab804609db9d4c34064a5ba9c5 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 29 Jul 2020 12:45:13 +1000 Subject: [PATCH 05/40] feat(lambda): pretty the output of invoke --- fzfaws/lambdaf/invoke_function.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fzfaws/lambdaf/invoke_function.py b/fzfaws/lambdaf/invoke_function.py index 0ff89c3..0562549 100644 --- a/fzfaws/lambdaf/invoke_function.py +++ b/fzfaws/lambdaf/invoke_function.py @@ -1,10 +1,11 @@ """Module contains functions to handle lambda invokations.""" -from typing import Any, Union, Dict +import base64 +import json +import pprint +from typing import Dict, Union + from fzfaws.lambdaf import Lambdaf from fzfaws.utils import Spinner -import json -import base64 -from pprint import pprint def invoke_function( @@ -43,8 +44,11 @@ def invoke_function_sync(lambdaf: Lambdaf) -> None: response = lambdaf.client.invoke(**function_args) response.pop("ResponseMetadata", None) response["Payload"] = json.loads(response["Payload"].read().decode("utf-8")) - response["LogResult"] = base64.b64decode(response["LogResult"]).decode("utf-8") - pprint(response) + log_result = response.pop("LogResult", None) + log_result = base64.b64decode(log_result).decode("utf-8") + pprint.pprint(log_result) + print(80 * "-") + print(json.dumps(response, indent=4, default=str)) def get_function_name(details: Dict[str, str]) -> Dict[str, str]: From 63aca6666437902d3f031040575f48c533cf6519 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 29 Jul 2020 15:00:01 +1000 Subject: [PATCH 06/40] test(lambda): test the new lambda class --- tests/data/lambda_function.json | 66 +++++++++++++++++++++++++++++++++ tests/lambdaf/__init__.py | 0 tests/lambdaf/test_lambdaf.py | 64 ++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 tests/data/lambda_function.json create mode 100644 tests/lambdaf/__init__.py create mode 100644 tests/lambdaf/test_lambdaf.py diff --git a/tests/data/lambda_function.json b/tests/data/lambda_function.json new file mode 100644 index 0000000..1d6b70d --- /dev/null +++ b/tests/data/lambda_function.json @@ -0,0 +1,66 @@ +[ + { + "ResponseMetadata": { + "RequestId": "e5bcf623-da65-48c6-9013-59e00ad4482a", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "date": "Wed, 29 Jul 2020 04:50:20 GMT", + "content-type": "application/json", + "content-length": "2958", + "connection": "keep-alive", + "x-amzn-requestid": "e5bcf623-da65-48c6-9013-59e00ad4482a" + }, + "RetryAttempts": 0 + }, + "Functions": [ + { + "FunctionName": "Auto-check-drift-LambdaStack-27L1HV-LambdaFunction-11LA8NZ2OZWIM", + "FunctionArn": "arn:aws:lambda:ap-southeast-2:378756445655:function:Auto-check-drift-LambdaStack-27L1HV-LambdaFunction-11LA8NZ2OZWIM", + "Runtime": "python3.8", + "Role": "arn:aws:iam::378756445655:role/Auto-check-drift-lambda-role", + "Handler": "DetectCloudformationDrift.lambda_handler", + "CodeSize": 544, + "Description": "Search through specified regions and init drift detection on cloudformation stacks and also print current drift status", + "Timeout": 60, + "MemorySize": 128, + "LastModified": "2020-04-23T07:52:42.583+0000", + "CodeSha256": "18KZnlzFBCT/0D/RlecUTI/JuIxfZNHtNC9Et54ZK2Q=", + "Version": "$LATEST", + "TracingConfig": { "Mode": "PassThrough" }, + "RevisionId": "e784a6ee-e0f0-4390-af45-0ea2924448fa" + }, + { + "FunctionName": "Auto-stop-ec2-LambdaStack-YUR0KXN6X-LambdaFunction-1Q7UDJ03FVX21", + "FunctionArn": "arn:aws:lambda:ap-southeast-2:378756445655:function:Auto-stop-ec2-LambdaStack-YUR0KXN6X-LambdaFunction-1Q7UDJ03FVX21", + "Runtime": "python3.8", + "Role": "arn:aws:iam::378756445655:role/Auto-stop-ec2-lambda-role", + "Handler": "index.lambda_handler", + "CodeSize": 519, + "Description": "Search through all region and stop all non server ec2 instances", + "Timeout": 60, + "MemorySize": 128, + "LastModified": "2020-03-13T09:55:14.849+0000", + "CodeSha256": "HNfExeUsL8af2pkkThaLHk7rSXZ94s4ZOwJvZ3eNaIw=", + "Version": "$LATEST", + "TracingConfig": { "Mode": "PassThrough" }, + "RevisionId": "9d6653c1-3b0a-4049-b74f-28640f4bc415" + }, + { + "FunctionName": "testing", + "FunctionArn": "arn:aws:lambda:ap-southeast-2:378756445655:function:testing", + "Runtime": "python3.8", + "Role": "arn:aws:iam::378756445655:role/lambda_invoke", + "Handler": "lambda_function.lambda_handler", + "CodeSize": 273, + "Description": "", + "Timeout": 3, + "MemorySize": 128, + "LastModified": "2020-07-29T01:37:53.054+0000", + "CodeSha256": "+X2UokFzBK7oCszeiNg8q4cf3uMIb0+AAIIshxxNjOQ=", + "Version": "$LATEST", + "TracingConfig": { "Mode": "PassThrough" }, + "RevisionId": "0a91548c-0886-42a7-aacb-7dd1f0c9481a" + } + ] + } +] diff --git a/tests/lambdaf/__init__.py b/tests/lambdaf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/lambdaf/test_lambdaf.py b/tests/lambdaf/test_lambdaf.py new file mode 100644 index 0000000..3257fcc --- /dev/null +++ b/tests/lambdaf/test_lambdaf.py @@ -0,0 +1,64 @@ +import json +from fzfaws.utils.pyfzf import Pyfzf +import io +import sys +from pathlib import Path +import unittest +from unittest.mock import ANY, patch +from fzfaws.lambdaf import Lambdaf +from fzfaws.utils import FileLoader +from botocore.paginate import Paginator + + +class TestLambdaf(unittest.TestCase): + def setUp(self): + self.capturedOutput = io.StringIO() + sys.stdout = self.capturedOutput + config_path = Path(__file__).resolve().parent.joinpath("../data/fzfaws.yml") + fileloader = FileLoader(path=str(config_path)) + fileloader.load_config_file() + self.lambdaf = Lambdaf() + + def tearDown(self): + sys.stdout = sys.__stdout__ + + def test_constructor(self): + self.assertEqual(self.lambdaf.profile, "default") + self.assertEqual(self.lambdaf.region, "ap-southeast-2") + self.assertEqual(self.lambdaf.function_name, "") + self.assertEqual(self.lambdaf.function_detail, {}) + + lambdaf = Lambdaf(profile="root", region="us-east-1") + self.assertEqual(lambdaf.profile, "root") + self.assertEqual(lambdaf.region, "us-east-1") + self.assertEqual(lambdaf.function_name, "") + self.assertEqual(lambdaf.function_detail, {}) + + @patch.object(Pyfzf, "execute_fzf") + @patch.object(Paginator, "paginate") + def test_set_lambdaf(self, mocked_pagiantor, mocked_execute): + data = Path(__file__).resolve().parent.joinpath("../data/lambda_function.json") + with data.open("r") as file: + mocked_pagiantor.return_value = json.load(file) + mocked_execute.return_value = "FunctionName: testing | Runtime: python3.8 | Version: $LATEST | Description: None" + self.lambdaf.set_lambdaf() + mocked_execute.assert_called_once_with(header="", print_col=0) + mocked_pagiantor.assert_called_once() + self.assertEqual( + self.lambdaf.function_detail, + { + "FunctionName": "testing", + "Runtime": "python3.8", + "Version": "$LATEST", + "Description": None, + }, + ) + self.assertEqual(self.lambdaf.function_name, "testing") + + self.capturedOutput.truncate(0) + self.capturedOutput.seek(0) + mocked_execute.reset_mock() + mocked_pagiantor.reset_mock() + self.lambdaf.set_lambdaf(no_progress=True, header="hello", all_version=True) + mocked_execute.assert_called_once_with(header="hello", print_col=0) + mocked_pagiantor.assert_called_once_with(ANY, FunctionVersion="ALL") From df0d4037cc1c584c29345c6412723dfa6f057f02 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 09:40:11 +1000 Subject: [PATCH 07/40] test(lambdaf): load config file during test --- tests/lambdaf/test_lambdaf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/lambdaf/test_lambdaf.py b/tests/lambdaf/test_lambdaf.py index 3257fcc..29ec02a 100644 --- a/tests/lambdaf/test_lambdaf.py +++ b/tests/lambdaf/test_lambdaf.py @@ -15,8 +15,8 @@ def setUp(self): self.capturedOutput = io.StringIO() sys.stdout = self.capturedOutput config_path = Path(__file__).resolve().parent.joinpath("../data/fzfaws.yml") - fileloader = FileLoader(path=str(config_path)) - fileloader.load_config_file() + fileloader = FileLoader() + fileloader.load_config_file(config_path=str(config_path)) self.lambdaf = Lambdaf() def tearDown(self): @@ -24,7 +24,7 @@ def tearDown(self): def test_constructor(self): self.assertEqual(self.lambdaf.profile, "default") - self.assertEqual(self.lambdaf.region, "ap-southeast-2") + self.assertEqual(self.lambdaf.region, "us-east-1") self.assertEqual(self.lambdaf.function_name, "") self.assertEqual(self.lambdaf.function_detail, {}) From ce3e71a6cafa1e9840ab0bb4d67f6644f9591f25 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 09:48:12 +1000 Subject: [PATCH 08/40] test(lambdaf): test main function --- tests/lambdaf/test_main.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/lambdaf/test_main.py diff --git a/tests/lambdaf/test_main.py b/tests/lambdaf/test_main.py new file mode 100644 index 0000000..cb2b111 --- /dev/null +++ b/tests/lambdaf/test_main.py @@ -0,0 +1,32 @@ +import io +import sys +import unittest +from unittest.mock import patch +from fzfaws.lambdaf.main import lambdaf + + +class TestLambdafMain(unittest.TestCase): + def setUp(self): + self.capturedOutput = io.StringIO() + sys.stdout = self.capturedOutput + + def tearDown(self): + sys.stdout = sys.__stdout__ + + def test_help(self): + self.assertRaises(SystemExit, lambdaf, ["-h"]) + self.assertRegex(self.capturedOutput.getvalue(), r"usage: fzfaws lambda \[-h\]") + + self.capturedOutput.truncate(0) + self.capturedOutput.seek(0) + self.assertRaises(SystemExit, lambdaf, ["invoke", "-h"]) + self.assertRegex( + self.capturedOutput.getvalue(), r"usage: fzfaws lambda invoke \[-h\]" + ) + + self.assertRaises(SystemExit, lambdaf, []) + + @patch("fzfaws.lambdaf.main.invoke_function") + def test_invoke(self, mocked_lambdaf): + lambdaf(["invoke", "--all", "--async"]) + mocked_lambdaf.assert_called_once_with(False, False, True, True) From c408fe29ca295e88367e93827ccdbba484088cd3 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 10:11:01 +1000 Subject: [PATCH 09/40] test(lambdaf): added test for sync invoke --- tests/lambdaf/test_invoke_function.py | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/lambdaf/test_invoke_function.py diff --git a/tests/lambdaf/test_invoke_function.py b/tests/lambdaf/test_invoke_function.py new file mode 100644 index 0000000..1d975cb --- /dev/null +++ b/tests/lambdaf/test_invoke_function.py @@ -0,0 +1,62 @@ +from fzfaws.lambdaf.lambdaf import Lambdaf +import io +import sys +import unittest +from unittest.mock import patch +from fzfaws.lambdaf.invoke_function import ( + invoke_function, + invoke_function_sync, + get_function_name, +) + + +class TestLambdafInvoke(unittest.TestCase): + def setUp(self): + self.capturedOutput = io.StringIO() + sys.stdout = self.capturedOutput + + def tearDown(self): + sys.stdout = sys.__stdout__ + + @patch("fzfaws.lambdaf.invoke_function.invoke_function_sync") + @patch("fzfaws.lambdaf.invoke_function.Lambdaf") + def test_invoke_function(self, MockedLambdaf, mocked_sync): + lambdaf = MockedLambdaf.return_value + invoke_function(all_version=True) + lambdaf.set_lambdaf.assert_called_once_with( + header="select function to invoke", all_version=True + ) + mocked_sync.assert_called_once() + + invoke_function(all_version=True, profile=True, region="us-east-2") + MockedLambdaf.assert_called_with(True, "us-east-2") + + @patch("fzfaws.lambdaf.invoke_function.base64") + @patch("fzfaws.lambdaf.invoke_function.json") + @patch("fzfaws.lambdaf.invoke_function.Lambdaf") + def test_invoke_function_sync(self, MockedLambdaf, mocked_json, mocked_base64): + lambdaf = MockedLambdaf.return_value + lambdaf.function_detail = { + "FunctionName": "testing", + "Runtime": "python3.8", + "Version": "$LATEST", + "Description": None, + } + invoke_function_sync(lambdaf) + mocked_json.loads.assert_called_once() + mocked_base64.b64decode.assert_called_once() + lambdaf.client.invoke.assert_called_once_with( + FunctionName="testing", InvocationType="RequestResponse", LogType="Tail" + ) + + lambdaf.client.invoke.reset_mock() + lambdaf.function_detail = { + "FunctionName": "testing", + "Runtime": "python3.8", + "Version": "1", + "Description": None, + } + invoke_function_sync(lambdaf) + lambdaf.client.invoke.assert_called_once_with( + FunctionName="testing:1", InvocationType="RequestResponse", LogType="Tail" + ) From 2db9be800fa52827877b7e7c3f7590118c67a4a8 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 10:23:17 +1000 Subject: [PATCH 10/40] feat(lambdaf): support async invoke --- fzfaws/lambdaf/invoke_function.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fzfaws/lambdaf/invoke_function.py b/fzfaws/lambdaf/invoke_function.py index 0562549..ab6ffe2 100644 --- a/fzfaws/lambdaf/invoke_function.py +++ b/fzfaws/lambdaf/invoke_function.py @@ -29,6 +29,22 @@ def invoke_function( lambdaf.set_lambdaf(header="select function to invoke", all_version=all_version) if not asynk: invoke_function_sync(lambdaf) + else: + invoke_function_async(lambdaf) + + +def invoke_function_async(lambdaf: Lambdaf) -> None: + """Invoke the lambda asynchronously. + + :param lambdaf: the instance of Lambdaf + :type lambdaf: Lambdaf + """ + function_args: Dict[str, str] = get_function_name(lambdaf.function_detail) + function_args["InvocationType"] = "Event" + response = lambdaf.client.invoke(**function_args) + response.pop("ResponseMetadata", None) + response.pop("Payload", None) + print(json.dumps(response, indent=4, default=str)) def invoke_function_sync(lambdaf: Lambdaf) -> None: From 292571942ef4e051383eb9c4729e2dd331350628 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 10:27:39 +1000 Subject: [PATCH 11/40] test(lambdaf): added test for async invoke --- tests/lambdaf/test_invoke_function.py | 41 +++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/lambdaf/test_invoke_function.py b/tests/lambdaf/test_invoke_function.py index 1d975cb..40d429c 100644 --- a/tests/lambdaf/test_invoke_function.py +++ b/tests/lambdaf/test_invoke_function.py @@ -6,7 +6,7 @@ from fzfaws.lambdaf.invoke_function import ( invoke_function, invoke_function_sync, - get_function_name, + invoke_function_async, ) @@ -18,19 +18,30 @@ def setUp(self): def tearDown(self): sys.stdout = sys.__stdout__ + @patch("fzfaws.lambdaf.invoke_function.invoke_function_async") @patch("fzfaws.lambdaf.invoke_function.invoke_function_sync") @patch("fzfaws.lambdaf.invoke_function.Lambdaf") - def test_invoke_function(self, MockedLambdaf, mocked_sync): + def test_invoke_function(self, MockedLambdaf, mocked_sync, mocked_async): lambdaf = MockedLambdaf.return_value invoke_function(all_version=True) lambdaf.set_lambdaf.assert_called_once_with( header="select function to invoke", all_version=True ) mocked_sync.assert_called_once() + mocked_async.assert_not_called() invoke_function(all_version=True, profile=True, region="us-east-2") MockedLambdaf.assert_called_with(True, "us-east-2") + lambdaf.set_lambdaf.reset_mock() + mocked_sync.reset_mock() + invoke_function(all_version=True, asynk=True) + lambdaf.set_lambdaf.assert_called_once_with( + header="select function to invoke", all_version=True + ) + mocked_sync.assert_not_called() + mocked_async.assert_called_once() + @patch("fzfaws.lambdaf.invoke_function.base64") @patch("fzfaws.lambdaf.invoke_function.json") @patch("fzfaws.lambdaf.invoke_function.Lambdaf") @@ -60,3 +71,29 @@ def test_invoke_function_sync(self, MockedLambdaf, mocked_json, mocked_base64): lambdaf.client.invoke.assert_called_once_with( FunctionName="testing:1", InvocationType="RequestResponse", LogType="Tail" ) + + @patch("fzfaws.lambdaf.invoke_function.Lambdaf") + def test_invoke_function_async(self, MockedLambdaf): + lambdaf = MockedLambdaf.return_value + lambdaf.function_detail = { + "FunctionName": "testing", + "Runtime": "python3.8", + "Version": "$LATEST", + "Description": None, + } + invoke_function_async(lambdaf) + lambdaf.client.invoke.assert_called_once_with( + FunctionName="testing", InvocationType="Event", LogType="Tail" + ) + + lambdaf.client.invoke.reset_mock() + lambdaf.function_detail = { + "FunctionName": "testing", + "Runtime": "python3.8", + "Version": "5", + "Description": None, + } + invoke_function_async(lambdaf) + lambdaf.client.invoke.assert_called_once_with( + FunctionName="testing:5", InvocationType="Event", LogType="Tail" + ) From c6fe46a7720da87ea48f26ea5459e82d3aab220d Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 11:10:28 +1000 Subject: [PATCH 12/40] feat(lambdaf): accept payload for invoke action --- fzfaws/lambdaf/invoke_function.py | 57 ++++++++++++++++++++++++------- fzfaws/lambdaf/main.py | 21 +++++++++++- fzfaws/utils/pyfzf.py | 5 +++ 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/fzfaws/lambdaf/invoke_function.py b/fzfaws/lambdaf/invoke_function.py index ab6ffe2..4c92ec9 100644 --- a/fzfaws/lambdaf/invoke_function.py +++ b/fzfaws/lambdaf/invoke_function.py @@ -1,8 +1,10 @@ """Module contains functions to handle lambda invokations.""" import base64 +from fzfaws.utils.pyfzf import Pyfzf import json import pprint -from typing import Dict, Union +from typing import Any, Dict, Union +from pathlib import Path from fzfaws.lambdaf import Lambdaf from fzfaws.utils import Spinner @@ -13,6 +15,8 @@ def invoke_function( region: Union[str, bool] = False, asynk: bool = False, all_version: bool = False, + payload: Union[str, bool] = False, + root: bool = False, ): """Invoke the selected lambda function. @@ -24,36 +28,60 @@ def invoke_function( :type asynk: bool, optional :param all_version: list all versions of lambda functions :type all_version: bool, optional + :param payload: a path to json file to provide lambda as input + :type payload: Union[str, bool] + :param root: search from home directory when payload = True + :type root: bool, optional """ lambdaf = Lambdaf(profile, region) lambdaf.set_lambdaf(header="select function to invoke", all_version=all_version) + payload_path: str = "" + if type(payload) == str: + payload_path = str(payload) + elif type(payload) == bool and payload == True: + fzf = Pyfzf() + payload_path = str( + fzf.get_local_file( + json=True, + header="select a json file as payload", + search_from_root=root, + ) + ) + if not asynk: - invoke_function_sync(lambdaf) + invoke_function_sync(lambdaf, payload_path) else: - invoke_function_async(lambdaf) + invoke_function_async(lambdaf, payload_path) -def invoke_function_async(lambdaf: Lambdaf) -> None: +def invoke_function_async(lambdaf: Lambdaf, payload_path: str = "") -> None: """Invoke the lambda asynchronously. - :param lambdaf: the instance of Lambdaf :type lambdaf: Lambdaf + :param payload_path: path to lambda function payload + :type payload_path: str, optional """ - function_args: Dict[str, str] = get_function_name(lambdaf.function_detail) + function_args: Dict[str, Any] = get_function_args( + lambdaf.function_detail, payload_path + ) function_args["InvocationType"] = "Event" response = lambdaf.client.invoke(**function_args) response.pop("ResponseMetadata", None) + response.pop("ResponseMetadata", None) response.pop("Payload", None) print(json.dumps(response, indent=4, default=str)) -def invoke_function_sync(lambdaf: Lambdaf) -> None: +def invoke_function_sync(lambdaf: Lambdaf, payload_path: str = "") -> None: """Invoke the lambda synchronously. - :param lambdaf: the instance of Lambdaf :type lambdaf: Lambdaf + :param payload_path: path to lambda function payload + :type payload_path: str, optional """ - function_args: Dict[str, str] = get_function_name(lambdaf.function_detail) + function_args: Dict[str, Any] = get_function_args( + lambdaf.function_detail, payload_path + ) function_args["InvocationType"] = "RequestResponse" with Spinner.spin(message="Invoking lambda function ..."): @@ -63,17 +91,20 @@ def invoke_function_sync(lambdaf: Lambdaf) -> None: log_result = response.pop("LogResult", None) log_result = base64.b64decode(log_result).decode("utf-8") pprint.pprint(log_result) - print(80 * "-") print(json.dumps(response, indent=4, default=str)) -def get_function_name(details: Dict[str, str]) -> Dict[str, str]: +def get_function_args( + details: Dict[str, str], payload_path: str = "" +) -> Dict[str, Any]: """Get the argument for lambda invoke function. :param details: the selected lambda details :type details: Dict[str, str] + :param payload_path: path to lambda function payload json file + :type payload_path: str, optional """ - function_args: Dict[str, str] = {} + function_args: Dict[str, Any] = {} if details.get("Version", "$LATEST") == "$LATEST": function_args["FunctionName"] = details["FunctionName"] else: @@ -82,4 +113,6 @@ def get_function_name(details: Dict[str, str]) -> Dict[str, str]: details["Version"], ) function_args["LogType"] = "Tail" + if payload_path: + function_args["Payload"] = Path(payload_path).read_bytes() return function_args diff --git a/fzfaws/lambdaf/main.py b/fzfaws/lambdaf/main.py index 405d89d..5c29619 100644 --- a/fzfaws/lambdaf/main.py +++ b/fzfaws/lambdaf/main.py @@ -24,6 +24,21 @@ def lambdaf(raw_args: List[Any]) -> None: invoke_cmd = subparsers.add_parser( "invoke", description="Invoke lambda function synchronously." ) + invoke_cmd.add_argument( + "-p", + "--payload", + action="store", + nargs="?", + default=False, + help="specify a json file to provide to lambda as input", + ) + invoke_cmd.add_argument( + "-r", + "--root", + action="store_true", + default=False, + help="search file from home directory when using --payload flag", + ) invoke_cmd.add_argument( "-A", "--all", @@ -68,4 +83,8 @@ def lambdaf(raw_args: List[Any]) -> None: args.region = True if args.subparser_name == "invoke": - invoke_function(args.profile, args.region, args.asynk, args.all) + if args.payload == None: + args.payload = True + invoke_function( + args.profile, args.region, args.asynk, args.all, args.payload, args.root + ) diff --git a/fzfaws/utils/pyfzf.py b/fzfaws/utils/pyfzf.py index 9255a83..295e3f2 100644 --- a/fzfaws/utils/pyfzf.py +++ b/fzfaws/utils/pyfzf.py @@ -173,6 +173,7 @@ def get_local_file( empty_allow: bool = False, multi_select: bool = False, header: Optional[str] = None, + json: bool = False, ) -> Union[List[Any], List[str], str]: """Get local files through fzf. @@ -212,6 +213,8 @@ def get_local_file( cmd = "echo \033[33m./\033[0m; fd --type d" elif cloudformation: cmd = "fd --type f --regex '(yaml|yml|json)$'" + elif json: + cmd = "fd --type f --regex 'json$'" else: cmd = "fd --type f" if hidden: @@ -222,6 +225,8 @@ def get_local_file( cmd = "echo \033[33m./\033[0m; find * -type d" elif cloudformation: cmd = 'find * -type f -name "*.json" -o -name "*.yaml" -o -name "*.yml"' + elif json: + cmd = 'find * -type f -name "*.json"' else: cmd = "find * -type f" From 5fb64bcbbf43c09fbecd2ef8d81817ccb9e3d34c Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 11:13:32 +1000 Subject: [PATCH 13/40] test(pyfzf): update pyfzf test to include the json param --- tests/utils/test_pyfzf.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/utils/test_pyfzf.py b/tests/utils/test_pyfzf.py index ba62e17..8c7f782 100644 --- a/tests/utils/test_pyfzf.py +++ b/tests/utils/test_pyfzf.py @@ -151,6 +151,11 @@ def test_get_local_file(self, mocked_output, mocked_popen, mocked_check): stdout=ANY, ) + result = self.fzf.get_local_file(json=True) + mocked_popen.assert_called_with( + 'find * -type f -name "*.json"', shell=True, stderr=ANY, stdout=ANY, + ) + mocked_output.reset_mock() mocked_check.return_value = True result = self.fzf.get_local_file(cloudformation=True, header="hello") @@ -162,6 +167,11 @@ def test_get_local_file(self, mocked_output, mocked_popen, mocked_check): ) mocked_output.assert_called_once() + result = self.fzf.get_local_file(json=True, header="hello") + mocked_popen.assert_called_with( + "fd --type f --regex 'json$'", shell=True, stderr=ANY, stdout=ANY, + ) + result = self.fzf.get_local_file(directory=True) mocked_popen.assert_called_with( "echo \033[33m./\033[0m; fd --type d", shell=True, stderr=ANY, stdout=ANY, From 3bd5941a40371a7714d1d3fae4f1d485e373af69 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 11:31:12 +1000 Subject: [PATCH 14/40] test(lambdaf): include payload in test --- tests/lambdaf/test_invoke_function.py | 49 +++++++++++++++++++++++++-- tests/lambdaf/test_main.py | 8 ++++- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/tests/lambdaf/test_invoke_function.py b/tests/lambdaf/test_invoke_function.py index 40d429c..7173ad1 100644 --- a/tests/lambdaf/test_invoke_function.py +++ b/tests/lambdaf/test_invoke_function.py @@ -1,8 +1,8 @@ -from fzfaws.lambdaf.lambdaf import Lambdaf import io +from pathlib import Path import sys import unittest -from unittest.mock import patch +from unittest.mock import ANY, patch from fzfaws.lambdaf.invoke_function import ( invoke_function, invoke_function_sync, @@ -18,10 +18,11 @@ def setUp(self): def tearDown(self): sys.stdout = sys.__stdout__ + @patch("fzfaws.lambdaf.invoke_function.Pyfzf") @patch("fzfaws.lambdaf.invoke_function.invoke_function_async") @patch("fzfaws.lambdaf.invoke_function.invoke_function_sync") @patch("fzfaws.lambdaf.invoke_function.Lambdaf") - def test_invoke_function(self, MockedLambdaf, mocked_sync, mocked_async): + def test_invoke_function(self, MockedLambdaf, mocked_sync, mocked_async, MockedFZF): lambdaf = MockedLambdaf.return_value invoke_function(all_version=True) lambdaf.set_lambdaf.assert_called_once_with( @@ -42,6 +43,20 @@ def test_invoke_function(self, MockedLambdaf, mocked_sync, mocked_async): mocked_sync.assert_not_called() mocked_async.assert_called_once() + mocked_sync.reset_mock() + fzf = MockedFZF.return_value + invoke_function(payload=True, root=True) + fzf.get_local_file.assert_called_once_with( + json=True, header="select a json file as payload", search_from_root=True + ) + mocked_sync.assert_called_once() + + mocked_sync.reset_mock() + fzf.get_local_file.reset_mock() + invoke_function(payload="./test.json", root=True) + fzf.get_local_file.assert_not_called() + mocked_sync.assert_called_once_with(lambdaf, "./test.json") + @patch("fzfaws.lambdaf.invoke_function.base64") @patch("fzfaws.lambdaf.invoke_function.json") @patch("fzfaws.lambdaf.invoke_function.Lambdaf") @@ -72,6 +87,20 @@ def test_invoke_function_sync(self, MockedLambdaf, mocked_json, mocked_base64): FunctionName="testing:1", InvocationType="RequestResponse", LogType="Tail" ) + lambdaf.client.invoke.reset_mock() + invoke_function_sync( + lambdaf, + payload_path=str( + Path(__file__).resolve().parent.joinpath("../data/ec2_az.json") + ), + ) + lambdaf.client.invoke.assert_called_once_with( + FunctionName="testing:1", + InvocationType="RequestResponse", + LogType="Tail", + Payload=ANY, + ) + @patch("fzfaws.lambdaf.invoke_function.Lambdaf") def test_invoke_function_async(self, MockedLambdaf): lambdaf = MockedLambdaf.return_value @@ -97,3 +126,17 @@ def test_invoke_function_async(self, MockedLambdaf): lambdaf.client.invoke.assert_called_once_with( FunctionName="testing:5", InvocationType="Event", LogType="Tail" ) + + lambdaf.client.invoke.reset_mock() + invoke_function_async( + lambdaf, + payload_path=str( + Path(__file__).resolve().parent.joinpath("../data/ec2_az.json") + ), + ) + lambdaf.client.invoke.assert_called_once_with( + FunctionName="testing:5", + InvocationType="Event", + LogType="Tail", + Payload=ANY, + ) diff --git a/tests/lambdaf/test_main.py b/tests/lambdaf/test_main.py index cb2b111..933e66c 100644 --- a/tests/lambdaf/test_main.py +++ b/tests/lambdaf/test_main.py @@ -29,4 +29,10 @@ def test_help(self): @patch("fzfaws.lambdaf.main.invoke_function") def test_invoke(self, mocked_lambdaf): lambdaf(["invoke", "--all", "--async"]) - mocked_lambdaf.assert_called_once_with(False, False, True, True) + mocked_lambdaf.assert_called_once_with(False, False, True, True, False, False) + + mocked_lambdaf.reset_mock() + lambdaf(["invoke", "--payload", "./test.json"]) + mocked_lambdaf.assert_called_once_with( + False, False, False, False, "./test.json", False + ) From 142d64844be133d51a73a99de680bbd4b2e3a8d4 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 12:12:45 +1000 Subject: [PATCH 15/40] feat: migrate prompt to PyInquirer --- fzfaws/utils/util.py | 12 +++++++----- tests/utils/test_util.py | 9 +++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index 0d0f141..b7934ce 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -1,6 +1,7 @@ """This module contains some common helper functions.""" import os from typing import Any, Dict, Generator, List, Optional, Union +from PyInquirer import prompt def remove_dict_from_list( @@ -64,7 +65,7 @@ def check_dict_value_in_list( return False -def get_confirmation(message: str) -> bool: +def get_confirmation(message: str = "Confirm?") -> bool: """Get user confirmation. :param message: message to ask @@ -72,10 +73,11 @@ def get_confirmation(message: str) -> bool: :return: user confirm status :rtype: bool """ - confirm = None - while confirm != "y" and confirm != "n": - confirm = input("%s(y/n): " % message).lower() - return True if confirm == "y" else False + questions = [ + {"type": "confirm", "message": message, "name": "continue", "default": False} + ] + answers = prompt(questions) + return answers.get("continue", False) def get_name_tag(response_item: Dict[str, Any]) -> Optional[str]: diff --git a/tests/utils/test_util.py b/tests/utils/test_util.py index 43bf52c..35367b5 100644 --- a/tests/utils/test_util.py +++ b/tests/utils/test_util.py @@ -42,12 +42,13 @@ def test_check_dict_value_in_list(self): result = check_dict_value_in_list("yes", test_list, "yes") self.assertFalse(result) - @patch("builtins.input") - def test_get_confirmation(self, mocked_input): - mocked_input.return_value = "y" + @patch("fzfaws.utils.util.prompt") + def test_get_confirmation(self, mocked_prompt): + mocked_prompt.return_value = {"continue": True} response = get_confirmation("Confirm?") self.assertTrue(response) - mocked_input.return_value = "n" + + mocked_prompt.return_value = {"continue": False} response = get_confirmation("Confirm?") self.assertFalse(response) From a147c12dc52e7bbe050360fb53a18a4e3f5827ea Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 12:29:21 +1000 Subject: [PATCH 16/40] fix(cloudformation): don't include yaml in stack policy finding --- fzfaws/cloudformation/helper/cloudformationargs.py | 2 +- tests/cloudformation/test_cloudformationargs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fzfaws/cloudformation/helper/cloudformationargs.py b/fzfaws/cloudformation/helper/cloudformationargs.py index 6543dd7..dca10d2 100644 --- a/fzfaws/cloudformation/helper/cloudformationargs.py +++ b/fzfaws/cloudformation/helper/cloudformationargs.py @@ -234,7 +234,7 @@ def _set_policy(self, update: bool = False, search_from_root: bool = False) -> N file_path: str = str( fzf.get_local_file( search_from_root=search_from_root, - cloudformation=True, + json=True, empty_allow=True, header="select the policy document you would like to use", ) diff --git a/tests/cloudformation/test_cloudformationargs.py b/tests/cloudformation/test_cloudformationargs.py index 41532eb..1f396a0 100644 --- a/tests/cloudformation/test_cloudformationargs.py +++ b/tests/cloudformation/test_cloudformationargs.py @@ -262,7 +262,7 @@ def test__set_policy(self, mocked_file): ) mocked_file.assert_called_once_with( search_from_root=False, - cloudformation=True, + json=True, empty_allow=True, header="select the policy document you would like to use", ) @@ -289,7 +289,7 @@ def test__set_policy(self, mocked_file): ) mocked_file.assert_called_once_with( search_from_root=True, - cloudformation=True, + json=True, empty_allow=True, header="select the policy document you would like to use", ) From e9d7a71bc56e1642f68ab177d562f3cc3519eb61 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 12:49:51 +1000 Subject: [PATCH 17/40] fix(util): properly detect keybord interrupt --- fzfaws/utils/util.py | 2 ++ tests/utils/test_util.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index b7934ce..d7551dd 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -77,6 +77,8 @@ def get_confirmation(message: str = "Confirm?") -> bool: {"type": "confirm", "message": message, "name": "continue", "default": False} ] answers = prompt(questions) + if not answers: + raise KeyboardInterrupt return answers.get("continue", False) diff --git a/tests/utils/test_util.py b/tests/utils/test_util.py index 35367b5..6bade97 100644 --- a/tests/utils/test_util.py +++ b/tests/utils/test_util.py @@ -52,6 +52,9 @@ def test_get_confirmation(self, mocked_prompt): response = get_confirmation("Confirm?") self.assertFalse(response) + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, get_confirmation) + def test_get_name_tag(self): data_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../data/ec2_instance.json" From b71d8059bb349a02a5defba5085ba98137144a0b Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 12:57:22 +1000 Subject: [PATCH 18/40] feat(s3): migrate path selection to PyInquirer --- fzfaws/s3/s3.py | 41 +++++++++++++++++--------------- tests/s3/test_s3.py | 57 +++++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 39 deletions(-) diff --git a/fzfaws/s3/s3.py b/fzfaws/s3/s3.py index b2edc44..ef5b26a 100644 --- a/fzfaws/s3/s3.py +++ b/fzfaws/s3/s3.py @@ -3,10 +3,9 @@ import re import itertools from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union +from PyInquirer import prompt -from botocore.exceptions import ClientError - -from fzfaws.utils import BaseSession, FileLoader, Pyfzf, Spinner, get_confirmation +from fzfaws.utils import BaseSession, FileLoader, Pyfzf, Spinner from fzfaws.utils.exceptions import ( InvalidFileType, InvalidS3PathPattern, @@ -414,29 +413,33 @@ def _validate_input_path( return (None, None) def _get_path_option(self, download: bool = False) -> str: - """Pop fzf for user to select what to do with the path. + """Prompt user to select a path option. :param download: if not download, insert append option :type download: bool, optional :return: selected option :rtype: str """ - fzf = Pyfzf() - fzf.append_fzf("root: operate on the root level of the bucket\n") - fzf.append_fzf("interactively: interactively select a path through s3\n") - fzf.append_fzf("input: manully input the path/name\n") + choices: List[str] = [ + "root: use the root level of the bucket", + "interactively: select a path through fzf", + "input: enter the path/name", + ] if not download: - fzf.append_fzf( - "append: interactively select a path and then input new path/name to append" - ) - selected_option = str( - fzf.execute_fzf( - print_col=1, - header="select which level of the bucket would you like to operate in", - delimiter=": ", - ) - ) - return selected_option + choices.append("append: select a path and then enter path/name to append") + questions: List[Dict[str, Any]] = [ + { + "type": "rawlist", + "name": "selected_option", + "message": "Select which level of the bucket would you like to operate in", + "choices": choices, + "filter": lambda val: val.split(": ")[0], + }, + ] + result = prompt(questions) + if not result: + raise KeyboardInterrupt + return result.get("selected_option", "input") def _version_generator( self, versions: List[dict], markers: List[dict], non_current: bool, delete: bool diff --git a/tests/s3/test_s3.py b/tests/s3/test_s3.py index 7324aa3..5e68fb1 100644 --- a/tests/s3/test_s3.py +++ b/tests/s3/test_s3.py @@ -4,7 +4,7 @@ from pathlib import Path import sys import unittest -from unittest.mock import PropertyMock, call, patch +from unittest.mock import ANY, PropertyMock, call, patch import boto3 from botocore.paginate import Paginator @@ -174,7 +174,6 @@ def test_validate_input_path(self): self.assertEqual(result, None) self.assertEqual(match, None) - @patch("fzfaws.s3.s3.get_confirmation") @patch.object(Pyfzf, "execute_fzf") @patch.object(Pyfzf, "append_fzf") @patch.object(Paginator, "paginate") @@ -187,7 +186,6 @@ def test_set_s3_path( mocked_paginator, mocked_append, mocked_execute, - mocked_confirmation, ): # input self.s3.bucket_name = "kazhala-version-testing" @@ -217,7 +215,6 @@ def test_set_s3_path( response = json.load(file) mocked_paginator.return_value = response mocked_execute.return_value = "./" - mocked_confirmation.return_value = True self.s3.set_s3_path() mocked_execute.assert_called_with( print_col=0, @@ -235,7 +232,6 @@ def test_set_s3_path( self.s3.bucket_name = "kazhala-version-testing" self.s3.path_list = ["hello/"] mocked_execute.return_value = "./" - mocked_confirmation.return_value = True self.s3.set_s3_path() mocked_execute.assert_called_with( print_col=0, @@ -256,7 +252,6 @@ def test_set_s3_path( mocked_option.return_value = "append" mocked_paginator.return_value = response mocked_execute.return_value = "./" - mocked_confirmation.return_value = True mocked_input.return_value = "newpath/" self.s3.set_s3_path() mocked_execute.assert_called_with( @@ -283,7 +278,6 @@ def test_set_s3_path( mocked_option.return_value = "append" mocked_paginator.return_value = response mocked_execute.return_value = "./" - mocked_confirmation.return_value = True mocked_input.return_value = "obj1" self.s3.set_s3_path() mocked_execute.assert_called_with( @@ -579,24 +573,47 @@ def test_get_s3_destination_key(self): ) self.assertEqual(result, "hello/tmp/hello.txt") - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test_get_path_option(self, mocked_append, mocked_execute): - mocked_execute.return_value = "root" + @patch("fzfaws.s3.s3.prompt") + def test_get_path_option(self, mocked_prompt): + mocked_prompt.return_value = {"selected_option": "root"} result = self.s3._get_path_option() self.assertEqual(result, "root") - mocked_append.assert_called_with( - "append: interactively select a path and then input new path/name to append" + mocked_prompt.assert_called_with( + [ + { + "type": "rawlist", + "name": "selected_option", + "message": "Select which level of the bucket would you like to operate in", + "choices": [ + "root: use the root level of the bucket", + "interactively: select a path through fzf", + "input: enter the path/name", + "append: select a path and then enter path/name to append", + ], + "filter": ANY, + } + ] ) - mocked_execute.return_value = "append" - result = self.s3._get_path_option() - self.assertEqual(result, "append") - - mocked_execute.return_value = "root" result = self.s3._get_path_option(download=True) - self.assertEqual(result, "root") - mocked_append.assert_called_with("input: manully input the path/name\n") + mocked_prompt.assert_called_with( + [ + { + "type": "rawlist", + "name": "selected_option", + "message": "Select which level of the bucket would you like to operate in", + "choices": [ + "root: use the root level of the bucket", + "interactively: select a path through fzf", + "input: enter the path/name", + ], + "filter": ANY, + } + ] + ) + + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.s3._get_path_option) def test_version_gernator(self): data_path = os.path.join( From 26cdd1b26cbdf7bae6aec9dc76e28f352ee12186 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 14:39:54 +1000 Subject: [PATCH 19/40] feat: custome PyInquirer style --- fzfaws/s3/s3.py | 4 ++-- fzfaws/utils/util.py | 15 +++++++++++++-- tests/s3/test_s3.py | 6 ++++-- tests/utils/test_util.py | 13 ++++++++++++- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/fzfaws/s3/s3.py b/fzfaws/s3/s3.py index ef5b26a..7af2f48 100644 --- a/fzfaws/s3/s3.py +++ b/fzfaws/s3/s3.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union from PyInquirer import prompt -from fzfaws.utils import BaseSession, FileLoader, Pyfzf, Spinner +from fzfaws.utils import BaseSession, FileLoader, Pyfzf, Spinner, prompt_style from fzfaws.utils.exceptions import ( InvalidFileType, InvalidS3PathPattern, @@ -436,7 +436,7 @@ def _get_path_option(self, download: bool = False) -> str: "filter": lambda val: val.split(": ")[0], }, ] - result = prompt(questions) + result = prompt(questions, style=prompt_style) if not result: raise KeyboardInterrupt return result.get("selected_option", "input") diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index d7551dd..614a349 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -1,7 +1,18 @@ """This module contains some common helper functions.""" import os from typing import Any, Dict, Generator, List, Optional, Union -from PyInquirer import prompt +from PyInquirer import prompt, style_from_dict, Token + +prompt_style = style_from_dict( + { + Token.QuestionMark: "#E5C07B bold", + Token.Selected: "#61AFEF bold", + Token.Instruction: "", + Token.Answer: "#61AFEF bold", + Token.Question: "", + Token.Pointer: "#C678DD bold", + } +) def remove_dict_from_list( @@ -76,7 +87,7 @@ def get_confirmation(message: str = "Confirm?") -> bool: questions = [ {"type": "confirm", "message": message, "name": "continue", "default": False} ] - answers = prompt(questions) + answers = prompt(questions, style=prompt_style) if not answers: raise KeyboardInterrupt return answers.get("continue", False) diff --git a/tests/s3/test_s3.py b/tests/s3/test_s3.py index 5e68fb1..2009714 100644 --- a/tests/s3/test_s3.py +++ b/tests/s3/test_s3.py @@ -592,7 +592,8 @@ def test_get_path_option(self, mocked_prompt): ], "filter": ANY, } - ] + ], + style=ANY, ) result = self.s3._get_path_option(download=True) @@ -609,7 +610,8 @@ def test_get_path_option(self, mocked_prompt): ], "filter": ANY, } - ] + ], + style=ANY, ) mocked_prompt.return_value = {} diff --git a/tests/utils/test_util.py b/tests/utils/test_util.py index 6bade97..0c64af6 100644 --- a/tests/utils/test_util.py +++ b/tests/utils/test_util.py @@ -1,7 +1,7 @@ import os import json import unittest -from unittest.mock import patch +from unittest.mock import ANY, patch from fzfaws.utils import ( check_dict_value_in_list, get_confirmation, @@ -47,6 +47,17 @@ def test_get_confirmation(self, mocked_prompt): mocked_prompt.return_value = {"continue": True} response = get_confirmation("Confirm?") self.assertTrue(response) + mocked_prompt.assert_called_with( + [ + { + "type": "confirm", + "message": "Confirm?", + "name": "continue", + "default": False, + } + ], + style=ANY, + ) mocked_prompt.return_value = {"continue": False} response = get_confirmation("Confirm?") From 2a78382d552c3e3f556ef0ef8895fdf2a8dcdf1b Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 15:00:44 +1000 Subject: [PATCH 20/40] feat(s3): migrate input prompt to PyInquirer --- fzfaws/s3/s3.py | 16 +++++++++++++--- fzfaws/utils/util.py | 9 +++++---- tests/s3/test_s3.py | 10 +++++----- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/fzfaws/s3/s3.py b/fzfaws/s3/s3.py index 7af2f48..f6eeffc 100644 --- a/fzfaws/s3/s3.py +++ b/fzfaws/s3/s3.py @@ -93,11 +93,15 @@ def set_s3_path(self, download: bool = False) -> None: :raises NoSelectionMade: when user did not make a bucket selection, exit """ selected_option = self._get_path_option(download=download) + questions: List[Dict[str, str]] = [{"type": "input", "name": "s3_path",}] if selected_option == "input": - self.path_list[0] = input("Input the path(newname or newpath/): ") + questions[0]["message"] = "Input the path(newname or newpath/)" + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + self.path_list[0] = result.get("s3_path", "") elif selected_option == "root": - # print("S3 file path is set to root") pass elif selected_option == "append" or selected_option == "interactively": paginator = self.client.get_paginator("list_objects") @@ -148,7 +152,13 @@ def set_s3_path(self, download: bool = False) -> None: print( "Current PWD is s3://%s/%s" % (self.bucket_name, self.path_list[0]) ) - new_path = input("Input the new path to append(newname or newpath/): ") + questions[0][ + "message" + ] = "Input the new path to append(newname or newpath/)" + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + new_path = result.get("s3_path", "") self.path_list[0] += new_path print( "S3 file path is set to %s" diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index 614a349..931a4c8 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -5,12 +5,13 @@ prompt_style = style_from_dict( { - Token.QuestionMark: "#E5C07B bold", + Token.Separator: "#6C6C6C", + Token.QuestionMark: "#5F819D", Token.Selected: "#61AFEF bold", Token.Instruction: "", - Token.Answer: "#61AFEF bold", - Token.Question: "", - Token.Pointer: "#C678DD bold", + Token.Answer: "#FF9D00 bold", + Token.Question: "bold", + Token.Pointer: "#FF9D00 bold", } ) diff --git a/tests/s3/test_s3.py b/tests/s3/test_s3.py index 2009714..02a9a19 100644 --- a/tests/s3/test_s3.py +++ b/tests/s3/test_s3.py @@ -177,12 +177,12 @@ def test_validate_input_path(self): @patch.object(Pyfzf, "execute_fzf") @patch.object(Pyfzf, "append_fzf") @patch.object(Paginator, "paginate") - @patch("builtins.input") + @patch("fzfaws.s3.s3.prompt") @patch.object(S3, "_get_path_option") def test_set_s3_path( self, mocked_option, - mocked_input, + mocked_prompt, mocked_paginator, mocked_append, mocked_execute, @@ -190,7 +190,7 @@ def test_set_s3_path( # input self.s3.bucket_name = "kazhala-version-testing" mocked_option.return_value = "input" - mocked_input.return_value = "hello" + mocked_prompt.return_value = {"s3_path": "hello"} self.s3.set_s3_path() self.assertEqual(self.s3.path_list[0], "hello") @@ -252,7 +252,7 @@ def test_set_s3_path( mocked_option.return_value = "append" mocked_paginator.return_value = response mocked_execute.return_value = "./" - mocked_input.return_value = "newpath/" + mocked_prompt.return_value = {"s3_path": "newpath/"} self.s3.set_s3_path() mocked_execute.assert_called_with( print_col=0, @@ -278,7 +278,7 @@ def test_set_s3_path( mocked_option.return_value = "append" mocked_paginator.return_value = response mocked_execute.return_value = "./" - mocked_input.return_value = "obj1" + mocked_prompt.return_value = {"s3_path": "obj1"} self.s3.set_s3_path() mocked_execute.assert_called_with( print_col=0, From 400431e1097b320342f9af8d540f0011f01e5398 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Fri, 31 Jul 2020 15:11:28 +1000 Subject: [PATCH 21/40] fix: fix requirements.txt and docstyle --- fzfaws/lambdaf/invoke_function.py | 2 ++ requirements.txt | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fzfaws/lambdaf/invoke_function.py b/fzfaws/lambdaf/invoke_function.py index 4c92ec9..1079714 100644 --- a/fzfaws/lambdaf/invoke_function.py +++ b/fzfaws/lambdaf/invoke_function.py @@ -56,6 +56,7 @@ def invoke_function( def invoke_function_async(lambdaf: Lambdaf, payload_path: str = "") -> None: """Invoke the lambda asynchronously. + :param lambdaf: the instance of Lambdaf :type lambdaf: Lambdaf :param payload_path: path to lambda function payload @@ -74,6 +75,7 @@ def invoke_function_async(lambdaf: Lambdaf, payload_path: str = "") -> None: def invoke_function_sync(lambdaf: Lambdaf, payload_path: str = "") -> None: """Invoke the lambda synchronously. + :param lambdaf: the instance of Lambdaf :type lambdaf: Lambdaf :param payload_path: path to lambda function payload diff --git a/requirements.txt b/requirements.txt index c527718..8abbc34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,14 @@ -boto3==1.14.20 -botocore==1.17.20 +boto3==1.14.32 +botocore==1.17.32 docutils==0.15.2 jmespath==0.10.0 +prompt-toolkit==1.0.14 +Pygments==2.6.1 +PyInquirer==1.0.3 python-dateutil==2.8.1 PyYAML==5.3.1 +regex==2020.7.14 s3transfer==0.3.3 six==1.15.0 -urllib3==1.25.9 +urllib3==1.25.10 +wcwidth==0.2.5 From 9f640450a92352cc6569c8ad1c3e62fd18313228 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 5 Aug 2020 10:20:00 +1000 Subject: [PATCH 22/40] fix: fix the incorrect wording in pyfzf header --- fzfaws/utils/pyfzf.py | 4 +++- requirements.txt | 13 +------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/fzfaws/utils/pyfzf.py b/fzfaws/utils/pyfzf.py index 295e3f2..ebde6f9 100644 --- a/fzfaws/utils/pyfzf.py +++ b/fzfaws/utils/pyfzf.py @@ -203,8 +203,10 @@ def get_local_file( if search_from_root: home_path = os.path.expanduser("~") os.chdir(home_path) - if not header and directory: + if not header and directory and not search_from_root: header = r"select ./ will use current directory" + elif not header and directory and search_from_root: + header = r"select ./ will use the home directory" cmd: str = "" diff --git a/requirements.txt b/requirements.txt index 8abbc34..ac55ab3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,3 @@ -boto3==1.14.32 -botocore==1.17.32 -docutils==0.15.2 -jmespath==0.10.0 -prompt-toolkit==1.0.14 -Pygments==2.6.1 +boto3==1.14.35 PyInquirer==1.0.3 -python-dateutil==2.8.1 PyYAML==5.3.1 -regex==2020.7.14 -s3transfer==0.3.3 -six==1.15.0 -urllib3==1.25.10 -wcwidth==0.2.5 From c61f0454ad63cc482425cadd578662a3a9dbc13b Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 5 Aug 2020 12:30:32 +1000 Subject: [PATCH 23/40] feat: start refactor s3args to use PyInquirer --- fzfaws/s3/helper/s3args.py | 242 +++++++++++++++++++++++-------------- fzfaws/s3/s3.py | 4 +- fzfaws/utils/util.py | 25 +++- tests/s3/test_s3.py | 8 +- 4 files changed, 178 insertions(+), 101 deletions(-) diff --git a/fzfaws/s3/helper/s3args.py b/fzfaws/s3/helper/s3args.py index 4cb69e3..732f2f0 100644 --- a/fzfaws/s3/helper/s3args.py +++ b/fzfaws/s3/helper/s3args.py @@ -2,9 +2,17 @@ import json from typing import Any, Dict, List, Optional +from PyInquirer import prompt + from fzfaws.kms.kms import KMS from fzfaws.s3 import S3 -from fzfaws.utils import Pyfzf, get_confirmation +from fzfaws.utils import ( + Pyfzf, + get_confirmation, + prompt_style, + URLQueryStringValidator, + CommaListValidator, +) class S3Args: @@ -51,41 +59,36 @@ def set_extra_args( """ if not version: version = [] - attributes: List[str] = [] + choices: List[Dict[str, str]] = [] if version: # only allow modification of the two attributes for versioned object # because other modification would introduce a new version if not metadata and not acl and not tags: - fzf = Pyfzf() - fzf.append_fzf("ACL\n") - fzf.append_fzf("Tagging") - attributes = list( - fzf.execute_fzf( - print_col=1, - multi_select=True, - empty_allow=False, - header="select attributes to configure", - ) - ) + choices = [{"name": "ACL"}, {"name": "Tagging"}] else: if not storage and not acl and not metadata and not encryption and not tags: - fzf = Pyfzf() - fzf.append_fzf("StorageClass\n") - fzf.append_fzf("ACL\n") - fzf.append_fzf("Encryption\n") - fzf.append_fzf("Metadata\n") - fzf.append_fzf("Tagging\n") - attributes = list( - fzf.execute_fzf( - print_col=1, - multi_select=True, - empty_allow=upload, - header="select attributes to configure", - ) - ) + choices = [ + {"name": "StorageClass"}, + {"name": "ACL"}, + {"name": "Encryption"}, + {"name": "Metadata"}, + {"name": "Tagging"}, + ] + + questions: List[Dict[str, Any]] = [ + { + "type": "checkbox", + "name": "selected_attributes", + "message": "Select attributes to configure", + "choices": choices, + } + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt - for attribute in attributes: + for attribute in result.get("selected_attributes", []): if attribute == "StorageClass": storage = True elif attribute == "ACL": @@ -147,25 +150,29 @@ def set_metadata(self, original: str = None) -> None: :param original: original value of metadata :type original: str, optional """ - print( - "Configure metadata for the objects, enter without value will skip metadata" - ) + questions: List[Dict[str, str]] = [ + { + "type": "input", + "name": "metadata", + "message": "Metadata", + "validate": URLQueryStringValidator, + "default": original, + } + ] print( "Metadata format should be a URL Query alike string (e.g. Content-Type=hello&Cache-Control=world)" ) - - if original: - print(80 * "-") - print("Orignal: %s" % original) - metadata = input("Metadata: ") - if metadata: - self._extra_args["Metadata"] = {} - for item in metadata.split("&"): - if "=" not in item: - # handle case for hello=world&foo=boo& - continue - key, value = item.split("=") - self._extra_args["Metadata"][key] = value + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + metadata = result.get("metadata", "") + self._extra_args["Metadata"] = {} + for item in metadata.split("&"): + if "=" not in item: + # handle case for hello=world&foo=boo& + continue + key, value = item.split("=") + self._extra_args["Metadata"][key] = value def set_storageclass(self, original: str = None) -> None: """Set valid storage class. @@ -173,21 +180,36 @@ def set_storageclass(self, original: str = None) -> None: :param original: original value of the storage_class :type original: str, optional """ - header = "select a storage class, esc to use the default storage class of the bucket setting" + choices = [ + "STANDARD", + "REDUCED_REDUNDANCY", + "STANDARD_IA", + "ONEZONE_IA", + "INTELLIGENT_TIERING", + "GLACIER", + "DEEP_ARCHIVE", + ] + questions = [ + { + "type": "rawlist", + "name": "selected_class", + "message": "Select a storage class", + "choices": choices, + } + ] + if original: - header += "\nOriginal: %s" % original + questions[0]["message"] = "Select a storage class (Orignal: %s)" % original - fzf = Pyfzf() - fzf.append_fzf("STANDARD\n") - fzf.append_fzf("REDUCED_REDUNDANCY\n") - fzf.append_fzf("STANDARD_IA\n") - fzf.append_fzf("ONEZONE_IA\n") - fzf.append_fzf("INTELLIGENT_TIERING\n") - fzf.append_fzf("GLACIER\n") - fzf.append_fzf("DEEP_ARCHIVE\n") - result = fzf.execute_fzf(empty_allow=True, print_col=1, header=header) - if result: - self._extra_args["StorageClass"] = result + result = prompt(questions, style=prompt_style) + + if not result: + raise KeyboardInterrupt + + storage_class = result.get("selected_class") + + if storage_class: + self._extra_args["StorageClass"] = storage_class def set_ACL( self, original: bool = False, version: Optional[List[Dict[str, str]]] = None @@ -202,15 +224,27 @@ def set_ACL( if not version: version = [] - fzf = Pyfzf() - fzf.append_fzf("None (use bucket default ACL setting)\n") - fzf.append_fzf("Canned ACL (predefined set of grantees and permissions)\n") - fzf.append_fzf("Explicit ACL (explicit set grantees and permissions)\n") - result = fzf.execute_fzf( - empty_allow=True, - print_col=1, - header="select a type of ACL to grant, aws accept one of canned ACL or explicit ACL", - ) + choices = [ + "None: use bucket default ACL setting", + "Canned ACL: select predefined set of grantees and permissions", + "Explicit ACL: explicitly define grantees and permissions", + ] + + questions = [ + { + "type": "rawlist", + "name": "selected_acl", + "message": "Select a type of ACL to grant, aws accept either canned ACL or explicit ACL", + "choices": choices, + "filter": lambda x: x.split(" ")[0], + } + ] + answers = prompt(questions, style=prompt_style) + if not answers: + raise KeyboardInterrupt + + result = answers.get("selected_acl", "None") + if result == "Canned": self._set_canned_ACL() elif result == "Explicit": @@ -275,53 +309,73 @@ def _set_explicit_ACL( print("Current ACL:") print(json.dumps(original_acl, indent=4, default=str)) - print("Note: fzf.aws cannot preserve previous ACL permission") + print("Note: fzfaws cannot preserve previous ACL permission") if not get_confirmation("Continue?"): return - # get what permission to set - fzf = Pyfzf() - fzf.append_fzf("GrantFullControl\n") - fzf.append_fzf("GrantRead\n") - fzf.append_fzf("GrantReadACP\n") - fzf.append_fzf("GrantWriteACP\n") - results: List[str] = list( - fzf.execute_fzf(empty_allow=True, print_col=1, multi_select=True) - ) + choices = [ + {"name": "GrantFullControl"}, + {"name": "GrantRead"}, + {"name": "GrantReadACP"}, + {"name": "GrantWriteACP"}, + ] + questions = [ + { + "type": "checkbox", + "name": "selected_acl", + "message": "Select ACL to configure", + "choices": choices, + } + ] + answers = prompt(questions, style=prompt_style) + if not answers: + raise KeyboardInterrupt + + results = answers.get("selected_acl", []) + if not results: print( "No permission is set, default ACL settings of the bucket would be used" ) else: + print( + "Enter a list of either the Canonical ID, Account email, Predefined Group url to grant permission (seperate by comma)" + ) + print( + "Format: id=XXX,id=XXX,emailAddress=XXX@gmail.com,uri=http://acs.amazonaws.com/groups/global/AllUsers" + ) for result in results: - print("Set permisstion for %s" % result) - print( - "Enter a list of either the Canonical ID, Account email, Predefined Group url to grant permission (Seperate by comma)" - ) - print( - "Format: id=XXX,id=XXX,emailAddress=XXX@gmail.com,uri=http://acs.amazonaws.com/groups/global/AllUsers" - ) + questions = [ + { + "type": "input", + "name": "input_acl", + "message": result, + "validate": CommaListValidator, + } + ] + if original: - print(80 * "-") if result == "GrantFullControl" and original_acl.get( "FULL_CONTROL" ): - print( - "Orignal: %s" - % ",".join(original_acl.get("FULL_CONTROL", [])) + questions[0]["default"] = ",".join( + original_acl.get("FULL_CONTROL", []) ) elif result == "GrantRead" and original_acl.get("READ"): - print("Orignal: %s" % ",".join(original_acl.get("READ", []))) + questions[0]["default"] = ",".join(original_acl.get("READ", [])) elif result == "GrantReadACP" and original_acl.get("READ_ACP"): - print( - "Orignal: %s" % ",".join(original_acl.get("READ_ACP", [])) + questions[0]["default"] = ",".join( + original_acl.get("READ_ACP", []) ) elif result == "GrantWriteACP" and original_acl.get("WRITE_ACP"): - print( - "Orignal: %s" % ",".join(original_acl.get("WRITE_ACP", [])) + questions[0]["default"] = ",".join( + original_acl.get("WRITE_ACP", []) ) - accounts = input("Accounts: ") - print(80 * "-") + + answers = prompt(questions, style=prompt_style) + if not answers: + raise KeyboardInterrupt + accounts = answers.get("input_acl", "") self._extra_args[result] = str(accounts) def _set_canned_ACL(self) -> None: diff --git a/fzfaws/s3/s3.py b/fzfaws/s3/s3.py index f6eeffc..7996ac4 100644 --- a/fzfaws/s3/s3.py +++ b/fzfaws/s3/s3.py @@ -103,7 +103,7 @@ def set_s3_path(self, download: bool = False) -> None: self.path_list[0] = result.get("s3_path", "") elif selected_option == "root": pass - elif selected_option == "append" or selected_option == "interactively": + elif selected_option == "append" or selected_option == "select": paginator = self.client.get_paginator("list_objects") fzf = Pyfzf() parents = [] @@ -432,7 +432,7 @@ def _get_path_option(self, download: bool = False) -> str: """ choices: List[str] = [ "root: use the root level of the bucket", - "interactively: select a path through fzf", + "select: select a path through fzf", "input: enter the path/name", ] if not download: diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index 931a4c8..2e848e7 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -1,7 +1,8 @@ """This module contains some common helper functions.""" import os from typing import Any, Dict, Generator, List, Optional, Union -from PyInquirer import prompt, style_from_dict, Token +from PyInquirer import prompt, style_from_dict, Token, Validator, ValidationError +import re prompt_style = style_from_dict( { @@ -16,6 +17,28 @@ ) +class URLQueryStringValidator(Validator): + def validate(self, document): + match = re.match(r"^([A-Za-z0-9-]+?=[A-Za-z0-9-]+(&|$))*$", document.text) + if not match: + raise ValidationError( + message="Format should be a URL Query alike string (e.g. Content-Type=hello&Cache-Control=world)", + cursor_position=len(document.text), + ) + + +class CommaListValidator(Validator): + def validate(self, document): + match = re.match( + r"^((id|emailAddress|uri)=[A-Za-z@.\/:]+?(,|$))+", document.text + ) + if not match: + raise ValidationError( + message="Format should be a comma seperated list (e.g. id=XXX,emailAddress=XXX@gmail.com,uri=http://acs.amazonaws.com/groups/global/AllUsers)", + cursor_position=len(document.text), + ) + + def remove_dict_from_list( value: Any, target_list: List[Dict[str, Any]], key_name: str ) -> List[Dict[str, Any]]: diff --git a/tests/s3/test_s3.py b/tests/s3/test_s3.py index 02a9a19..ed5d7e4 100644 --- a/tests/s3/test_s3.py +++ b/tests/s3/test_s3.py @@ -204,12 +204,12 @@ def test_set_s3_path( self.capturedOutput.getvalue(), "S3 file path is set to root\n" ) - # interactively normal + # select normal self.capturedOutput.truncate(0) self.capturedOutput.seek(0) self.s3.bucket_name = "kazhala-version-testing" self.s3.path_list = [""] - mocked_option.return_value = "interactively" + mocked_option.return_value = "select" data_path = Path(__file__).resolve().parent.joinpath("../data/s3_object.json") with data_path.open("r") as file: response = json.load(file) @@ -586,7 +586,7 @@ def test_get_path_option(self, mocked_prompt): "message": "Select which level of the bucket would you like to operate in", "choices": [ "root: use the root level of the bucket", - "interactively: select a path through fzf", + "select: select a path through fzf", "input: enter the path/name", "append: select a path and then enter path/name to append", ], @@ -605,7 +605,7 @@ def test_get_path_option(self, mocked_prompt): "message": "Select which level of the bucket would you like to operate in", "choices": [ "root: use the root level of the bucket", - "interactively: select a path through fzf", + "select: select a path through fzf", "input: enter the path/name", ], "filter": ANY, From 21857b92db8fa02ba2efbfe3c7e9ff51dcedd6d8 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 5 Aug 2020 12:55:11 +1000 Subject: [PATCH 24/40] feat(s3): refactor s3 args to use PyInquirer --- fzfaws/s3/helper/s3args.py | 90 ++++++++++++++++++++++++-------------- fzfaws/utils/util.py | 2 +- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/fzfaws/s3/helper/s3args.py b/fzfaws/s3/helper/s3args.py index 732f2f0..1a200b0 100644 --- a/fzfaws/s3/helper/s3args.py +++ b/fzfaws/s3/helper/s3args.py @@ -199,7 +199,7 @@ def set_storageclass(self, original: str = None) -> None: ] if original: - questions[0]["message"] = "Select a storage class (Orignal: %s)" % original + questions[0]["message"] = "Select a storage class (Original: %s)" % original result = prompt(questions, style=prompt_style) @@ -380,21 +380,27 @@ def _set_explicit_ACL( def _set_canned_ACL(self) -> None: """Set the canned ACL for the current operation.""" - fzf = Pyfzf() - fzf.append_fzf("private\n") - fzf.append_fzf("public-read\n") - fzf.append_fzf("public-read-write\n") - fzf.append_fzf("authenticated-read\n") - fzf.append_fzf("aws-exec-read\n") - fzf.append_fzf("bucket-owner-read\n") - fzf.append_fzf("bucket-owner-full-control\n") - result: str = str( - fzf.execute_fzf( - empty_allow=True, - print_col=1, - header="select a Canned ACL option, esc to use the default ACL setting for the bucket", - ) - ) + choices = [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + "aws-exec-read", + "bucket-owner-read", + "bucket-owner-full-control", + ] + questions = [ + { + "type": "rawlist", + "name": "selected_acl", + "message": "Select a Canned ACL option", + "choices": choices, + } + ] + answers = prompt(questions, style=prompt_style) + if not questions: + raise KeyboardInterrupt + result = answers.get("selected_acl") if result: self._extra_args["ACL"] = result @@ -404,15 +410,29 @@ def set_encryption(self, original: str = None) -> None: :param original: previous value of the encryption :type original: str, optional """ - header = "select an ecryption setting, esc to use the default encryption setting for the bucket" + choices = [ + "None (Use bucket default setting)", + "AES256", + "aws:kms", + ] + questions = [ + { + "type": "rawlist", + "name": "selected_encryption", + "message": "select an encryption setting", + "choices": choices, + } + ] if original: - header += "\nOriginal: %s" % original + questions[0]["message"] = ( + "select an encryption setting (Original: %s)" % original + ) - fzf = Pyfzf() - fzf.append_fzf("None (Use bucket default setting)\n") - fzf.append_fzf("AES256\n") - fzf.append_fzf("aws:kms\n") - result: str = str(fzf.execute_fzf(empty_allow=True, print_col=1, header=header)) + answers = prompt(questions, style=prompt_style) + if not answers: + raise KeyboardInterrupt + + result = answers.get("selected_encryption") if result: self._extra_args["ServerSideEncryption"] = result if result == "aws:kms": @@ -434,15 +454,20 @@ def set_tags( :param version: version information :type version: List[Dict[str, str]], optional """ - print( - "Enter tags for the upload objects, enter without value will skip tagging" - ) print( "Tag format should be a URL Query alike string (e.g. tagname=hello&tag2=world)" ) + questions = [ + { + "type": "input", + "name": "answer", + "message": "Tags", + "validate": URLQueryStringValidator, + } + ] + if original: - print(80 * "-") original_tags: list = [] original_values: str = "" if not version: @@ -452,7 +477,6 @@ def set_tags( for tag in tags.get("TagSet", []): original_tags.append("%s=%s" % (tag.get("Key"), tag.get("Value"))) original_values = "&".join(original_tags) - print("Orignal: %s" % original_values) elif len(version) == 1: tags = self.s3.client.get_object_tagging( Bucket=self.s3.bucket_name, @@ -462,11 +486,13 @@ def set_tags( for tag in tags.get("TagSet", []): original_tags.append("%s=%s" % (tag.get("Key"), tag.get("Value"))) original_values = "&".join(original_tags) - print("Orignal: %s" % original_values) + questions[0]["default"] = original_values - tags = input("Tags: ") - if tags: - self._extra_args["Tagging"] = tags + answer = prompt(questions, style=prompt_style) + if not answer: + raise KeyboardInterrupt + tags = answer.get("answer") + self._extra_args["Tagging"] = tags def check_tag_acl(self) -> Dict[str, Any]: """Check if the only attributes to configure is ACL or Tags. diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index 2e848e7..9156284 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -30,7 +30,7 @@ def validate(self, document): class CommaListValidator(Validator): def validate(self, document): match = re.match( - r"^((id|emailAddress|uri)=[A-Za-z@.\/:]+?(,|$))+", document.text + r"^((id|emailAddress|uri)=[A-Za-z@.\/:-]+?(,|$))+", document.text ) if not match: raise ValidationError( From 48e9fc729788e88b635d5c34ff2e4adbce1cbab5 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 10:34:45 +1000 Subject: [PATCH 25/40] test(s3): update s3 test for PyInquirer --- fzfaws/s3/helper/s3args.py | 10 +- tests/s3/test_s3args.py | 319 +++++++++++++++++++++---------------- 2 files changed, 189 insertions(+), 140 deletions(-) diff --git a/fzfaws/s3/helper/s3args.py b/fzfaws/s3/helper/s3args.py index 1a200b0..d3a59ba 100644 --- a/fzfaws/s3/helper/s3args.py +++ b/fzfaws/s3/helper/s3args.py @@ -7,7 +7,6 @@ from fzfaws.kms.kms import KMS from fzfaws.s3 import S3 from fzfaws.utils import ( - Pyfzf, get_confirmation, prompt_style, URLQueryStringValidator, @@ -57,7 +56,7 @@ def set_extra_args( allow empty selection during upload operation but not for other operations :type upload: bool, optional """ - if not version: + if version is None: version = [] choices: List[Dict[str, str]] = [] @@ -84,7 +83,10 @@ def set_extra_args( "choices": choices, } ] - result = prompt(questions, style=prompt_style) + if choices: + result = prompt(questions, style=prompt_style) + else: + result = {"selected_attributes": []} if not result: raise KeyboardInterrupt @@ -398,7 +400,7 @@ def _set_canned_ACL(self) -> None: } ] answers = prompt(questions, style=prompt_style) - if not questions: + if not answers: raise KeyboardInterrupt result = answers.get("selected_acl") if result: diff --git a/tests/s3/test_s3args.py b/tests/s3/test_s3args.py index ff428cc..3d7fa83 100644 --- a/tests/s3/test_s3args.py +++ b/tests/s3/test_s3args.py @@ -1,3 +1,4 @@ +from fzfaws.utils.util import CommaListValidator, URLQueryStringValidator, prompt_style from fzfaws.utils.pyfzf import Pyfzf from fzfaws.utils.session import BaseSession import io @@ -5,12 +6,13 @@ import os import sys import unittest -from unittest.mock import PropertyMock, patch +from unittest.mock import ANY, PropertyMock, patch from fzfaws.s3.helper.s3args import S3Args from fzfaws.s3 import S3 import boto3 from botocore.stub import Stubber from fzfaws.kms import KMS +from pathlib import Path class TestS3Args(unittest.TestCase): @@ -29,29 +31,25 @@ def test_constructor(self): self.assertIsInstance(self.s3_args.s3, S3) self.assertEqual(self.s3_args._extra_args, {}) + @patch("fzfaws.s3.helper.s3args.prompt") @patch.object(S3Args, "set_tags") @patch.object(S3Args, "set_metadata") @patch.object(S3Args, "set_encryption") @patch.object(S3Args, "set_ACL") @patch.object(S3Args, "set_storageclass") - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") @patch.object(BaseSession, "resource", new_callable=PropertyMock) def test_set_extra_args( self, mocked_resource, - mocked_append, - mocked_execute, mocked_storage, mocked_acl, mocked_encryption, mocked_metadata, mocked_tags, + mocked_prompt, ): - data_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "../data/s3_obj.json" - ) - with open(data_path, "r") as file: + data_path = Path(__file__).resolve().parent.joinpath("../data/s3_obj.json") + with data_path.open("r") as file: response = json.load(file) # normal no version test @@ -59,32 +57,44 @@ def test_set_extra_args( stubber = Stubber(s3.meta.client) stubber.add_response("get_object", response) stubber.activate() - mocked_execute.return_value = [ - "StorageClass", - "ACL", - "Metadata", - "Encryption", - "Tagging", - ] + mocked_prompt.return_value = { + "selected_attributes": [ + "StorageClass", + "ACL", + "Metadata", + "Encryption", + "Tagging", + ] + } + self.s3_args.set_extra_args() - mocked_append.assert_called_with("Tagging\n") + mocked_prompt.assert_called_once_with( + [ + { + "type": "checkbox", + "name": "selected_attributes", + "message": "Select attributes to configure", + "choices": [ + {"name": "StorageClass"}, + {"name": "ACL"}, + {"name": "Encryption"}, + {"name": "Metadata"}, + {"name": "Tagging"}, + ], + } + ], + style=prompt_style, + ) mocked_storage.assert_called() mocked_acl.assert_called_with(original=True, version=[]) mocked_encryption.assert_called() mocked_metadata.assert_called() mocked_tags.assert_called_with(original=True, version=[]) - mocked_execute.assert_called_with( - print_col=1, - multi_select=True, - empty_allow=False, - header="select attributes to configure", - ) # normal no call no version test mocked_encryption.reset_mock() - mocked_execute.reset_mock() + mocked_prompt.reset_mock() mocked_tags.reset_mock() - mocked_append.reset_mock() mocked_tags.reset_mock() mocked_metadata.reset_mock() mocked_storage.reset_mock() @@ -95,10 +105,8 @@ def test_set_extra_args( stubber.activate() self.s3_args.set_extra_args(storage=True, upload=True) mocked_storage.assert_called() - mocked_append.assert_not_called() - mocked_execute.assert_not_called() + mocked_prompt.assert_not_called() mocked_tags.assert_not_called() - mocked_append.assert_not_called() mocked_tags.assert_not_called() mocked_metadata.assert_not_called() mocked_acl.assert_not_called() @@ -106,9 +114,8 @@ def test_set_extra_args( # version test mocked_encryption.reset_mock() - mocked_execute.reset_mock() + mocked_prompt.reset_mock() mocked_tags.reset_mock() - mocked_append.reset_mock() mocked_tags.reset_mock() mocked_metadata.reset_mock() mocked_storage.reset_mock() @@ -117,17 +124,10 @@ def test_set_extra_args( stubber = Stubber(s3.meta.client) stubber.add_response("get_object", response) stubber.activate() - mocked_execute.return_value = ["ACL", "Tagging"] + mocked_prompt.return_value = {"selected_attributes": ["ACL", "Tagging"]} self.s3_args.set_extra_args( version=[{"Key": "hello.json", "VersionId": "11111111"}] ) - mocked_append.assert_called_with("Tagging") - mocked_execute.assert_called_with( - print_col=1, - multi_select=True, - empty_allow=False, - header="select attributes to configure", - ) mocked_storage.assert_not_called() mocked_acl.assert_called_with( original=True, version=[{"Key": "hello.json", "VersionId": "11111111"}] @@ -138,99 +138,113 @@ def test_set_extra_args( original=True, version=[{"Key": "hello.json", "VersionId": "11111111"}] ) - @patch("builtins.input") - def test_set_metadata(self, mocked_input): + @patch("fzfaws.s3.helper.s3args.prompt") + def test_set_metadata(self, mocked_prompt): self.capturedOutput.truncate(0) self.capturedOutput.seek(0) self.s3_args._extra_args = {} - mocked_input.return_value = "foo=boo" + mocked_prompt.return_value = {"metadata": "foo=boo"} self.s3_args.set_metadata(original="hello") self.assertEqual(self.s3_args._extra_args, {"Metadata": {"foo": "boo"}}) - self.assertRegex(self.capturedOutput.getvalue(), "Orignal: hello") + mocked_prompt.assert_called_once_with( + [ + { + "type": "input", + "name": "metadata", + "message": "Metadata", + "validate": URLQueryStringValidator, + "default": "hello", + } + ], + style=prompt_style, + ) - mocked_input.return_value = "foo=boo&hello=world&" + mocked_prompt.return_value = {"metadata": "foo=boo&hello=world&"} self.s3_args.set_metadata() self.assertEqual( self.s3_args._extra_args, {"Metadata": {"foo": "boo", "hello": "world"}} ) - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test_set_storageclass(self, mocked_append, mocked_execute): + @patch("fzfaws.s3.helper.s3args.prompt") + def test_set_storageclass(self, mocked_prompt): self.s3_args._extra_args = {} - mocked_execute.return_value = "STANDARD" + mocked_prompt.return_value = {"selected_class": "STANDARD"} self.s3_args.set_storageclass(original="GLACIER") - mocked_append.assert_called_with("DEEP_ARCHIVE\n") - mocked_execute.assert_called_with( - empty_allow=True, - print_col=1, - header="select a storage class, esc to use the default storage class of the bucket setting\nOriginal: GLACIER", + mocked_prompt.assert_called_once_with( + [ + { + "type": "rawlist", + "name": "selected_class", + "message": "Select a storage class (Original: GLACIER)", + "choices": [ + "STANDARD", + "REDUCED_REDUNDANCY", + "STANDARD_IA", + "ONEZONE_IA", + "INTELLIGENT_TIERING", + "GLACIER", + "DEEP_ARCHIVE", + ], + } + ], + style=prompt_style, ) self.assertEqual(self.s3_args._extra_args, {"StorageClass": "STANDARD"}) - mocked_execute.return_value = "REDUCED_REDUNDANCY" + mocked_prompt.return_value = {"selected_class": "REDUCED_REDUNDANCY"} self.s3_args.set_storageclass() - mocked_append.assert_called_with("DEEP_ARCHIVE\n") - mocked_execute.assert_called_with( - empty_allow=True, - print_col=1, - header="select a storage class, esc to use the default storage class of the bucket setting", - ) self.assertEqual( self.s3_args._extra_args, {"StorageClass": "REDUCED_REDUNDANCY"} ) + @patch("fzfaws.s3.helper.s3args.prompt") @patch.object(S3Args, "_set_explicit_ACL") @patch.object(S3Args, "_set_canned_ACL") - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test_set_ACL( - self, mocked_append, mocked_execute, mocked_canned, mocked_explicit - ): - - mocked_execute.return_value = "None" + def test_set_ACL(self, mocked_canned, mocked_explicit, mocked_prompt): + mocked_prompt.return_value = {"selected_acl": "None"} self.s3_args.set_ACL() - mocked_append.assert_called_with( - "Explicit ACL (explicit set grantees and permissions)\n" - ) mocked_canned.assert_not_called() mocked_explicit.assert_not_called() + mocked_prompt.assert_called_once_with( + [ + { + "type": "rawlist", + "name": "selected_acl", + "message": "Select a type of ACL to grant, aws accept either canned ACL or explicit ACL", + "choices": [ + "None: use bucket default ACL setting", + "Canned ACL: select predefined set of grantees and permissions", + "Explicit ACL: explicitly define grantees and permissions", + ], + "filter": ANY, + } + ], + style=ANY, + ) - mocked_execute.return_value = "Canned" + mocked_prompt.return_value = {"selected_acl": "Canned"} self.s3_args.set_ACL( original=True, version=[{"Key": "hello.json", "VersionId": "11111111"}] ) - mocked_append.assert_called_with( - "Explicit ACL (explicit set grantees and permissions)\n" - ) - mocked_canned.assert_called() + mocked_canned.assert_called_once() mocked_explicit.assert_not_called() mocked_canned.reset_mock() - mocked_execute.return_value = "Explicit" + mocked_prompt.return_value = {"selected_acl": "Explicit"} self.s3_args.set_ACL( original=True, version=[{"Key": "hello.json", "VersionId": "11111111"}] ) - mocked_append.assert_called_with( - "Explicit ACL (explicit set grantees and permissions)\n" - ) mocked_canned.assert_not_called() mocked_explicit.assert_called_with( original=True, version=[{"Key": "hello.json", "VersionId": "11111111"}] ) - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") + @patch("fzfaws.s3.helper.s3args.prompt") @patch.object(BaseSession, "client", new_callable=PropertyMock) @patch("fzfaws.s3.helper.s3args.get_confirmation") - @patch("builtins.input") - def test_set_explicit_ACL( - self, mocked_input, mocked_confirm, mocked_client, mocked_append, mocked_execute - ): - data_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "../data/s3_acl.json" - ) - with open(data_path, "r") as file: + def test_set_explicit_ACL(self, mocked_confirm, mocked_client, mocked_prompt): + data_path = Path(__file__).resolve().parent.joinpath("../data/s3_acl.json") + with data_path.open("r") as file: response = json.load(file) # test orignal values @@ -249,9 +263,6 @@ def test_set_explicit_ACL( self.capturedOutput.getvalue(), r".*uri=http://acs.amazonaws.com/groups/global/AllUsers", ) - self.assertRegex( - self.capturedOutput.getvalue(), r".*\"FULL_CONTROL\": \[\]", - ) # test no original value, set permissions self.capturedOutput.truncate(0) @@ -261,65 +272,91 @@ def test_set_explicit_ACL( stubber.add_response("get_object_acl", response) stubber.activate() mocked_client.return_value = s3 - mocked_execute.return_value = ["GrantFullControl", "GrantRead"] - mocked_input.return_value = "id=11111111,emailAddress=hello@gmail.com" + mocked_prompt.return_value = {"selected_acl": ["GrantFullControl", "GrantRead"]} self.s3_args._set_explicit_ACL() + self.assertEqual( - self.s3_args._extra_args["GrantFullControl"], - "id=11111111,emailAddress=hello@gmail.com", + self.s3_args._extra_args["GrantFullControl"], "", ) self.assertEqual( - self.s3_args._extra_args["GrantRead"], - "id=11111111,emailAddress=hello@gmail.com", + self.s3_args._extra_args["GrantRead"], "", + ) + mocked_prompt.assert_called_with( + [ + { + "type": "input", + "name": "input_acl", + "message": "GrantRead", + "validate": CommaListValidator, + } + ], + style=prompt_style, ) - self.assertNotRegex(self.capturedOutput.getvalue(), r"Orignal") # test original value, set permissions - self.capturedOutput.truncate(0) - self.capturedOutput.seek(0) + mocked_prompt.reset_mock() s3 = boto3.client("s3") stubber = Stubber(s3) stubber.add_response("get_object_acl", response) stubber.activate() mocked_client.return_value = s3 - mocked_execute.return_value = ["GrantRead"] - mocked_input.return_value = "id=2222222,emailAddress=hello@gmail.com" + mocked_prompt.return_value = {"selected_acl": "GrantRead"} mocked_confirm.return_value = True self.s3_args._set_explicit_ACL(original=True) self.assertEqual( - self.s3_args._extra_args["GrantRead"], - "id=2222222,emailAddress=hello@gmail.com", - ) - self.assertRegex( - self.capturedOutput.getvalue(), - r"Orignal: uri=http://acs.amazonaws.com/groups/global/AllUsers", + self.s3_args._extra_args["GrantRead"], "", ) - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test_set_canned_ACL(self, mocked_append, mocked_execute): - mocked_execute.return_value = "private" + @patch("fzfaws.s3.helper.s3args.prompt") + def test_set_canned_ACL(self, mocked_prompt): + mocked_prompt.return_value = {"selected_acl": "private"} self.s3_args._set_canned_ACL() self.assertEqual(self.s3_args._extra_args["ACL"], "private") - mocked_append.assert_called_with("bucket-owner-full-control\n") + mocked_prompt.assert_called_once_with( + [ + { + "type": "rawlist", + "name": "selected_acl", + "message": "Select a Canned ACL option", + "choices": [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + "aws-exec-read", + "bucket-owner-read", + "bucket-owner-full-control", + ], + } + ], + style=prompt_style, + ) self.s3_args._extra_args["ACL"] = None - mocked_execute.return_value = "" - self.s3_args._set_canned_ACL() - self.assertEqual(self.s3_args._extra_args["ACL"], None) + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.s3_args._set_canned_ACL) + @patch("fzfaws.s3.helper.s3args.prompt") @patch.object(BaseSession, "client", new_callable=PropertyMock) @patch.object(KMS, "set_keyids") - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test_encryption(self, mocked_append, mocked_execute, mocked_kms, mocked_client): - mocked_execute.return_value = "None" + def test_encryption(self, mocked_kms, mocked_client, mocked_prompt): + mocked_prompt.return_value = {"selected_encryption": "None"} self.s3_args.set_encryption(original="AES256") self.assertEqual(self.s3_args._extra_args["ServerSideEncryption"], "None") - mocked_execute.assert_called_with( - empty_allow=True, - print_col=1, - header="select an ecryption setting, esc to use the default encryption setting for the bucket\nOriginal: AES256", + mocked_prompt( + [ + { + "type": "rawlist", + "name": "selected_encryption", + "message": "select an encryption setting (Original: AES256)", + "choices": [ + "None (Use bucket default setting)", + "AES256", + "aws:kms", + ], + } + ], + style=ANY, ) # test kms @@ -329,49 +366,59 @@ def test_encryption(self, mocked_append, mocked_execute, mocked_kms, mocked_clie stubber.activate() mocked_client.return_value = s3 - mocked_execute.return_value = "aws:kms" + mocked_prompt.return_value = {"selected_encryption": "aws:kms"} self.s3_args.set_encryption(original="AES256") self.assertEqual(self.s3_args._extra_args["ServerSideEncryption"], "aws:kms") self.assertEqual(self.s3_args._extra_args["SSEKMSKeyId"], "") + @patch("fzfaws.s3.helper.s3args.prompt") @patch.object(BaseSession, "client", new_callable=PropertyMock) - @patch("builtins.input") - def test_set_tags(self, mocked_input, mocked_client): - mocked_input.return_value = "hello=world" + def test_set_tags(self, mocked_client, mocked_prompt): + mocked_prompt.return_value = {"answer": "hello=world"} self.s3_args.set_tags() self.assertEqual(self.s3_args._extra_args["Tagging"], "hello=world") + data_path = Path(__file__).resolve().parent.joinpath("../data/s3_tags.json") + with data_path.open("r") as file: + response = json.load(file) data_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../data/s3_tags.json" ) with open(data_path, "r") as file: response = json.load(file) - self.capturedOutput.truncate(0) - self.capturedOutput.seek(0) + mocked_prompt.reset_mock() s3 = boto3.client("s3") stubber = Stubber(s3) stubber.add_response("get_object_tagging", response) stubber.activate() mocked_client.return_value = s3 - mocked_input.return_value = "foo=boo" + mocked_prompt.return_value = {"answer": "foo=boo"} self.s3_args.set_tags(original=True) self.assertEqual(self.s3_args._extra_args["Tagging"], "foo=boo") - self.assertRegex(self.capturedOutput.getvalue(), r"Orignal: name=yes") + mocked_prompt.assert_called_once_with( + [ + { + "type": "input", + "name": "answer", + "message": "Tags", + "validate": URLQueryStringValidator, + "default": "name=yes", + } + ], + style=prompt_style, + ) - self.capturedOutput.truncate(0) - self.capturedOutput.seek(0) s3 = boto3.client("s3") stubber = Stubber(s3) stubber.add_response("get_object_tagging", response) stubber.activate() mocked_client.return_value = s3 - mocked_input.return_value = "foo=boo" + mocked_prompt.return_value = {"answer": "foo=boo"} self.s3_args.set_tags( original=True, version=[{"Key": "hello", "VersionId": "11111111"}] ) self.assertEqual(self.s3_args._extra_args["Tagging"], "foo=boo") - self.assertRegex(self.capturedOutput.getvalue(), r"Orignal: name=yes") def test_check_tag_acl(self): self.s3_args._extra_args["StorageClass"] = "None" From 5b03301ae72795bb7ff13f0f687c7d8ae725838f Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 10:35:33 +1000 Subject: [PATCH 26/40] fix(s3): typo in prompt --- fzfaws/s3/helper/s3args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fzfaws/s3/helper/s3args.py b/fzfaws/s3/helper/s3args.py index d3a59ba..3e222ef 100644 --- a/fzfaws/s3/helper/s3args.py +++ b/fzfaws/s3/helper/s3args.py @@ -421,13 +421,13 @@ def set_encryption(self, original: str = None) -> None: { "type": "rawlist", "name": "selected_encryption", - "message": "select an encryption setting", + "message": "Select an encryption setting", "choices": choices, } ] if original: questions[0]["message"] = ( - "select an encryption setting (Original: %s)" % original + "Select an encryption setting (Original: %s)" % original ) answers = prompt(questions, style=prompt_style) From 76ab4fef6ca670ec5f1f8a2cb0958b09d818680e Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 10:37:43 +1000 Subject: [PATCH 27/40] docs: docstring for validation class --- fzfaws/utils/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index 9156284..599bc80 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -18,7 +18,10 @@ class URLQueryStringValidator(Validator): + """Validate Query strying for PyInquirer input.""" + def validate(self, document): + """Validate user input.""" match = re.match(r"^([A-Za-z0-9-]+?=[A-Za-z0-9-]+(&|$))*$", document.text) if not match: raise ValidationError( @@ -28,7 +31,10 @@ def validate(self, document): class CommaListValidator(Validator): + """Validate comma seperated list for PyInquirer input.""" + def validate(self, document): + """Validate user input.""" match = re.match( r"^((id|emailAddress|uri)=[A-Za-z@.\/:-]+?(,|$))+", document.text ) From 9530e8642e5afe580c4068db28c4e77e0f024537 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 11:00:46 +1000 Subject: [PATCH 28/40] refactor: move validation class to dedicated module --- fzfaws/utils/__init__.py | 1 + fzfaws/utils/util.py | 5 ++-- fzfaws/utils/validation.py | 49 ++++++++++++++++++++++++++++++++++++++ tests/s3/test_s3args.py | 18 +++++++++----- 4 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 fzfaws/utils/validation.py diff --git a/fzfaws/utils/__init__.py b/fzfaws/utils/__init__.py index 596aec3..b93a01b 100644 --- a/fzfaws/utils/__init__.py +++ b/fzfaws/utils/__init__.py @@ -3,3 +3,4 @@ from .spinner import Spinner from .session import BaseSession from .fileloader import FileLoader +from .validation import * diff --git a/fzfaws/utils/util.py b/fzfaws/utils/util.py index 599bc80..94e856a 100644 --- a/fzfaws/utils/util.py +++ b/fzfaws/utils/util.py @@ -1,8 +1,9 @@ """This module contains some common helper functions.""" import os -from typing import Any, Dict, Generator, List, Optional, Union -from PyInquirer import prompt, style_from_dict, Token, Validator, ValidationError import re +from typing import Any, Dict, Generator, List, Optional, Union + +from PyInquirer import Token, ValidationError, Validator, prompt, style_from_dict prompt_style = style_from_dict( { diff --git a/fzfaws/utils/validation.py b/fzfaws/utils/validation.py new file mode 100644 index 0000000..767c7af --- /dev/null +++ b/fzfaws/utils/validation.py @@ -0,0 +1,49 @@ +"""This module contails related validation classes for PyInquirer.""" +import re + +from PyInquirer import ValidationError, Validator + + +class URLQueryStringValidator(Validator): + """Validate Query strying for PyInquirer input.""" + + def validate(self, document): + """Validate user input.""" + match = re.match(r"^([A-Za-z0-9-]+?=[A-Za-z0-9-]+(&|$))*$", document.text) + if not match: + raise ValidationError( + message="Format should be a URL Query alike string (e.g. Content-Type=hello&Cache-Control=world)", + cursor_position=len(document.text), + ) + + +class CommaListValidator(Validator): + """Validate comma seperated list for PyInquirer input.""" + + def validate(self, document): + """Validate user input.""" + match = re.match( + r"^((id|emailAddress|uri)=[A-Za-z@.\/:-]+?(,|$))+", document.text + ) + if not match: + raise ValidationError( + message="Format should be a comma seperated list (e.g. id=XXX,emailAddress=XXX@gmail.com,uri=http://acs.amazonaws.com/groups/global/AllUsers)", + cursor_position=len(document.text), + ) + + +class StackNameValidator(Validator): + """Validate cloudformation stack name input.""" + + def validate(self, document): + """Validate user input.""" + if not document.text: + raise ValidationError( + message="StackName is required", cursor_position=len(document.text) + ) + match = re.match(r"^[a-zA-Z0-9-]+$", document.text) + if not match: + raise ValidationError( + message="Cloudformation StackName can only contain alphanumeric characters and hyphens", + cursor_position=len(document.text), + ) diff --git a/tests/s3/test_s3args.py b/tests/s3/test_s3args.py index 3d7fa83..a9a493c 100644 --- a/tests/s3/test_s3args.py +++ b/tests/s3/test_s3args.py @@ -1,18 +1,24 @@ -from fzfaws.utils.util import CommaListValidator, URLQueryStringValidator, prompt_style -from fzfaws.utils.pyfzf import Pyfzf -from fzfaws.utils.session import BaseSession import io import json import os +from pathlib import Path import sys import unittest from unittest.mock import ANY, PropertyMock, patch -from fzfaws.s3.helper.s3args import S3Args -from fzfaws.s3 import S3 + import boto3 from botocore.stub import Stubber + from fzfaws.kms import KMS -from pathlib import Path +from fzfaws.s3 import S3 +from fzfaws.s3.helper.s3args import S3Args +from fzfaws.utils import ( + Pyfzf, + BaseSession, + CommaListValidator, + URLQueryStringValidator, + prompt_style, +) class TestS3Args(unittest.TestCase): From b81d303fb68cced73cc22af2df4831f979c40d7d Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 11:01:02 +1000 Subject: [PATCH 29/40] feat(cloudformation): use PyInquirer for stack name enter --- fzfaws/cloudformation/create_stack.py | 29 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/fzfaws/cloudformation/create_stack.py b/fzfaws/cloudformation/create_stack.py index 061613c..6c71c2a 100644 --- a/fzfaws/cloudformation/create_stack.py +++ b/fzfaws/cloudformation/create_stack.py @@ -2,6 +2,8 @@ import json from typing import Any, Dict, Optional, Union +from PyInquirer import prompt + from fzfaws.cloudformation import Cloudformation from fzfaws.cloudformation.helper.cloudformationargs import CloudformationArgs from fzfaws.cloudformation.helper.file_validation import ( @@ -12,8 +14,7 @@ from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor from fzfaws.cloudformation.validate_stack import validate_stack from fzfaws.s3 import S3 -from fzfaws.utils import FileLoader, Pyfzf -from fzfaws.utils.exceptions import NoNameEntered +from fzfaws.utils import FileLoader, Pyfzf, prompt_style, StackNameValidator def create_stack( @@ -106,9 +107,7 @@ def construct_local_creation_args( no_print=True, ) - stack_name: str = input("StackName: ") - if not stack_name: - raise NoNameEntered("No stack name specified") + stack_name = get_stack_name() fileloader = FileLoader(path=local_path) file_data: Dict[str, Any] = {} @@ -185,9 +184,7 @@ def construct_s3_creation_args( elif is_json(s3.path_list[0]): file_type = "json" - stack_name: str = input("StackName: ") - if not stack_name: - raise NoNameEntered("No stack name specified") + stack_name = get_stack_name() file_data: dict = s3.get_object_data(file_type) if "Parameters" in file_data: @@ -210,3 +207,19 @@ def construct_s3_creation_args( } return cloudformation_args + + +def get_stack_name() -> str: + """Get user to input the stack name.""" + questions = [ + { + "type": "input", + "name": "answer", + "message": "StackName", + "validate": StackNameValidator, + } + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + return result.get("answer", "") From eebdd48f20985381a3e351003681c5985cb7e671 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 11:12:16 +1000 Subject: [PATCH 30/40] test(cloudformation): update cloudformation test --- fzfaws/cloudformation/create_stack.py | 2 +- tests/cloudformation/test_create_stack.py | 55 ++++++++++++----------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/fzfaws/cloudformation/create_stack.py b/fzfaws/cloudformation/create_stack.py index 6c71c2a..76fb204 100644 --- a/fzfaws/cloudformation/create_stack.py +++ b/fzfaws/cloudformation/create_stack.py @@ -14,7 +14,7 @@ from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor from fzfaws.cloudformation.validate_stack import validate_stack from fzfaws.s3 import S3 -from fzfaws.utils import FileLoader, Pyfzf, prompt_style, StackNameValidator +from fzfaws.utils import FileLoader, Pyfzf, StackNameValidator, prompt_style def create_stack( diff --git a/tests/cloudformation/test_create_stack.py b/tests/cloudformation/test_create_stack.py index 3ddc2f0..25e1dee 100644 --- a/tests/cloudformation/test_create_stack.py +++ b/tests/cloudformation/test_create_stack.py @@ -1,24 +1,25 @@ -from fzfaws.cloudformation.helper.cloudformationargs import CloudformationArgs -from fzfaws.s3.s3 import S3 import io import os +from pathlib import Path import sys import unittest from unittest.mock import ANY, patch -from pathlib import Path from fzfaws.cloudformation.cloudformation import Cloudformation -from fzfaws.cloudformation.create_stack import create_stack +from fzfaws.cloudformation.create_stack import create_stack, get_stack_name +from fzfaws.cloudformation.helper.cloudformationargs import CloudformationArgs from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor -from fzfaws.utils import Pyfzf, FileLoader +from fzfaws.s3.s3 import S3 +from fzfaws.utils import FileLoader, Pyfzf class TestCloudformationCreateStack(unittest.TestCase): def setUp(self): self.capturedOutput = io.StringIO() - self.data_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../data/cloudformation_template.yaml", + self.data_path = str( + Path(__file__) + .resolve() + .parent.joinpath("../data/cloudformation_template.yaml") ) sys.stdout = self.capturedOutput fileloader = FileLoader() @@ -28,27 +29,22 @@ def setUp(self): def tearDown(self): sys.stdout = sys.__stdout__ + @patch("fzfaws.cloudformation.create_stack.prompt") @patch.object(ParamProcessor, "process_stack_params") @patch.object(Cloudformation, "wait") @patch.object(Cloudformation, "execute_with_capabilities") - @patch("builtins.input") @patch("fzfaws.cloudformation.create_stack.validate_stack") @patch.object(Pyfzf, "get_local_file") def test_local_creation( self, mocked_local, mocked_validate, - mocked_input, mocked_execute, mocked_wait, mocked_process, + mocked_prompt, ): - - mocked_input.return_value = "testing_stack" - self.data_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../data/cloudformation_template.yaml", - ) + mocked_prompt.return_value = {"answer": "testing-stack"} mocked_local.return_value = self.data_path create_stack(local_path=True, root=True, wait=True) @@ -58,7 +54,7 @@ def test_local_creation( ) mocked_execute.assert_called_with( Parameters=[], - StackName="testing_stack", + StackName="testing-stack", TemplateBody=ANY, cloudformation_action=ANY, ) @@ -76,7 +72,7 @@ def test_local_creation( ) mocked_execute.assert_called_with( Parameters=[], - StackName="testing_stack", + StackName="testing-stack", TemplateBody=ANY, cloudformation_action=ANY, ) @@ -84,10 +80,10 @@ def test_local_creation( "stack_create_complete", "Waiting for stack to be ready ..." ) + @patch("fzfaws.cloudformation.create_stack.prompt") @patch.object(ParamProcessor, "process_stack_params") @patch.object(Cloudformation, "wait") @patch.object(Cloudformation, "execute_with_capabilities") - @patch("builtins.input") @patch("fzfaws.cloudformation.create_stack.validate_stack") @patch.object(S3, "get_object_url") @patch.object(S3, "get_object_data") @@ -98,12 +94,12 @@ def test_s3_creation( mocked_data, mocked_url, mocked_validate, - mocked_input, mocked_execute, mocked_wait, mocked_process, + mocked_prompt, ): - mocked_input.return_value = "testing_stack" + mocked_prompt.return_value = {"answer": "testing-stack"} mocked_version.return_value = [{"VersionId": "111111"}] fileloader = FileLoader(self.data_path) mocked_data.return_value = fileloader.process_yaml_file() @@ -122,7 +118,7 @@ def test_s3_creation( mocked_url.assert_called_with(version="111111") mocked_execute.assert_called_with( Parameters=[], - StackName="testing_stack", + StackName="testing-stack", TemplateURL="https://s3-ap-southeast-2.amazonaws.com/kazhala-lol/hello.yaml?versionId=111111", cloudformation_action=ANY, ) @@ -148,7 +144,7 @@ def test_s3_creation( mocked_url.assert_called_with(version="111111") mocked_execute.assert_called_with( Parameters=[], - StackName="testing_stack", + StackName="testing-stack", TemplateURL="https://s3-ap-southeast-2.amazonaws.com/kazhala-lol/hello.yaml?versionId=111111", cloudformation_action=ANY, ) @@ -163,8 +159,17 @@ def test_s3_creation( def test_create_stack_with_extra( self, mocked_args, mocked_set_args, mocked_execute, mocked_wait ): - mocked_args.return_value = {"StackName": "testing_stack"} + mocked_args.return_value = {"StackName": "testing-stack"} create_stack(wait=True, extra=True) - mocked_execute.assert_called_with(StackName="testing_stack") + mocked_execute.assert_called_with(StackName="testing-stack") mocked_set_args.assert_called_with(search_from_root=False) mocked_wait.assert_called_once() + + @patch("fzfaws.cloudformation.create_stack.prompt") + def test_get_stack_name(self, mocked_prompt): + mocked_prompt.return_value = {"answer": "hello"} + result = get_stack_name() + self.assertEqual(result, "hello") + + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, get_stack_name) From 0ef5b2ada86debca7e5eca8a2435eeea2e2f36aa Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 11:47:18 +1000 Subject: [PATCH 31/40] refactor(cloudformation): move get_stack_name to helper module --- fzfaws/cloudformation/create_stack.py | 20 ++---------- fzfaws/cloudformation/helper/__init__.py | 18 +++++++++++ tests/cloudformation/test_create_stack.py | 24 +++++--------- tests/cloudformation/test_helper.py | 39 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 34 deletions(-) create mode 100644 tests/cloudformation/test_helper.py diff --git a/fzfaws/cloudformation/create_stack.py b/fzfaws/cloudformation/create_stack.py index 76fb204..257df6e 100644 --- a/fzfaws/cloudformation/create_stack.py +++ b/fzfaws/cloudformation/create_stack.py @@ -2,7 +2,6 @@ import json from typing import Any, Dict, Optional, Union -from PyInquirer import prompt from fzfaws.cloudformation import Cloudformation from fzfaws.cloudformation.helper.cloudformationargs import CloudformationArgs @@ -14,7 +13,8 @@ from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor from fzfaws.cloudformation.validate_stack import validate_stack from fzfaws.s3 import S3 -from fzfaws.utils import FileLoader, Pyfzf, StackNameValidator, prompt_style +from fzfaws.utils import FileLoader, Pyfzf +from fzfaws.cloudformation.helper import get_stack_name def create_stack( @@ -207,19 +207,3 @@ def construct_s3_creation_args( } return cloudformation_args - - -def get_stack_name() -> str: - """Get user to input the stack name.""" - questions = [ - { - "type": "input", - "name": "answer", - "message": "StackName", - "validate": StackNameValidator, - } - ] - result = prompt(questions, style=prompt_style) - if not result: - raise KeyboardInterrupt - return result.get("answer", "") diff --git a/fzfaws/cloudformation/helper/__init__.py b/fzfaws/cloudformation/helper/__init__.py index e69de29..b273fd1 100644 --- a/fzfaws/cloudformation/helper/__init__.py +++ b/fzfaws/cloudformation/helper/__init__.py @@ -0,0 +1,18 @@ +from fzfaws.utils import StackNameValidator, prompt_style +from PyInquirer import prompt + + +def get_stack_name(message="StackName") -> str: + """Get user to input the stack name.""" + questions = [ + { + "type": "input", + "name": "answer", + "message": message, + "validate": StackNameValidator, + } + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + return result.get("answer", "") diff --git a/tests/cloudformation/test_create_stack.py b/tests/cloudformation/test_create_stack.py index 25e1dee..3a37085 100644 --- a/tests/cloudformation/test_create_stack.py +++ b/tests/cloudformation/test_create_stack.py @@ -6,7 +6,8 @@ from unittest.mock import ANY, patch from fzfaws.cloudformation.cloudformation import Cloudformation -from fzfaws.cloudformation.create_stack import create_stack, get_stack_name +from fzfaws.cloudformation.create_stack import create_stack +from fzfaws.cloudformation.helper import get_stack_name from fzfaws.cloudformation.helper.cloudformationargs import CloudformationArgs from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor from fzfaws.s3.s3 import S3 @@ -29,7 +30,7 @@ def setUp(self): def tearDown(self): sys.stdout = sys.__stdout__ - @patch("fzfaws.cloudformation.create_stack.prompt") + @patch("fzfaws.cloudformation.create_stack.get_stack_name") @patch.object(ParamProcessor, "process_stack_params") @patch.object(Cloudformation, "wait") @patch.object(Cloudformation, "execute_with_capabilities") @@ -42,9 +43,9 @@ def test_local_creation( mocked_execute, mocked_wait, mocked_process, - mocked_prompt, + mocked_stackname, ): - mocked_prompt.return_value = {"answer": "testing-stack"} + mocked_stackname.return_value = "testing-stack" mocked_local.return_value = self.data_path create_stack(local_path=True, root=True, wait=True) @@ -80,7 +81,7 @@ def test_local_creation( "stack_create_complete", "Waiting for stack to be ready ..." ) - @patch("fzfaws.cloudformation.create_stack.prompt") + @patch("fzfaws.cloudformation.create_stack.get_stack_name") @patch.object(ParamProcessor, "process_stack_params") @patch.object(Cloudformation, "wait") @patch.object(Cloudformation, "execute_with_capabilities") @@ -97,9 +98,9 @@ def test_s3_creation( mocked_execute, mocked_wait, mocked_process, - mocked_prompt, + mocked_stackname, ): - mocked_prompt.return_value = {"answer": "testing-stack"} + mocked_stackname.return_value = "testing-stack" mocked_version.return_value = [{"VersionId": "111111"}] fileloader = FileLoader(self.data_path) mocked_data.return_value = fileloader.process_yaml_file() @@ -164,12 +165,3 @@ def test_create_stack_with_extra( mocked_execute.assert_called_with(StackName="testing-stack") mocked_set_args.assert_called_with(search_from_root=False) mocked_wait.assert_called_once() - - @patch("fzfaws.cloudformation.create_stack.prompt") - def test_get_stack_name(self, mocked_prompt): - mocked_prompt.return_value = {"answer": "hello"} - result = get_stack_name() - self.assertEqual(result, "hello") - - mocked_prompt.return_value = {} - self.assertRaises(KeyboardInterrupt, get_stack_name) diff --git a/tests/cloudformation/test_helper.py b/tests/cloudformation/test_helper.py new file mode 100644 index 0000000..edf6e03 --- /dev/null +++ b/tests/cloudformation/test_helper.py @@ -0,0 +1,39 @@ +from fzfaws.cloudformation.helper import get_stack_name +import unittest +from unittest.mock import patch +from fzfaws.utils import StackNameValidator, prompt_style + + +class TestCloudformationHelper(unittest.TestCase): + @patch("fzfaws.cloudformation.helper.prompt") + def test_get_stack_name(self, mocked_prompt): + mocked_prompt.return_value = {"answer": "hello"} + result = get_stack_name() + self.assertEqual(result, "hello") + mocked_prompt.assert_called_with( + [ + { + "type": "input", + "name": "answer", + "message": "StackName", + "validate": StackNameValidator, + } + ], + style=prompt_style, + ) + + result = get_stack_name(message="hello") + mocked_prompt.assert_called_with( + [ + { + "type": "input", + "name": "answer", + "message": "hello", + "validate": StackNameValidator, + } + ], + style=prompt_style, + ) + + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, get_stack_name) From 9f24613eb4175071c844f8760468a0049ab33964 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 11:47:32 +1000 Subject: [PATCH 32/40] feat(cloudformation): use PyInquirer for changeset_stack --- fzfaws/cloudformation/changeset_stack.py | 15 +++++++++------ fzfaws/cloudformation/update_stack.py | 2 +- tests/cloudformation/test_changeset_stack.py | 15 ++++++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/fzfaws/cloudformation/changeset_stack.py b/fzfaws/cloudformation/changeset_stack.py index af36192..614de68 100644 --- a/fzfaws/cloudformation/changeset_stack.py +++ b/fzfaws/cloudformation/changeset_stack.py @@ -2,10 +2,12 @@ import json from typing import Any, Dict, Union +from PyInquirer import prompt + from fzfaws.cloudformation import Cloudformation +from fzfaws.cloudformation.helper import get_stack_name from fzfaws.cloudformation.update_stack import update_stack -from fzfaws.utils import Pyfzf, get_confirmation -from fzfaws.utils.exceptions import NoNameEntered +from fzfaws.utils import Pyfzf, get_confirmation, prompt_style def describe_changes(cloudformation: Cloudformation, changeset_name: str) -> None: @@ -123,10 +125,11 @@ def changeset_stack( ) else: - changeset_name = input("Enter name of this changeset: ") - if not changeset_name: - raise NoNameEntered("No changeset name specified") - changeset_description = input("Description: ") + changeset_name = get_stack_name(message="ChangeSetName") + questions = [{"type": "input", "message": "Description", "name": "answer"}] + result = prompt(questions, style=prompt_style) + changeset_description = result.get("answer", "") + # since is almost same operation as update stack # let update_stack handle it, but return update details instead of execute cloudformation_args = update_stack( diff --git a/fzfaws/cloudformation/update_stack.py b/fzfaws/cloudformation/update_stack.py index e03c889..4088b77 100644 --- a/fzfaws/cloudformation/update_stack.py +++ b/fzfaws/cloudformation/update_stack.py @@ -12,7 +12,7 @@ from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor from fzfaws.cloudformation.validate_stack import validate_stack from fzfaws.s3 import S3 -from fzfaws.utils import Pyfzf, FileLoader +from fzfaws.utils import FileLoader, Pyfzf def update_stack( diff --git a/tests/cloudformation/test_changeset_stack.py b/tests/cloudformation/test_changeset_stack.py index d2786d5..79d2e18 100644 --- a/tests/cloudformation/test_changeset_stack.py +++ b/tests/cloudformation/test_changeset_stack.py @@ -1,9 +1,10 @@ -from fzfaws.utils.pyfzf import Pyfzf import io import sys import unittest from unittest.mock import call, patch + from fzfaws.cloudformation.changeset_stack import changeset_stack +from fzfaws.utils.pyfzf import Pyfzf class TestCloudformationChangesetStack(unittest.TestCase): @@ -117,13 +118,17 @@ def test_delete_changeset( ] ) + @patch("fzfaws.cloudformation.changeset_stack.get_stack_name") + @patch("fzfaws.cloudformation.changeset_stack.prompt") @patch("fzfaws.cloudformation.changeset_stack.update_stack") - @patch("builtins.input") @patch("fzfaws.cloudformation.changeset_stack.Cloudformation") - def test_create_changeset(self, MockedCloudformation, mocked_input, mocked_udpate): + def test_create_changeset( + self, MockedCloudformation, mocked_udpate, mocked_prompt, mocked_stackname + ): cloudformation = MockedCloudformation() cloudformation.stack_name = "testing1" - mocked_input.return_value = "fooboo" + mocked_stackname.return_value = "fooboo" + mocked_prompt.return_value = {"answer": "hello"} mocked_udpate.return_value = { "Parameters": [ {"ParameterKey": "SSHLocation", "UsePreviousValue": True}, @@ -138,7 +143,7 @@ def test_create_changeset(self, MockedCloudformation, mocked_input, mocked_udpat changeset_stack(profile="root", region="us-east-1") cloudformation.execute_with_capabilities.assert_called_once_with( ChangeSetName="fooboo", - Description="fooboo", + Description="hello", Parameters=[ {"ParameterKey": "SSHLocation", "UsePreviousValue": True}, {"ParameterKey": "Hello", "UsePreviousValue": True}, From d94da6e527bc7c1037361069d9d4c4b19b9093b2 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Mon, 10 Aug 2020 12:18:56 +1000 Subject: [PATCH 33/40] feat(cloudformation): get capabilities through PyInquirer --- fzfaws/cloudformation/cloudformation.py | 31 +++++++---- tests/cloudformation/test_cloudformation.py | 62 +++++++++------------ 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/fzfaws/cloudformation/cloudformation.py b/fzfaws/cloudformation/cloudformation.py index 6d4b010..aa5d1ec 100644 --- a/fzfaws/cloudformation/cloudformation.py +++ b/fzfaws/cloudformation/cloudformation.py @@ -4,6 +4,8 @@ import re import sys from typing import Any, Callable, Dict, Generator, List, Tuple, Union +from PyInquirer import prompt +from fzfaws.utils import prompt_style from fzfaws.utils import ( BaseSession, @@ -165,17 +167,26 @@ def _get_capabilities(self, message: str = "") -> List[str]: :return: selected capabilities to acknowledge :rtype: List[str] """ - fzf = Pyfzf() - fzf.append_fzf("CAPABILITY_IAM\n") - fzf.append_fzf("CAPABILITY_NAMED_IAM\n") - fzf.append_fzf("CAPABILITY_AUTO_EXPAND") - message += "\nPlease select the capabilities to acknowledge and proceed" - message += "\nMore information: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html" - return list( - fzf.execute_fzf( - empty_allow=True, print_col=1, multi_select=True, header=message - ) + print(message) + print( + "More information: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html" ) + questions = [ + { + "type": "checkbox", + "name": "answer", + "message": "Select the capabilities to acknowledge and proceed", + "choices": [ + {"name": "CAPABILITY_IAM"}, + {"name": "CAPABILITY_NAMED_IAM"}, + {"name": "CAPABILITY_AUTO_EXPAND"}, + ], + } + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + return result.get("answer", []) def _get_stack_generator( self, response: List[Dict[str, Any]] diff --git a/tests/cloudformation/test_cloudformation.py b/tests/cloudformation/test_cloudformation.py index 7a80805..02ca41e 100644 --- a/tests/cloudformation/test_cloudformation.py +++ b/tests/cloudformation/test_cloudformation.py @@ -1,17 +1,16 @@ import io import json import os +from pathlib import Path import sys import unittest -from unittest.mock import ANY, call, patch -from pathlib import Path +from unittest.mock import ANY, patch from botocore.paginate import Paginator from botocore.waiter import Waiter from fzfaws.cloudformation import Cloudformation -from fzfaws.utils import FileLoader -from fzfaws.utils.pyfzf import Pyfzf +from fzfaws.utils import FileLoader, Pyfzf, prompt_style class TestCloudformation(unittest.TestCase): @@ -231,43 +230,36 @@ def hello(**kwargs): mocked_confirm.return_value = False self.assertRaises(SystemExit, self.cloudformation.execute_with_capabilities) - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test_get_capabilities(self, mocked_append, mocked_execute): - mocked_execute.return_value = ["CAPABILITY_IAM"] + @patch("fzfaws.cloudformation.cloudformation.prompt") + def test_get_capabilities(self, mocked_prompt): + mocked_prompt.return_value = {"answer": ["CAPABILITY_IAM"]} result = self.cloudformation._get_capabilities(message="lol") - mocked_append.assert_has_calls( + mocked_prompt.assert_called_once_with( [ - call("CAPABILITY_IAM\n"), - call("CAPABILITY_NAMED_IAM\n"), - call("CAPABILITY_AUTO_EXPAND"), - ] - ) - mocked_execute.assert_called_once_with( - empty_allow=True, - print_col=1, - multi_select=True, - header="lol\nPlease select the capabilities to acknowledge and proceed\nMore information: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html", + { + "type": "checkbox", + "name": "answer", + "message": "Select the capabilities to acknowledge and proceed", + "choices": [ + {"name": "CAPABILITY_IAM"}, + {"name": "CAPABILITY_NAMED_IAM"}, + {"name": "CAPABILITY_AUTO_EXPAND"}, + ], + } + ], + style=prompt_style, ) self.assertEqual(result, ["CAPABILITY_IAM"]) + self.assertEqual( + self.capturedOutput.getvalue(), + "lol\nMore information: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html\n", + ) - mocked_execute.reset_mock() - mocked_append.reset_mock() - mocked_execute.return_value = ["CAPABILITY_IAM", "CAPABILITY_AUTO_EXPAND"] + mocked_prompt.reset_mock() + mocked_prompt.return_value = { + "answer": ["CAPABILITY_IAM", "CAPABILITY_AUTO_EXPAND"] + } result = self.cloudformation._get_capabilities() - mocked_append.assert_has_calls( - [ - call("CAPABILITY_IAM\n"), - call("CAPABILITY_NAMED_IAM\n"), - call("CAPABILITY_AUTO_EXPAND"), - ] - ) - mocked_execute.assert_called_once_with( - empty_allow=True, - print_col=1, - multi_select=True, - header="\nPlease select the capabilities to acknowledge and proceed\nMore information: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html", - ) self.assertEqual(result, ["CAPABILITY_IAM", "CAPABILITY_AUTO_EXPAND"]) def test_get_stack_generator(self): From 0bf81406eb1b1ea489d764dd6897d36284358ccc Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Tue, 11 Aug 2020 08:54:20 +1000 Subject: [PATCH 34/40] feat(cloudformation): use PyInquirer for creation option --- fzfaws/cloudformation/cloudformation.py | 2 +- .../helper/cloudformationargs.py | 136 +++++++++--------- 2 files changed, 70 insertions(+), 68 deletions(-) diff --git a/fzfaws/cloudformation/cloudformation.py b/fzfaws/cloudformation/cloudformation.py index aa5d1ec..dcdab45 100644 --- a/fzfaws/cloudformation/cloudformation.py +++ b/fzfaws/cloudformation/cloudformation.py @@ -5,8 +5,8 @@ import sys from typing import Any, Callable, Dict, Generator, List, Tuple, Union from PyInquirer import prompt -from fzfaws.utils import prompt_style +from fzfaws.utils import prompt_style from fzfaws.utils import ( BaseSession, Pyfzf, diff --git a/fzfaws/cloudformation/helper/cloudformationargs.py b/fzfaws/cloudformation/helper/cloudformationargs.py index dca10d2..416df5c 100644 --- a/fzfaws/cloudformation/helper/cloudformationargs.py +++ b/fzfaws/cloudformation/helper/cloudformationargs.py @@ -1,11 +1,13 @@ """Contains helper class to set extra arguments for cloudformation.""" -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional + +from PyInquirer import prompt from fzfaws.cloudformation import Cloudformation from fzfaws.cloudwatch import Cloudwatch from fzfaws.iam import IAM from fzfaws.sns import SNS -from fzfaws.utils import Pyfzf +from fzfaws.utils import Pyfzf, prompt_style class CloudformationArgs: @@ -68,23 +70,28 @@ def set_extra_args( and not creation_option and not notification ): - fzf = Pyfzf() - fzf.append_fzf("Tags\n") - fzf.append_fzf("Permissions\n") + choices: List[Dict[str, str]] = [ + {"name": "Tags"}, + {"name": "Permissions"}, + {"name": "Notifications"}, + {"name": "RollbackConfiguration"}, + ] if not dryrun: - fzf.append_fzf("StackPolicy\n") - fzf.append_fzf("Notifications\n") - fzf.append_fzf("RollbackConfiguration\n") + choices.append({"name": "StackPolicy"}) if not dryrun and not update: - fzf.append_fzf("CreationOption\n") - attributes = list( - fzf.execute_fzf( - empty_allow=True, - print_col=1, - multi_select=True, - header="select options to configure", - ) - ) + choices.append({"name": "CreationOption"}) + questions: List[Dict[str, Any]] = [ + { + "type": "checkbox", + "name": "answer", + "message": "Select options to configure", + "choices": choices, + } + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + attributes = result.get("answer", []) for attribute in attributes: if attribute == "Tags": @@ -115,57 +122,52 @@ def set_extra_args( def _set_creation(self) -> None: """Set creation option for stack.""" - print(80 * "-") - fzf = Pyfzf() - fzf.append_fzf("RollbackOnFailure\n") - fzf.append_fzf("TimeoutInMinutes\n") - fzf.append_fzf("EnableTerminationProtection\n") - selected_options: List[str] = list( - fzf.execute_fzf( - empty_allow=True, - print_col=1, - multi_select=True, - header="select options to configure", + questions: List[Dict[str, Any]] = [ + { + "type": "checkbox", + "name": "selected_options", + "message": "Select creation options to configure", + "choices": [ + {"name": "RollbackOnFailure"}, + {"name": "TimeoutInMinutes"}, + {"name": "EnableTerminationProtection"}, + ], + }, + { + "type": "rawlist", + "name": "rollback", + "message": "Roll back on failure?", + "choices": ["True", "False"], + "when": lambda x: "RollbackOnFailure" in x["selected_options"], + }, + { + "type": "input", + "name": "timeout", + "message": "Specify number of minutes before timeout", + "when": lambda x: "TimeoutInMinutes" in x["selected_options"], + }, + { + "type": "rawlist", + "name": "termination", + "message": "Enable termination protection?", + "choices": ["True", "False"], + "when": lambda x: "EnableTerminationProtection" + in x["selected_options"], + }, + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + if result.get("rollback"): + self._extra_args["OnFailure"] = ( + "ROLLBACK" if result["rollback"] == "True" else "DO_NOTHING" + ) + if result.get("timeout"): + self._extra_args["TimeoutInMinutes"] = int(result["timeout"]) + if result.get("termination"): + self._extra_args["EnableTerminationProtection"] = ( + True if result["termination"] == "True" else False ) - ) - - for option in selected_options: - result: str = "" - if option == "RollbackOnFailure": - fzf.fzf_string = "" - fzf.append_fzf("True\n") - fzf.append_fzf("False\n") - result = str( - fzf.execute_fzf( - empty_allow=True, - print_col=1, - header="roll back on failue? (Default: True)", - ) - ) - if result: - self._extra_args["OnFailure"] = ( - "ROLLBACK" if result == "True" else "DO_NOTHING" - ) - elif option == "TimeoutInMinutes": - message = "Specify number of minutes before stack timeout (Default: no timeout): " - timeout = input(message) - if timeout: - self._extra_args["TimeoutInMinutes"] = int(timeout) - elif option == "EnableTerminationProtection": - fzf.fzf_string = "" - fzf.append_fzf("True\n") - fzf.append_fzf("False\n") - result = str( - fzf.execute_fzf( - empty_allow=True, - print_col=1, - header="enable termination protection? (Default: False)", - ) - ) - if result: - self._extra_args["EnableTerminationProtection"] = ( - True if result == "True" else False - ) def _set_rollback(self, update: bool = False) -> None: """Set rollback configuration for cloudformation. From ec289e5d1705db42d4c249c7e83e753ae44516f0 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Tue, 11 Aug 2020 09:15:46 +1000 Subject: [PATCH 35/40] feat(cloudformation): use PyInquirer to get rollback config --- fzfaws/cli.py | 3 +- .../helper/cloudformationargs.py | 30 ++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/fzfaws/cli.py b/fzfaws/cli.py index 78cb39e..8f59bb6 100644 --- a/fzfaws/cli.py +++ b/fzfaws/cli.py @@ -3,7 +3,6 @@ Typical usage example: fzfaws --options """ - import argparse import os from pathlib import Path @@ -15,8 +14,8 @@ from fzfaws.cloudformation.main import cloudformation from fzfaws.ec2.main import ec2 -from fzfaws.s3.main import s3 from fzfaws.lambdaf.main import lambdaf +from fzfaws.s3.main import s3 from fzfaws.utils import FileLoader, get_default_args from fzfaws.utils.exceptions import InvalidFileType, NoSelectionMade diff --git a/fzfaws/cloudformation/helper/cloudformationargs.py b/fzfaws/cloudformation/helper/cloudformationargs.py index 416df5c..a886913 100644 --- a/fzfaws/cloudformation/helper/cloudformationargs.py +++ b/fzfaws/cloudformation/helper/cloudformationargs.py @@ -175,23 +175,26 @@ def _set_rollback(self, update: bool = False) -> None: :param update: show previous values if true :type update: bool, optional """ - print(80 * "-") - cloudwatch = Cloudwatch(self.cloudformation.profile, self.cloudformation.region) header: str = "select a cloudwatch alarm to monitor the stack" - message: str = "MonitoringTimeInMinutes(Default: 0): " + questions: List[Dict[str, str]] = [ + {"type": "input", "message": "MonitoringTimeInMinutes", "name": "answer"} + ] if update and self.cloudformation.stack_details.get("RollbackConfiguration"): header += "\nOriginal value: %s" % self.cloudformation.stack_details[ "RollbackConfiguration" ].get("RollbackTriggers") - message = "MonitoringTimeInMinutes(Original: %s): " % self.cloudformation.stack_details[ - "RollbackConfiguration" - ].get( - "MonitoringTimeInMinutes" + questions[0]["default"] = str( + self.cloudformation.stack_details["RollbackConfiguration"].get( + "MonitoringTimeInMinutes", "" + ) ) cloudwatch.set_arns(empty_allow=True, header=header, multi_select=True) - print("Selected arns: %s" % cloudwatch.arns) - monitor_time = input(message) + print("Selected alarm: %s" % cloudwatch.arns) + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + monitor_time = result.get("answer") if cloudwatch.arns: self._extra_args["RollbackConfiguration"] = { "RollbackTriggers": [ @@ -207,7 +210,6 @@ def _set_notification(self, update: bool = False) -> None: :param update: show previous values if true :type update: bool, optional """ - print(80 * "-") sns = SNS( profile=self.cloudformation.profile, region=self.cloudformation.region ) @@ -217,6 +219,7 @@ def _set_notification(self, update: bool = False) -> None: "NotificationARNs" ) sns.set_arns(empty_allow=True, header=header, multi_select=True) + print("Selected notification: %s" % sns.arns) if sns.arns: self._extra_args["NotificationARNs"] = sns.arns @@ -231,7 +234,6 @@ def _set_policy(self, update: bool = False, search_from_root: bool = False) -> N :param search_from_root: search files from root :type search_from_root: bool, optional """ - print(80 * "-") fzf = Pyfzf() file_path: str = str( fzf.get_local_file( @@ -249,6 +251,7 @@ def _set_policy(self, update: bool = False, search_from_root: bool = False) -> N with open(str(file_path), "r") as body: body = body.read() self._extra_args["StackPolicyDuringUpdateBody"] = body + print("Selected policy: %s" % file_path) def _set_permissions(self, update: bool = False) -> None: """Set the iam user for the current stack. @@ -260,7 +263,6 @@ def _set_permissions(self, update: bool = False) -> None: :param update: show previous values if true :type update: bool, optional """ - print(80 * "-") iam = IAM(profile=self.cloudformation.profile) if not update: header = ( @@ -274,6 +276,7 @@ def _set_permissions(self, update: bool = False) -> None: "RoleARN" ) iam.set_arns(header=header, service="cloudformation.amazonaws.com") + print("Selected role: %s" % iam.arns[0]) if iam.arns: self._extra_args["RoleARN"] = iam.arns[0] @@ -290,9 +293,8 @@ def _set_tags(self, update: bool = False) -> None: :param update: determine if is updating the stack, it will show different prompt :type update: bool, optional """ - print(80 * "-") - tag_list: List[Dict[str, str]] = [] + if update: if self.cloudformation.stack_details.get("Tags"): print("Update original tags") From 3777f8950e150f1d02179ea2015f59fb8a696162 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Tue, 11 Aug 2020 09:36:31 +1000 Subject: [PATCH 36/40] feat(cloudformation): update the tag asking process --- .../helper/cloudformationargs.py | 57 +++++++++---------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/fzfaws/cloudformation/helper/cloudformationargs.py b/fzfaws/cloudformation/helper/cloudformationargs.py index a886913..310898c 100644 --- a/fzfaws/cloudformation/helper/cloudformationargs.py +++ b/fzfaws/cloudformation/helper/cloudformationargs.py @@ -7,7 +7,7 @@ from fzfaws.cloudwatch import Cloudwatch from fzfaws.iam import IAM from fzfaws.sns import SNS -from fzfaws.utils import Pyfzf, prompt_style +from fzfaws.utils import Pyfzf, URLQueryStringValidator, prompt_style class CloudformationArgs: @@ -293,37 +293,32 @@ def _set_tags(self, update: bool = False) -> None: :param update: determine if is updating the stack, it will show different prompt :type update: bool, optional """ - tag_list: List[Dict[str, str]] = [] + print("Tag format should be a URL Query alike string (e.g. foo=boo&name=yes)") - if update: - if self.cloudformation.stack_details.get("Tags"): - print("Update original tags") - print("Skip the value to use previous value") - print('Enter "deletetag" in any field to remove a tag') - for tag in self.cloudformation.stack_details["Tags"]: - tag_key = input("Key(%s): " % tag["Key"]) - if not tag_key: - tag_key = tag["Key"] - tag_value = input("Value(%s): " % tag["Value"]) - if not tag_value: - tag_value = tag["Value"] - if tag_key == "deletetag" or tag_value == "deletetag": - continue - tag_list.append({"Key": tag_key, "Value": tag_value}) - print("Enter new tags below") - print("Enter an empty value to stop entering for new tags") - while True: - tag_name: str = input("TagName: ") - if not tag_name: - break - tag_value: str = input("TagValue: ") - if not tag_value: - break - tag_list.append({"Key": tag_name, "Value": tag_value}) - if tag_list: - self._extra_args["Tags"] = tag_list - elif not tag_list and update: - self._extra_args["Tags"] = [] + questions: List[Dict[str, Any]] = [ + { + "type": "input", + "message": "Tags", + "name": "answer", + "validate": URLQueryStringValidator, + } + ] + if update and self.cloudformation.stack_details.get("Tags"): + default_tag_value: List[str] = [] + for tag in self.cloudformation.stack_details.get("Tags", []): + default_tag_value.append( + "%s=%s" % (tag.get("Key", ""), tag.get("Value", "")) + ) + questions[0]["default"] = "&".join(default_tag_value) + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + tag_list: List[Dict[str, str]] = [] + for tag in result.get("answer", "").split("&"): + if tag != "": + tag_name, tag_value = tag.split("=") + tag_list.append({"Key": tag_name, "Value": tag_value}) + self._extra_args["Tags"] = tag_list @property def extra_args(self): From 89e57f92cda38f831d331ffd1fa71e2161ad01a0 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Tue, 11 Aug 2020 11:05:42 +1000 Subject: [PATCH 37/40] test(cloudformation): update test --- .../helper/cloudformationargs.py | 8 +- .../cloudformation/test_cloudformationargs.py | 340 +++++++++++------- 2 files changed, 212 insertions(+), 136 deletions(-) diff --git a/fzfaws/cloudformation/helper/cloudformationargs.py b/fzfaws/cloudformation/helper/cloudformationargs.py index 310898c..904cac4 100644 --- a/fzfaws/cloudformation/helper/cloudformationargs.py +++ b/fzfaws/cloudformation/helper/cloudformationargs.py @@ -172,11 +172,11 @@ def _set_creation(self) -> None: def _set_rollback(self, update: bool = False) -> None: """Set rollback configuration for cloudformation. - :param update: show previous values if true + :param update: show previous values and set default values if true :type update: bool, optional """ cloudwatch = Cloudwatch(self.cloudformation.profile, self.cloudformation.region) - header: str = "select a cloudwatch alarm to monitor the stack" + header: str = "select cloudwatch alarms to monitor the stack" questions: List[Dict[str, str]] = [ {"type": "input", "message": "MonitoringTimeInMinutes", "name": "answer"} ] @@ -213,7 +213,7 @@ def _set_notification(self, update: bool = False) -> None: sns = SNS( profile=self.cloudformation.profile, region=self.cloudformation.region ) - header = "select sns topic to notify" + header = "select sns topics to notify" if update: header += "\nOriginal value: %s" % self.cloudformation.stack_details.get( "NotificationARNs" @@ -290,7 +290,7 @@ def _set_tags(self, update: bool = False) -> None: } ] - :param update: determine if is updating the stack, it will show different prompt + :param update: determine if is updating the stack, it will set default tag value :type update: bool, optional """ print("Tag format should be a URL Query alike string (e.g. foo=boo&name=yes)") diff --git a/tests/cloudformation/test_cloudformationargs.py b/tests/cloudformation/test_cloudformationargs.py index 1f396a0..bf2d0f1 100644 --- a/tests/cloudformation/test_cloudformationargs.py +++ b/tests/cloudformation/test_cloudformationargs.py @@ -1,16 +1,16 @@ -from fzfaws.iam.iam import IAM import io import json import os import sys import unittest -from unittest.mock import call, patch +from unittest.mock import ANY, call, patch from fzfaws.cloudformation import Cloudformation from fzfaws.cloudformation.helper.cloudformationargs import CloudformationArgs from fzfaws.cloudwatch import Cloudwatch -from fzfaws.sns.sns import SNS -from fzfaws.utils import Pyfzf +from fzfaws.iam.iam import IAM +from fzfaws.sns import SNS +from fzfaws.utils import Pyfzf, prompt_style class TestCloudformationArgs(unittest.TestCase): @@ -28,8 +28,7 @@ def test_constructor(self): self.assertEqual(self.cloudformationargs.update_termination, None) self.assertIsInstance(self.cloudformationargs.cloudformation, Cloudformation) - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") + @patch("fzfaws.cloudformation.helper.cloudformationargs.prompt") @patch.object(CloudformationArgs, "_set_creation") @patch.object(CloudformationArgs, "_set_rollback") @patch.object(CloudformationArgs, "_set_notification") @@ -44,18 +43,19 @@ def test_set_extra_args( mocked_notify, mocked_rollback, mocked_create, - mocked_append, - mocked_execute, + mocked_prompt, ): # normal test - mocked_execute.return_value = [ - "Tags", - "Permissions", - "StackPolicy", - "Notifications", - "RollbackConfiguration", - "CreationOption", - ] + mocked_prompt.return_value = { + "answer": [ + "Tags", + "Permissions", + "StackPolicy", + "Notifications", + "RollbackConfiguration", + "CreationOption", + ] + } self.cloudformationargs.set_extra_args() mocked_tags.assert_called_once_with(False) mocked_perm.assert_called_once_with(False) @@ -63,15 +63,23 @@ def test_set_extra_args( mocked_rollback.assert_called_once_with(False) mocked_policy.assert_called_once_with(False, False) mocked_create.assert_called_once_with() - mocked_append.assert_has_calls( + mocked_prompt.assert_called_once_with( [ - call("Tags\n"), - call("Permissions\n"), - call("StackPolicy\n"), - call("Notifications\n"), - call("RollbackConfiguration\n"), - call("CreationOption\n"), - ] + { + "type": "checkbox", + "name": "answer", + "message": "Select options to configure", + "choices": [ + {"name": "Tags"}, + {"name": "Permissions"}, + {"name": "Notifications"}, + {"name": "RollbackConfiguration"}, + {"name": "StackPolicy"}, + {"name": "CreationOption"}, + ], + } + ], + style=prompt_style, ) mocked_tags.reset_mock() @@ -80,7 +88,7 @@ def test_set_extra_args( mocked_rollback.reset_mock() mocked_policy.reset_mock() mocked_create.reset_mock() - mocked_append.reset_mock() + mocked_prompt.reset_mock() # update test self.cloudformationargs.set_extra_args(update=True, search_from_root=True) mocked_tags.assert_called_once_with(True) @@ -89,14 +97,22 @@ def test_set_extra_args( mocked_rollback.assert_called_once_with(True) mocked_policy.assert_called_once_with(True, True) mocked_create.assert_called_once_with() - mocked_append.assert_has_calls( + mocked_prompt.assert_called_once_with( [ - call("Tags\n"), - call("Permissions\n"), - call("StackPolicy\n"), - call("Notifications\n"), - call("RollbackConfiguration\n"), - ] + { + "type": "checkbox", + "name": "answer", + "message": "Select options to configure", + "choices": [ + {"name": "Tags"}, + {"name": "Permissions"}, + {"name": "Notifications"}, + {"name": "RollbackConfiguration"}, + {"name": "StackPolicy"}, + ], + } + ], + style=prompt_style, ) mocked_tags.reset_mock() @@ -105,7 +121,7 @@ def test_set_extra_args( mocked_rollback.reset_mock() mocked_policy.reset_mock() mocked_create.reset_mock() - mocked_append.reset_mock() + mocked_prompt.reset_mock() # dryrun test self.cloudformationargs.set_extra_args(dryrun=True) mocked_tags.assert_called_once_with(False) @@ -114,92 +130,122 @@ def test_set_extra_args( mocked_rollback.assert_called_once_with(False) mocked_policy.assert_called_once_with(False, False) mocked_create.assert_called_once_with() - mocked_append.assert_has_calls( + mocked_prompt.assert_called_once_with( [ - call("Tags\n"), - call("Permissions\n"), - call("Notifications\n"), - call("RollbackConfiguration\n"), - ] + { + "type": "checkbox", + "name": "answer", + "message": "Select options to configure", + "choices": [ + {"name": "Tags"}, + {"name": "Permissions"}, + {"name": "Notifications"}, + {"name": "RollbackConfiguration"}, + ], + } + ], + style=prompt_style, ) - @patch("builtins.input") - @patch.object(Pyfzf, "execute_fzf") - @patch.object(Pyfzf, "append_fzf") - def test__set_creation(self, mocked_append, mocked_execute, mocked_input): + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.cloudformationargs.set_extra_args) + + @patch("fzfaws.cloudformation.helper.cloudformationargs.prompt") + def test__set_creation(self, mocked_prompt): self.cloudformationargs._extra_args = {} # normal test - mocked_execute.return_value = [ - "RollbackOnFailure", - "TimeoutInMinutes", - "EnableTerminationProtection", - ] - mocked_input.return_value = 1 + mocked_prompt.return_value = { + "selected_options": [ + "RollbackOnFailure", + "TimeoutInMinutes", + "EnableTerminationProtection", + ], + "rollback": "True", + "timeout": "5", + "termination": "False", + } self.cloudformationargs._set_creation() - mocked_append.assert_has_calls( - [ - call("RollbackOnFailure\n"), - call("TimeoutInMinutes\n"), - call("EnableTerminationProtection\n"), - call("True\n"), - call("False\n"), - call("True\n"), - call("False\n"), - ] - ) - mocked_execute.assert_has_calls( - [ - call( - empty_allow=True, - print_col=1, - multi_select=True, - header="select options to configure", - ), - call( - empty_allow=True, - print_col=1, - header="roll back on failue? (Default: True)", - ), - call( - empty_allow=True, - print_col=1, - header="enable termination protection? (Default: False)", - ), - ] - ) self.assertEqual( - self.cloudformationargs.extra_args, + self.cloudformationargs._extra_args, { - "OnFailure": "DO_NOTHING", - "TimeoutInMinutes": 1, + "OnFailure": "ROLLBACK", + "TimeoutInMinutes": 5, "EnableTerminationProtection": False, }, ) + mocked_prompt.assert_called_once_with( + [ + { + "type": "checkbox", + "name": "selected_options", + "message": "Select creation options to configure", + "choices": [ + {"name": "RollbackOnFailure"}, + {"name": "TimeoutInMinutes"}, + {"name": "EnableTerminationProtection"}, + ], + }, + { + "type": "rawlist", + "name": "rollback", + "message": "Roll back on failure?", + "choices": ["True", "False"], + "when": ANY, + }, + { + "type": "input", + "name": "timeout", + "message": "Specify number of minutes before timeout", + "when": ANY, + }, + { + "type": "rawlist", + "name": "termination", + "message": "Enable termination protection?", + "choices": ["True", "False"], + "when": ANY, + }, + ], + style=prompt_style, + ) - @patch("builtins.input") - @patch.object(Cloudwatch, "set_arns") - def test__set_rollback(self, mocked_arn, mocked_input): + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.cloudformationargs._set_creation) + + @patch("fzfaws.cloudformation.helper.cloudformationargs.prompt") + @patch("fzfaws.cloudformation.helper.cloudformationargs.Cloudwatch") + def test__set_rollback(self, MockedCloudwatch, mocked_prompt): + self.cloudformationargs._extra_args = {} - self.capturedOutput.truncate(0) - self.capturedOutput.seek(0) # normal test + cloudwatch = MockedCloudwatch.return_value + cloudwatch.arns = ["hello"] + mocked_prompt.return_value = {"answer": "5"} self.cloudformationargs._set_rollback(update=False) - self.assertEqual( - self.capturedOutput.getvalue(), - "--------------------------------------------------------------------------------\nSelected arns: ['']\n", - ) - mocked_arn.assert_called_once_with( + cloudwatch.set_arns.assert_called_once_with( empty_allow=True, - header="select a cloudwatch alarm to monitor the stack", + header="select cloudwatch alarms to monitor the stack", multi_select=True, ) - mocked_input.assert_called_once_with("MonitoringTimeInMinutes(Default: 0): ") + mocked_prompt.assert_called_once_with( + [{"type": "input", "message": "MonitoringTimeInMinutes", "name": "answer"}], + style=prompt_style, + ) + self.assertEqual( + self.cloudformationargs._extra_args, + { + "RollbackConfiguration": { + "RollbackTriggers": [ + {"Arn": "hello", "Type": "AWS::CloudWatch::Alarm"} + ], + "MonitoringTimeInMinutes": 5, + } + }, + ) - mocked_arn.reset_mock() - mocked_input.reset_mock() - self.capturedOutput.truncate(0) - self.capturedOutput.seek(0) + cloudwatch.set_arns.reset_mock() + mocked_prompt.reset_mock() # update test self.cloudformationargs.cloudformation.stack_details = { "RollbackConfiguration": { @@ -208,32 +254,46 @@ def test__set_rollback(self, mocked_arn, mocked_input): } } self.cloudformationargs._set_rollback(update=True) - self.assertEqual( - self.capturedOutput.getvalue(), - "--------------------------------------------------------------------------------\nSelected arns: ['']\n", - ) - mocked_arn.assert_called_once_with( + cloudwatch.set_arns.assert_called_once_with( empty_allow=True, - header="select a cloudwatch alarm to monitor the stack\nOriginal value: 111111", + header="select cloudwatch alarms to monitor the stack\nOriginal value: 111111", multi_select=True, ) - mocked_input.assert_called_once_with("MonitoringTimeInMinutes(Original: 1): ") + mocked_prompt.assert_called_once_with( + [ + { + "type": "input", + "message": "MonitoringTimeInMinutes", + "name": "answer", + "default": "1", + } + ], + style=prompt_style, + ) + + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.cloudformationargs._set_rollback) - @patch.object(SNS, "set_arns") - def test__set_notification(self, mocked_arn): + @patch("fzfaws.cloudformation.helper.cloudformationargs.SNS") + def test__set_notification(self, MockedSNS): + sns = MockedSNS.return_value + sns.arns = ["hello"] self.cloudformationargs._set_notification() - mocked_arn.assert_called_once_with( - empty_allow=True, header="select sns topic to notify", multi_select=True + sns.set_arns.assert_called_once_with( + empty_allow=True, header="select sns topics to notify", multi_select=True + ) + self.assertEqual( + self.cloudformationargs._extra_args, {"NotificationARNs": ["hello"]} ) - mocked_arn.reset_mock() + sns.set_arns.reset_mock() self.cloudformationargs.cloudformation.stack_details = { "NotificationARNs": "111111" } self.cloudformationargs._set_notification(update=True) - mocked_arn.assert_called_once_with( + sns.set_arns.assert_called_once_with( empty_allow=True, - header="select sns topic to notify\nOriginal value: 111111", + header="select sns topics to notify\nOriginal value: 111111", multi_select=True, ) @@ -294,40 +354,56 @@ def test__set_policy(self, mocked_file): header="select the policy document you would like to use", ) - @patch.object(IAM, "set_arns") - def test__set_permissions(self, mocked_arn): + @patch("fzfaws.cloudformation.helper.cloudformationargs.IAM") + def test__set_permissions(self, MockedIAM): + iam = MockedIAM.return_value + iam.arns = ["hello"] self.cloudformationargs._set_permissions() - mocked_arn.assert_called_once_with( + iam.set_arns.assert_called_once_with( header="choose an IAM role to explicitly define CloudFormation's permissions\nNote: only IAM role can be assumed by CloudFormation is listed", service="cloudformation.amazonaws.com", ) + self.assertEqual(self.cloudformationargs.extra_args, {"RoleARN": "hello"}) - mocked_arn.reset_mock() + iam.set_arns.reset_mock() self.cloudformationargs.cloudformation.stack_details = {"RoleARN": "111111"} self.cloudformationargs._set_permissions(update=True) - mocked_arn.assert_called_once_with( + iam.set_arns.assert_called_once_with( header="select a role Choose an IAM role to explicitly define CloudFormation's permissions\nOriginal value: 111111", service="cloudformation.amazonaws.com", ) - @patch("builtins.input") - def test_set_tags(self, mocked_input): - self.cloudformationargs._extra_args = {} - mocked_input.return_value = "" + @patch("fzfaws.cloudformation.helper.cloudformationargs.prompt") + def test_set_tags(self, mocked_prompt): + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.cloudformationargs._set_tags) + + mocked_prompt.reset_mock() + mocked_prompt.return_value = {"answer": "foo=boo&yes=no"} self.cloudformationargs._set_tags() - mocked_input.assert_has_calls( - [call("TagName: "),] + self.assertEqual( + self.cloudformationargs._extra_args, + {"Tags": [{"Key": "foo", "Value": "boo"}, {"Key": "yes", "Value": "no"},]}, + ) + mocked_prompt.assert_called_once_with( + [{"type": "input", "message": "Tags", "name": "answer", "validate": ANY,}], + style=prompt_style, ) - self.assertEqual(self.cloudformationargs._extra_args, {}) - mocked_input.reset_mock() - mocked_input.return_value = "" + mocked_prompt.reset_mock() self.cloudformationargs.cloudformation.stack_details = { - "Tags": [{"Key": "foo", "Value": "boo"}] + "Tags": [{"Key": "foo", "Value": "boo"}, {"Key": "hello", "Value": "yes"}] } self.cloudformationargs._set_tags(update=True) - mocked_input.assert_has_calls([call("Key(foo): "), call("Value(boo): ")]) - self.assertEqual( - self.cloudformationargs._extra_args, - {"Tags": [{"Key": "foo", "Value": "boo"}]}, + mocked_prompt.assert_called_once_with( + [ + { + "type": "input", + "message": "Tags", + "name": "answer", + "validate": ANY, + "default": "foo=boo&hello=yes", + } + ], + style=prompt_style, ) From f9e97469d8250a1cc71be88adb1de27bc6b3dfc4 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 12 Aug 2020 10:28:39 +1000 Subject: [PATCH 38/40] feat(cloudformation): prompt list of param to select before processing --- .../cloudformation/helper/paramprocessor.py | 82 ++++++++++++++++--- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/fzfaws/cloudformation/helper/paramprocessor.py b/fzfaws/cloudformation/helper/paramprocessor.py index 7492f93..fe1df42 100644 --- a/fzfaws/cloudformation/helper/paramprocessor.py +++ b/fzfaws/cloudformation/helper/paramprocessor.py @@ -4,9 +4,11 @@ """ from typing import Any, Dict, List, Optional, Union +from PyInquirer import prompt + from fzfaws.ec2 import EC2 from fzfaws.route53 import Route53 -from fzfaws.utils import Pyfzf, Spinner, check_dict_value_in_list, search_dict_in_list +from fzfaws.utils import Pyfzf, Spinner, prompt_style class ParamProcessor: @@ -40,8 +42,6 @@ def __init__( self.ec2 = EC2(profile, region) self.route53 = Route53(profile, region) self.params: Dict[str, Any] = params - self.original_params: List[Dict[str, Any]] = original_params - self.processed_params: List[Dict[str, Any]] = [] self._aws_specific_param: List[str] = [ "AWS::EC2::AvailabilityZone::Name", "AWS::EC2::Instance::Id", @@ -63,6 +63,11 @@ def __init__( "List", "List", ] + self.original_params: Dict[str, Any] = { + param["ParameterKey"]: param.get("ParameterValue", "") + for param in original_params + } + self.processed_params: List[Dict[str, Any]] = [] def process_stack_params(self) -> None: """Process the template file parameters. @@ -70,9 +75,9 @@ def process_stack_params(self) -> None: Loop through the keys in the loaded dict object of params and leverage self._get_user_input to get user input through fzf or cmd input """ - print("Enter parameters specified in your template below") + selected_params: List[str] = self._get_param_selection() - for parameter_key in self.params: + for parameter_key in selected_params: print(80 * "-") default_value: str = "" param_header: str = "" @@ -99,13 +104,9 @@ def process_stack_params(self) -> None: ): param_header += "For list type parameters, use comma to sperate items(e.g. values: value1, value2)" - if check_dict_value_in_list( - parameter_key, self.original_params, "ParameterKey" - ): + if parameter_key in self.original_params: # check if there is original value i.e. udpating the stack - original_value: str = search_dict_in_list( - parameter_key, self.original_params, "ParameterKey" - ).get("ParameterValue", "") + original_value = self.original_params[parameter_key] parameter_value = self._get_user_input( parameter_key, parameter_type, @@ -151,8 +152,30 @@ def process_stack_params(self) -> None: # it's hard for user to track what they have gone through # hence printing the header information to terminal as well print(param_header.rstrip()) + self.params.pop(parameter_key, None) print("ParameterValue: %s" % parameter_value) + # add all the unproccessed parameter to the processed_params list + for parameter_key in self.params: + if parameter_key in self.original_params: + self.processed_params.append( + { + "ParameterKey": parameter_key, + "ParameterValue": self.original_params[parameter_key], + } + ) + elif self.params[parameter_key].get("Default"): + self.processed_params.append( + { + "ParameterKey": parameter_key, + "ParameterValue": self.params[parameter_key]["Default"], + } + ) + else: + self.processed_params.append( + {"ParameterKey": parameter_key, "ParameterValue": ""} + ) + def _get_user_input( self, parameter_key: str, @@ -319,3 +342,40 @@ def _get_list_param_value(self, type_name: str, param_header: str) -> List[str]: return list( fzf.execute_fzf(multi_select=True, empty_allow=True, header=param_header) ) + + def _get_param_selection(self) -> List[str]: + """Prompt user to select parameter to edit. + + :return: return the selected list of parameter + :rtype: List[str] + """ + choices: List[Dict[str, str]] = [] + for parameter_key in self.params: + if not self.original_params: + title = ( + "%s: %s" + % (parameter_key, self.params[parameter_key].get("Default")) + if self.params[parameter_key].get("Default") + else parameter_key + ) + choices.append({"name": title}) + else: + original_value = self.original_params.get(parameter_key) + if original_value: + title = "%s: %s" % (parameter_key, original_value,) + choices.append({"name": title}) + else: + choices.append({"name": parameter_key}) + questions: List[Dict[str, Any]] = [ + { + "type": "checkbox", + "name": "answer", + "message": "Select parameters to edit", + "choices": choices, + "filter": lambda x: [i.split(": ")[0] for i in x], + } + ] + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + return result.get("answer", []) From d867d3c6594ec6ffaf91c1a3567cd065efe6c147 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 12 Aug 2020 11:21:23 +1000 Subject: [PATCH 39/40] feat(cloudformation): update user input for cloudformation params --- .../cloudformation/helper/paramprocessor.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/fzfaws/cloudformation/helper/paramprocessor.py b/fzfaws/cloudformation/helper/paramprocessor.py index fe1df42..c4004f7 100644 --- a/fzfaws/cloudformation/helper/paramprocessor.py +++ b/fzfaws/cloudformation/helper/paramprocessor.py @@ -78,7 +78,6 @@ def process_stack_params(self) -> None: selected_params: List[str] = self._get_param_selection() for parameter_key in selected_params: - print(80 * "-") default_value: str = "" param_header: str = "" @@ -154,6 +153,7 @@ def process_stack_params(self) -> None: print(param_header.rstrip()) self.params.pop(parameter_key, None) print("ParameterValue: %s" % parameter_value) + print(80 * "-") # add all the unproccessed parameter to the processed_params list for parameter_key in self.params: @@ -182,7 +182,7 @@ def _get_user_input( parameter_type: str, param_header: str, value_type: str = None, - default: str = None, + default: str = "", ) -> Union[str, List[str]]: """Get user input. @@ -227,27 +227,24 @@ def _get_user_input( user_input = self._get_list_param_value(parameter_type, param_header) else: print(param_header.rstrip()) - if not value_type: - user_input = input("%s: " % parameter_key) - elif value_type == "Default": - user_input = input("%s(Default: %s): " % (parameter_key, default)) - elif value_type == "Original": - user_input = input("%s(Original: %s): " % (parameter_key, default)) - if not user_input and default: - return default - elif user_input == "''": - return "" - elif user_input == '""': - return "" - else: - return user_input + questions = [ + {"type": "input", "message": parameter_key, "name": "answer",} + ] + if default: + questions[0]["default"] = default + result = prompt(questions, style=prompt_style) + if not result: + raise KeyboardInterrupt + user_input = result.get("answer", "") + return user_input + return user_input if user_input else default def _print_parameter_key( self, parameter_key: str, value_type: str = None, default: str = None ) -> str: """Print parameter_key.""" if value_type: - return "choose a value for %s(%s: %s)" % ( + return "choose a value for %s (%s: %s)" % ( parameter_key, value_type, default, From f41f2d72d250012a0cb70f1697243401398cdc68 Mon Sep 17 00:00:00 2001 From: Kevin Zhuang Date: Wed, 12 Aug 2020 11:21:41 +1000 Subject: [PATCH 40/40] test(cloudformation): update test to reflect new changes on paramprocessor --- tests/cloudformation/test_paramprocessor.py | 254 ++++++++++++++------ 1 file changed, 174 insertions(+), 80 deletions(-) diff --git a/tests/cloudformation/test_paramprocessor.py b/tests/cloudformation/test_paramprocessor.py index a086fcb..9ba4278 100644 --- a/tests/cloudformation/test_paramprocessor.py +++ b/tests/cloudformation/test_paramprocessor.py @@ -1,33 +1,37 @@ -from fzfaws.route53.route53 import Route53 -from fzfaws.ec2.ec2 import EC2 -from fzfaws.utils.session import BaseSession -from fzfaws.utils.pyfzf import Pyfzf -import os import io +import json +import os +from pathlib import Path import sys import unittest -from unittest.mock import PropertyMock, call, patch -from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor -from fzfaws.utils import FileLoader +from unittest.mock import ANY, PropertyMock, call, patch + import boto3 from botocore.stub import Stubber -import json -from pathlib import Path + +from fzfaws.cloudformation.helper.paramprocessor import ParamProcessor +from fzfaws.ec2.ec2 import EC2 +from fzfaws.route53.route53 import Route53 +from fzfaws.utils import FileLoader +from fzfaws.utils.pyfzf import Pyfzf +from fzfaws.utils.session import BaseSession class TestCloudformationParams(unittest.TestCase): def setUp(self): - data_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../data/cloudformation_template.yaml", + data_path = ( + Path(__file__) + .resolve() + .parent.joinpath("../data/cloudformation_template.yaml") ) - fileloader = FileLoader(path=data_path) + fileloader = FileLoader(path=str(data_path)) params = fileloader.process_yaml_file()["dictBody"].get("Parameters", {}) config_path = Path(__file__).resolve().parent.joinpath("../data/fzfaws.yml") fileloader.load_config_file(config_path=str(config_path)) self.paramprocessor = ParamProcessor(params=params) self.capturedOutput = io.StringIO() sys.stdout = self.capturedOutput + self.maxDiff = None def tearDown(self): sys.stdout = sys.__stdout__ @@ -38,7 +42,7 @@ def test_constructor(self): self.assertEqual(self.paramprocessor.route53.profile, "default") self.assertEqual(self.paramprocessor.route53.region, "us-east-1") self.assertIsInstance(self.paramprocessor.params, dict) - self.assertEqual(self.paramprocessor.original_params, []) + self.assertEqual(self.paramprocessor.original_params, {}) self.assertEqual(self.paramprocessor.processed_params, []) paramprocessor = ParamProcessor(profile="root", region="us-east-1") @@ -48,11 +52,21 @@ def test_constructor(self): self.assertEqual(paramprocessor.route53.region, "us-east-1") self.assertEqual(paramprocessor.params, {}) self.assertEqual(paramprocessor.processed_params, []) - self.assertEqual(paramprocessor.original_params, []) + self.assertEqual(paramprocessor.original_params, {}) + @patch.object(ParamProcessor, "_get_param_selection") @patch.object(ParamProcessor, "_get_user_input") - def test_process_stack_params(self, mocked_input): + def test_process_stack_params1(self, mocked_input, mocked_selection): mocked_input.return_value = "111111" + mocked_selection.return_value = [ + "InstanceRole", + "LatestAmiId", + "SubnetId", + "SecurityGroups", + "KeyName", + "WebServer", + "InstanceType", + ] self.capturedOutput.truncate(0) self.capturedOutput.seek(0) @@ -97,58 +111,21 @@ def test_process_stack_params(self, mocked_input): ] ) - mocked_input.return_value = "222222" - self.paramprocessor.processed_params = [] - self.paramprocessor.original_params = [ - {"ParameterKey": "KeyName", "ParameterValue": "fooboo"}, - {"ParameterKey": "SecurityGroups", "ParameterValue": "sg-111111",}, - {"ParameterKey": "WebServer", "ParameterValue": "No"}, - { - "ParameterKey": "LatestAmiId", - "ParameterValue": "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2", - "ResolvedValue": "ami-08fdde86b93accf1c", - }, - {"ParameterKey": "InstanceRole", "ParameterValue": ""}, - {"ParameterKey": "SubnetId", "ParameterValue": "subnet-111111"}, - {"ParameterKey": "InstanceType", "ParameterValue": "t2.micro"}, - ] - self.capturedOutput.truncate(0) - self.capturedOutput.seek(0) - self.paramprocessor.process_stack_params() - self.assertEqual( - self.paramprocessor.processed_params, - [ - {"ParameterKey": "InstanceRole", "ParameterValue": "222222"}, - {"ParameterKey": "LatestAmiId", "ParameterValue": "222222"}, - {"ParameterKey": "SubnetId", "ParameterValue": "222222"}, - {"ParameterKey": "SecurityGroups", "ParameterValue": "222222"}, - {"ParameterKey": "KeyName", "ParameterValue": "222222"}, - {"ParameterKey": "WebServer", "ParameterValue": "222222"}, - {"ParameterKey": "InstanceType", "ParameterValue": "222222"}, - ], - ) - self.assertRegex( - self.capturedOutput.getvalue(), - r"Description: The subnet this instance should be deployed to", - ) - self.assertRegex( - self.capturedOutput.getvalue(), - r"ConstraintDescription: must be the name of an existing EC2 KeyPair", - ) - self.assertRegex( - self.capturedOutput.getvalue(), r"ParameterValue: 222222", - ) - mocked_input.assert_called_with( - "InstanceType", - "String", - "Description: EC2 instance type\nConstraintDescription: must be a valid EC2 instance type\nType: String\n", - "Original", - "t2.micro", - ) - + @patch.object(ParamProcessor, "_get_param_selection") + @patch.object(ParamProcessor, "_get_user_input") + def test_process_stack_params2(self, mocked_input, mocked_selection): mocked_input.return_value = ["111111", "222222"] + mocked_selection.return_value = [ + "InstanceRole", + "LatestAmiId", + "SubnetId", + "SecurityGroups", + "KeyName", + "WebServer", + "InstanceType", + ] self.paramprocessor.processed_params = [] - self.paramprocessor.original_params = [] + self.paramprocessor.original_params = {} self.paramprocessor.process_stack_params() self.assertEqual( self.paramprocessor.processed_params, @@ -163,7 +140,35 @@ def test_process_stack_params(self, mocked_input): ], ) - @patch("builtins.input") + @patch.object(ParamProcessor, "_get_param_selection") + @patch.object(ParamProcessor, "_get_user_input") + def test_process_stack_params3(self, mocked_input, mocked_selection): + mocked_selection.return_value = [ + "SecurityGroups", + "SubnetId", + "KeyName", + ] + mocked_input.return_value = ["111111"] + self.paramprocessor.processed_params = [] + self.paramprocessor.original_params = {} + self.paramprocessor.process_stack_params() + self.assertEqual( + self.paramprocessor.processed_params, + [ + {"ParameterKey": "SecurityGroups", "ParameterValue": "111111"}, + {"ParameterKey": "SubnetId", "ParameterValue": "111111"}, + {"ParameterKey": "KeyName", "ParameterValue": "111111"}, + {"ParameterKey": "InstanceRole", "ParameterValue": ""}, + { + "ParameterKey": "LatestAmiId", + "ParameterValue": "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2", + }, + {"ParameterKey": "WebServer", "ParameterValue": "No"}, + {"ParameterKey": "InstanceType", "ParameterValue": "t2.micro"}, + ], + ) + + @patch("fzfaws.cloudformation.helper.paramprocessor.prompt") @patch.object(ParamProcessor, "_get_list_param_value") @patch.object(ParamProcessor, "_get_selected_param_value") @patch.object(Pyfzf, "execute_fzf") @@ -176,15 +181,51 @@ def test_get_user_input( mocked_execute, mocked_select, mocked_list, - mocked_input, + mocked_prompt, ): + mocked_prompt.return_value = {} + self.assertRaises( + KeyboardInterrupt, + self.paramprocessor._get_user_input, + "InstanceRole", + "String", + "foo boo", + ) + + mocked_prompt.reset_mock() # normal var with no default value test - mocked_input.return_value = "111111" + mocked_prompt.return_value = {"answer": "111111"} result = self.paramprocessor._get_user_input( "InstanceRole", "String", "foo boo" ) - mocked_input.assert_called_once_with("InstanceRole: ") + mocked_prompt.assert_called_once_with( + [{"type": "input", "message": "InstanceRole", "name": "answer"}], style=ANY + ) + mocked_print.assert_not_called() + mocked_append.assert_not_called() + mocked_execute.assert_not_called() + mocked_select.assert_not_called() + mocked_list.assert_not_called() + self.assertEqual(result, "111111") + + mocked_prompt.reset_mock() + # normal var with no default value test + mocked_prompt.return_value = {"answer": "111111"} + result = self.paramprocessor._get_user_input( + "InstanceRole", "String", "foo boo", default="wtf" + ) + mocked_prompt.assert_called_once_with( + [ + { + "type": "input", + "message": "InstanceRole", + "name": "answer", + "default": "wtf", + } + ], + style=ANY, + ) mocked_print.assert_not_called() mocked_append.assert_not_called() mocked_execute.assert_not_called() @@ -192,7 +233,7 @@ def test_get_user_input( mocked_list.assert_not_called() self.assertEqual(result, "111111") - mocked_input.reset_mock() + mocked_prompt.reset_mock() mocked_print.reset_mock() mocked_append.reset_mock() mocked_execute.reset_mock() @@ -211,10 +252,10 @@ def test_get_user_input( ) mocked_select.assert_not_called() mocked_list.assert_not_called() - mocked_input.assert_not_called() + mocked_prompt.assert_not_called() self.assertEqual(result, "111111") - mocked_input.reset_mock() + mocked_prompt.reset_mock() mocked_print.reset_mock() mocked_append.reset_mock() mocked_execute.reset_mock() @@ -231,10 +272,10 @@ def test_get_user_input( mocked_append.assert_not_called() mocked_execute.assert_not_called() mocked_list.assert_not_called() - mocked_input.assert_not_called() + mocked_prompt.assert_not_called() self.assertEqual(result, "111111") - mocked_input.reset_mock() + mocked_prompt.reset_mock() mocked_print.reset_mock() mocked_append.reset_mock() mocked_execute.reset_mock() @@ -245,26 +286,26 @@ def test_get_user_input( result = self.paramprocessor._get_user_input( "SecurityGroups", "List", "foo boo" ) - mocked_print.assert_called_with("SecurityGroups", None, None) + mocked_print.assert_called_with("SecurityGroups", None, "") mocked_list.assert_called_once_with( "List", "foo boo" ) mocked_append.assert_not_called() mocked_execute.assert_not_called() mocked_select.assert_not_called() - mocked_input.assert_not_called() + mocked_prompt.assert_not_called() self.assertEqual(result, "111111") def test_print_parameter_key(self): result = self.paramprocessor._print_parameter_key( "SecurityGroups", "Default", "111111" ) - self.assertEqual(result, "choose a value for SecurityGroups(Default: 111111)") + self.assertEqual(result, "choose a value for SecurityGroups (Default: 111111)") result = self.paramprocessor._print_parameter_key( "SecurityGroups", "Original", "111111" ) - self.assertEqual(result, "choose a value for SecurityGroups(Original: 111111)") + self.assertEqual(result, "choose a value for SecurityGroups (Original: 111111)") result = self.paramprocessor._print_parameter_key("SecurityGroups") self.assertEqual(result, "choose a value for SecurityGroups") @@ -381,3 +422,56 @@ def test_get_list_param_value( ) mocked_zone.assert_called_once() self.assertEqual(result, [""]) + + @patch("fzfaws.cloudformation.helper.paramprocessor.prompt") + def test_get_param_selection(self, mocked_prompt): + mocked_prompt.return_value = {} + self.assertRaises(KeyboardInterrupt, self.paramprocessor._get_param_selection) + + mocked_prompt.reset_mock() + mocked_prompt.return_value = { + "answer": [ + "InstanceRole", + "LatestAmiId", + "SubnetId", + "SecurityGroups", + "KeyName", + "WebServer", + "InstanceType", + ] + } + result = self.paramprocessor._get_param_selection() + self.assertEqual( + result, + [ + "InstanceRole", + "LatestAmiId", + "SubnetId", + "SecurityGroups", + "KeyName", + "WebServer", + "InstanceType", + ], + ) + mocked_prompt.assert_called_once_with( + [ + { + "type": "checkbox", + "name": "answer", + "message": "Select parameters to edit", + "choices": [ + {"name": "InstanceRole"}, + { + "name": "LatestAmiId: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" + }, + {"name": "SubnetId"}, + {"name": "SecurityGroups"}, + {"name": "KeyName"}, + {"name": "WebServer: No"}, + {"name": "InstanceType: t2.micro"}, + ], + "filter": ANY, + } + ], + style=ANY, + )