diff --git a/.gitignore b/.gitignore index a6596ff41..970cad0c5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ dist .newt.yml tox.ini test-reports/* +config.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..0cc0baa2e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.4.0 # Use the ref you want to point at + hooks: + - id: trailing-whitespace + - id: autopep8-wrapper + - id: check-ast + - id: check-case-conflict + - id: check-yaml + - id: flake8 + args: [] + - id: pretty-format-json + args: ["--autofix"] + +- repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + language_version: python3.7 + +- repo: local + hooks: + - id: python-bandit-vulnerability-check + name: bandit + entry: bandit + args: ['--ini', 'tox.ini', '-r', 'consoleme'] + language: system + pass_filenames: false diff --git a/.travis.yml b/.travis.yml index 04624375e..b15b75d02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: python matrix: include: - - python: "2.7" + - python: "3.7" # horrible thing to fix cloudaux/boto/google issue before_install: @@ -24,5 +24,6 @@ after_success: notifications: email: - tmcpeak@netflix.com - pkelley@netflix.com + - tmcpeak@netflix.com + - ccastrapel@netflix.com + diff --git a/repokid/__init__.py b/repokid/__init__.py index 2e152f8ee..d98b651d7 100644 --- a/repokid/__init__.py +++ b/repokid/__init__.py @@ -19,7 +19,7 @@ import import_string -__version__ = '0.9.5' +__version__ = "0.10.0" def init_config(): @@ -33,12 +33,14 @@ def init_config(): Returns: None """ - load_config_paths = [os.path.join(os.getcwd(), 'config.json'), - '/etc/repokid/config.json', - '/apps/repokid/config.json'] + load_config_paths = [ + os.path.join(os.getcwd(), "config.json"), + "/etc/repokid/config.json", + "/apps/repokid/config.json", + ] for path in load_config_paths: try: - with open(path, 'r') as f: + with open(path, "r") as f: print("Loaded config from {}".format(path)) return json.load(f) @@ -59,12 +61,12 @@ def init_logging(): None """ if CONFIG: - logging.config.dictConfig(CONFIG['logging']) + logging.config.dictConfig(CONFIG["logging"]) # these loggers are very noisy suppressed_loggers = [ - 'botocore.vendored.requests.packages.urllib3.connectionpool', - 'urllib3' + "botocore.vendored.requests.packages.urllib3.connectionpool", + "urllib3", ] for logger in suppressed_loggers: @@ -94,7 +96,9 @@ def _get_hooks(hooks_list): if hasattr(func, "_implements_hook"): # append to the dictionary in whatever order we see them, we'll sort later. Dictionary value should be # a list of tuples (priority, function) - hooks[func._implements_hook['hook_name']].append((func._implements_hook['priority'], func)) + hooks[func._implements_hook["hook_name"]].append( + (func._implements_hook["priority"], func) + ) # sort by priority for k in hooks.keys(): diff --git a/repokid/cli/dispatcher_cli.py b/repokid/cli/dispatcher_cli.py index 606cc33cc..5197e3bc2 100644 --- a/repokid/cli/dispatcher_cli.py +++ b/repokid/cli/dispatcher_cli.py @@ -10,8 +10,17 @@ class Message(object): - def __init__(self, command, account, role_name, respond_channel, respond_user=None, requestor=None, reason=None, - selection=None): + def __init__( + self, + command, + account, + role_name, + respond_channel, + respond_user=None, + requestor=None, + reason=None, + selection=None, + ): self.command = command self.account = account self.role_name = role_name @@ -38,32 +47,37 @@ def make_message(self, data): def get_failure_message(channel=None, message=None): - return {'channel': channel, - 'message': message, - 'title': 'Repokid Failure'} + return {"channel": channel, "message": message, "title": "Repokid Failure"} -@sts_conn('sqs') +@sts_conn("sqs") def delete_message(receipt_handle, client=None): - client.delete_message(QueueUrl=CONFIG['dispatcher']['to_rr_queue'], ReceiptHandle=receipt_handle) + client.delete_message( + QueueUrl=CONFIG["dispatcher"]["to_rr_queue"], ReceiptHandle=receipt_handle + ) -@sts_conn('sqs') +@sts_conn("sqs") def receive_message(client=None): - return client.receive_message(QueueUrl=CONFIG['dispatcher']['to_rr_queue'], MaxNumberOfMessages=1, - WaitTimeSeconds=10) + return client.receive_message( + QueueUrl=CONFIG["dispatcher"]["to_rr_queue"], + MaxNumberOfMessages=1, + WaitTimeSeconds=10, + ) -@sts_conn('sns') +@sts_conn("sns") def send_message(message_dict, client=None): - client.publish(TopicArn=CONFIG['dispatcher']['from_rr_sns'], Message=json.dumps(message_dict)) + client.publish( + TopicArn=CONFIG["dispatcher"]["from_rr_sns"], Message=json.dumps(message_dict) + ) @contextlib.contextmanager def message_context(message_object, connection): try: - receipt_handle = message_object['Messages'][0]['ReceiptHandle'] - yield json.loads(message_object['Messages'][0]['Body']) + receipt_handle = message_object["Messages"][0]["ReceiptHandle"] + yield json.loads(message_object["Messages"][0]["Body"]) except KeyError: # we might not actually have a message yield None @@ -73,21 +87,26 @@ def message_context(message_object, connection): all_funcs = inspect.getmembers(repokid.dispatcher, inspect.isfunction) -RESPONDER_FUNCTIONS = {func[1]._implements_command: func[1] for func in all_funcs if - hasattr(func[1], '_implements_command')} +RESPONDER_FUNCTIONS = { + func[1]._implements_command: func[1] + for func in all_funcs + if hasattr(func[1], "_implements_command") +} def main(): - dynamo_table = dynamo.dynamo_get_or_create_table(**CONFIG['dynamo_db']) + dynamo_table = dynamo.dynamo_get_or_create_table(**CONFIG["dynamo_db"]) message_schema = MessageSchema() - connection = {'assume_role': CONFIG['dispatcher'].get('assume_role', None), - 'session_name': CONFIG['dispatcher'].get('session_name', 'Repokid'), - 'region': CONFIG['dispatcher'].get('region', 'us-west-2')} + connection = { + "assume_role": CONFIG["dispatcher"].get("assume_role", None), + "session_name": CONFIG["dispatcher"].get("session_name", "Repokid"), + "region": CONFIG["dispatcher"].get("region", "us-west-2"), + } while True: message = receive_message(**connection) - if not message or 'Messages' not in message: + if not message or "Messages" not in message: continue with message_context(message, connection) as msg: @@ -98,22 +117,37 @@ def main(): command_data = parsed_msg.data if parsed_msg.errors: - failure_message = get_failure_message(channel=command_data.get('respond_channel', None), - message='Malformed message: {}'.format(parsed_msg.errors)) + failure_message = get_failure_message( + channel=command_data.get("respond_channel", None), + message="Malformed message: {}".format(parsed_msg.errors), + ) send_message(failure_message, **connection) continue try: - return_val = RESPONDER_FUNCTIONS[command_data.command](dynamo_table, command_data) + return_val = RESPONDER_FUNCTIONS[command_data.command]( + dynamo_table, command_data + ) except KeyError: - failure_message = get_failure_message(channel=command_data.respond_channel, - message='Unknown function {}'.format(command_data.command)) + failure_message = get_failure_message( + channel=command_data.respond_channel, + message="Unknown function {}".format(command_data.command), + ) send_message(failure_message, **connection) continue - send_message({'message': '@{} {}'.format(command_data.respond_user, return_val.return_message), - 'channel': command_data.respond_channel, - 'title': 'Repokid Success' if return_val.successful else 'Repokid Failure'}, **connection) + send_message( + { + "message": "@{} {}".format( + command_data.respond_user, return_val.return_message + ), + "channel": command_data.respond_channel, + "title": "Repokid Success" + if return_val.successful + else "Repokid Failure", + }, + **connection + ) if __name__ == "__main__": diff --git a/repokid/cli/repokid_cli.py b/repokid/cli/repokid_cli.py index ac779ef85..be4b28f79 100644 --- a/repokid/cli/repokid_cli.py +++ b/repokid/cli/repokid_cli.py @@ -45,8 +45,12 @@ import time import botocore -from cloudaux.aws.iam import (delete_role_policy, get_account_authorization_details, get_role_inline_policies, - put_role_policy) +from cloudaux.aws.iam import ( + delete_role_policy, + get_account_authorization_details, + get_role_inline_policies, + put_role_policy, +) from cloudaux.aws.sts import sts_conn from docopt import docopt import import_string @@ -57,8 +61,14 @@ from repokid import LOGGER import repokid.hooks from repokid.role import Role, Roles -from repokid.utils.dynamo import (dynamo_get_or_create_table, find_role_in_cache, get_role_data, role_ids_for_account, - role_ids_for_all_accounts, set_role_data) +from repokid.utils.dynamo import ( + dynamo_get_or_create_table, + find_role_in_cache, + get_role_data, + role_ids_for_account, + role_ids_for_all_accounts, + set_role_data, +) import repokid.utils.roledata as roledata import requests from tabulate import tabulate @@ -84,57 +94,42 @@ def _generate_default_config(filename=None): "query_role_data_in_batch": False, "batch_processing_size": 100, "filter_config": { - "AgeFilter": { - "minimum_age": 90 - }, + "AgeFilter": {"minimum_age": 90}, "BlocklistFilter": { - "all": [ - ], + "all": [], "blocklist_bucket": { "bucket": "", "key": "", "account_number": "", "region": "" - } + "assume_role": "", + }, }, "ExclusiveFilter": { - "all": [ - "" - ], - "": [ - "" - ] - } + "all": [""], + "": [""], + }, }, - "active_filters": [ "repokid.filters.age:AgeFilter", "repokid.filters.lambda:LambdaFilter", "repokid.filters.blocklist:BlocklistFilter", - "repokid.filters.optout:OptOutFilter" + "repokid.filters.optout:OptOutFilter", ], - "aardvark_api_location": "", - "connection_iam": { "assume_role": "RepokidRole", "session_name": "repokid", - "region": "us-east-1" + "region": "us-east-1", }, - "dynamo_db": { "assume_role": "RepokidRole", "account_number": "", "endpoint": "", "region": "", - "session_name": "repokid" + "session_name": "repokid", }, - - "hooks": [ - "repokid.hooks.loggers" - ], - + "hooks": ["repokid.hooks.loggers"], "logging": { "version": 1, "disable_existing_loggers": "False", @@ -151,46 +146,34 @@ def _generate_default_config(filename=None): "filename": "repokid.log", "maxBytes": 10485760, "backupCount": 100, - "encoding": "utf8" + "encoding": "utf8", }, "console": { "class": "logging.StreamHandler", "level": "INFO", "formatter": "standard", - "stream": "ext://sys.stdout" - } + "stream": "ext://sys.stdout", + }, }, - "loggers": { - "repokid": { - "handlers": ["file", "console"], - "level": "INFO" - } - } + "loggers": {"repokid": {"handlers": ["file", "console"], "level": "INFO"}}, }, - "opt_out_period_days": 90, - "dispatcher": { "session_name": "repokid", "region": "us-west-2", "to_rr_queue": "COMMAND_QUEUE_TO_REPOKID_URL", - "from_rr_sns": "RESPONSES_FROM_REPOKID_SNS_ARN" + "from_rr_sns": "RESPONSES_FROM_REPOKID_SNS_ARN", }, - "repo_requirements": { "oldest_aa_data_days": 5, - "exclude_new_permissions_for_days": 14 + "exclude_new_permissions_for_days": 14, }, - "repo_schedule_period_days": 7, - - "warnings": { - "unknown_permissions": False - } + "warnings": {"unknown_permissions": False}, } if filename: try: - with open(filename, 'w') as f: + with open(filename, "w") as f: json.dump(config_dict, f, indent=4, sort_keys=True) except OSError as e: print("Unable to open {} for writing: {}".format(filename, e.message)) @@ -220,57 +203,67 @@ def _get_aardvark_data(aardvark_api_location, account_number=None, arn=None): page_num = 1 if account_number: - payload = {'phrase': '{}'.format(account_number)} + payload = {"phrase": "{}".format(account_number)} elif arn: - payload = {'arn': [arn]} + payload = {"arn": [arn]} else: return while True: - params = {'count': PAGE_SIZE, 'page': page_num} + params = {"count": PAGE_SIZE, "page": page_num} try: - r_aardvark = requests.post(aardvark_api_location, params=params, json=payload) + r_aardvark = requests.post( + aardvark_api_location, params=params, json=payload + ) except requests.exceptions.RequestException as e: - LOGGER.error('Unable to get Aardvark data: {}'.format(e)) + LOGGER.error("Unable to get Aardvark data: {}".format(e)) sys.exit(1) else: - if(r_aardvark.status_code != 200): - LOGGER.error('Unable to get Aardvark data') + if r_aardvark.status_code != 200: + LOGGER.error("Unable to get Aardvark data") sys.exit(1) response_data.update(r_aardvark.json()) # don't want these in our Aardvark data - response_data.pop('count') - response_data.pop('page') - response_data.pop('total') - if PAGE_SIZE * page_num < r_aardvark.json().get('total'): + response_data.pop("count") + response_data.pop("page") + response_data.pop("total") + if PAGE_SIZE * page_num < r_aardvark.json().get("total"): page_num += 1 else: break return response_data -@sts_conn('iam') +@sts_conn("iam") def _update_repoed_description(role_name, client=None): description = None try: - description = client.get_role(RoleName=role_name)['Role'].get('Description', '') + description = client.get_role(RoleName=role_name)["Role"].get("Description", "") except KeyError: return - date_string = datetime.datetime.utcnow().strftime('%m/%d/%y') - if '; Repokid repoed' in description: - new_description = re.sub(r'; Repokid repoed [0-9]{2}\/[0-9]{2}\/[0-9]{2}', '; Repokid repoed {}'.format( - date_string), description) + date_string = datetime.datetime.utcnow().strftime("%m/%d/%y") + if "; Repokid repoed" in description: + new_description = re.sub( + r"; Repokid repoed [0-9]{2}\/[0-9]{2}\/[0-9]{2}", + "; Repokid repoed {}".format(date_string), + description, + ) else: - new_description = description + ' ; Repokid repoed {}'.format(date_string) + new_description = description + " ; Repokid repoed {}".format(date_string) # IAM role descriptions have a max length of 1000, if our new length would be longer, skip this if len(new_description) < 1000: client.update_role_description(RoleName=role_name, Description=new_description) else: - LOGGER.error('Unable to set repo description ({}) for role {}, length would be too long'.format( - new_description, role_name)) + LOGGER.error( + "Unable to set repo description ({}) for role {}, length would be too long".format( + new_description, role_name + ) + ) -def _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source, add_no_repo=True): +def _update_role_data( + role, dynamo_table, account_number, config, conn, hooks, source, add_no_repo=True +): """ Perform a scaled down version of role update, this is used to get an accurate count of repoable permissions after a rollback or repo. @@ -299,9 +292,15 @@ def _update_role_data(role, dynamo_table, account_number, config, conn, hooks, s None """ current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} - roledata.update_role_data(dynamo_table, account_number, role, current_policies, source=source, - add_no_repo=add_no_repo) - aardvark_data = _get_aardvark_data(config['aardvark_api_location'], arn=role.arn) + roledata.update_role_data( + dynamo_table, + account_number, + role, + current_policies, + source=source, + add_no_repo=add_no_repo, + ) + aardvark_data = _get_aardvark_data(config["aardvark_api_location"], arn=role.arn) if not aardvark_data: return @@ -310,12 +309,23 @@ def _update_role_data(role, dynamo_table, account_number, config, conn, hooks, s batch_size = config.get("batch_processing_size", 100) role.aa_data = aardvark_data[role.arn] - roledata._calculate_repo_scores([role], config['filter_config']['AgeFilter'] - ['minimum_age'], hooks, batch_processing, batch_size) - set_role_data(dynamo_table, role.role_id, {'AAData': role.aa_data, - 'TotalPermissions': role.total_permissions, - 'RepoablePermissions': role.repoable_permissions, - 'RepoableServices': role.repoable_services}) + roledata._calculate_repo_scores( + [role], + config["filter_config"]["AgeFilter"]["minimum_age"], + hooks, + batch_processing, + batch_size, + ) + set_role_data( + dynamo_table, + role.role_id, + { + "AAData": role.aa_data, + "TotalPermissions": role.total_permissions, + "RepoablePermissions": role.repoable_permissions, + "RepoableServices": role.repoable_services, + }, + ) roledata.update_stats(dynamo_table, [role], source=source) @@ -343,7 +353,7 @@ def load_plugin(self, module, config=None): plugin = cls(config=config) except KeyError: plugin = cls() - LOGGER.info('Loaded plugin {}'.format(module)) + LOGGER.info("Loaded plugin {}".format(module)) self.filter_plugins.append(plugin) @@ -381,86 +391,128 @@ def update_role_cache(account_number, dynamo_table, config, hooks): Returns: None """ - conn = config['connection_iam'] - conn['account_number'] = account_number + conn = config["connection_iam"] + conn["account_number"] = account_number - LOGGER.info('Getting current role data for account {} (this may take a while for large accounts)'.format( - account_number)) + LOGGER.info( + "Getting current role data for account {} (this may take a while for large accounts)".format( + account_number + ) + ) - role_data = get_account_authorization_details(filter='Role', **conn) - role_data_by_id = {item['RoleId']: item for item in role_data} + role_data = get_account_authorization_details(filter="Role", **conn) + role_data_by_id = {item["RoleId"]: item for item in role_data} # convert policies list to dictionary to maintain consistency with old call which returned a dict for _, data in role_data_by_id.items(): - data['RolePolicyList'] = {item['PolicyName']: item['PolicyDocument'] for item in data['RolePolicyList']} + data["RolePolicyList"] = { + item["PolicyName"]: item["PolicyDocument"] + for item in data["RolePolicyList"] + } roles = Roles([Role(rd) for rd in role_data]) active_roles = [] - LOGGER.info('Updating role data for account {}'.format(account_number)) + LOGGER.info("Updating role data for account {}".format(account_number)) for role in tqdm(roles): role.account = account_number - current_policies = role_data_by_id[role.role_id]['RolePolicyList'] + current_policies = role_data_by_id[role.role_id]["RolePolicyList"] active_roles.append(role.role_id) roledata.update_role_data(dynamo_table, account_number, role, current_policies) - LOGGER.info('Finding inactive roles in account {}'.format(account_number)) + LOGGER.info("Finding inactive roles in account {}".format(account_number)) roledata.find_and_mark_inactive(dynamo_table, account_number, active_roles) - LOGGER.info('Filtering roles') + LOGGER.info("Filtering roles") plugins = FilterPlugins() # Blocklist needs to know the current account - filter_config = config['filter_config'] - blocklist_filter_config = filter_config.get('BlocklistFilter', filter_config.get('BlacklistFilter')) - blocklist_filter_config['current_account'] = account_number - - for plugin_path in config.get('active_filters'): - plugin_name = plugin_path.split(':')[1] - if plugin_name == 'ExclusiveFilter': + filter_config = config["filter_config"] + blocklist_filter_config = filter_config.get( + "BlocklistFilter", filter_config.get("BlacklistFilter") + ) + blocklist_filter_config["current_account"] = account_number + + for plugin_path in config.get("active_filters"): + plugin_name = plugin_path.split(":")[1] + if plugin_name == "ExclusiveFilter": # ExclusiveFilter plugin active; try loading its config. Also, it requires the current account, so add it. - exclusive_filter_config = filter_config.get('ExclusiveFilter', {}) - exclusive_filter_config['current_account'] = account_number - plugins.load_plugin(plugin_path, config=config['filter_config'].get(plugin_name, None)) + exclusive_filter_config = filter_config.get("ExclusiveFilter", {}) + exclusive_filter_config["current_account"] = account_number + plugins.load_plugin( + plugin_path, config=config["filter_config"].get(plugin_name, None) + ) for plugin in plugins.filter_plugins: filtered_list = plugin.apply(roles) class_name = plugin.__class__.__name__ for filtered_role in filtered_list: - LOGGER.info('Role {} filtered by {}'.format(filtered_role.role_name, class_name)) + LOGGER.info( + "Role {} filtered by {}".format(filtered_role.role_name, class_name) + ) filtered_role.disqualified_by.append(class_name) for role in roles: - set_role_data(dynamo_table, role.role_id, {'DisqualifiedBy': role.disqualified_by}) - - LOGGER.info('Getting data from Aardvark for account {}'.format(account_number)) - aardvark_data = _get_aardvark_data(config['aardvark_api_location'], account_number=account_number) - - LOGGER.info('Updating roles with Aardvark data in account {}'.format(account_number)) + set_role_data( + dynamo_table, role.role_id, {"DisqualifiedBy": role.disqualified_by} + ) + + LOGGER.info("Getting data from Aardvark for account {}".format(account_number)) + aardvark_data = _get_aardvark_data( + config["aardvark_api_location"], account_number=account_number + ) + + LOGGER.info( + "Updating roles with Aardvark data in account {}".format(account_number) + ) for role in roles: try: role.aa_data = aardvark_data[role.arn] except KeyError: - LOGGER.warning('Aardvark data not found for role: {} ({})'.format(role.role_id, role.role_name)) + LOGGER.warning( + "Aardvark data not found for role: {} ({})".format( + role.role_id, role.role_name + ) + ) else: - set_role_data(dynamo_table, role.role_id, {'AAData': role.aa_data}) + set_role_data(dynamo_table, role.role_id, {"AAData": role.aa_data}) - LOGGER.info('Calculating repoable permissions and services for account {}'.format(account_number)) + LOGGER.info( + "Calculating repoable permissions and services for account {}".format( + account_number + ) + ) batch_processing = config.get("query_role_data_in_batch", False) batch_size = config.get("batch_processing_size", 100) - roledata._calculate_repo_scores(roles, config['filter_config']['AgeFilter'] - ['minimum_age'], hooks, batch_processing, batch_size) + roledata._calculate_repo_scores( + roles, + config["filter_config"]["AgeFilter"]["minimum_age"], + hooks, + batch_processing, + batch_size, + ) for role in roles: - LOGGER.debug('Role {} in account {} has\nrepoable permissions: {}\nrepoable services:'.format( - role.role_name, account_number, role.repoable_permissions, role.repoable_services - )) - set_role_data(dynamo_table, role.role_id, {'TotalPermissions': role.total_permissions, - 'RepoablePermissions': role.repoable_permissions, - 'RepoableServices': role.repoable_services}) + LOGGER.debug( + "Role {} in account {} has\nrepoable permissions: {}\nrepoable services: {}".format( + role.role_name, + account_number, + role.repoable_permissions, + role.repoable_services, + ) + ) + set_role_data( + dynamo_table, + role.role_id, + { + "TotalPermissions": role.total_permissions, + "RepoablePermissions": role.repoable_permissions, + "RepoableServices": role.repoable_services, + }, + ) - LOGGER.info('Updating stats in account {}'.format(account_number)) - roledata.update_stats(dynamo_table, roles, source='Scan') + LOGGER.info("Updating stats in account {}".format(account_number)) + roledata.update_stats(dynamo_table, roles, source="Scan") def display_roles(account_number, dynamo_table, inactive=False): @@ -474,32 +526,48 @@ def display_roles(account_number, dynamo_table, inactive=False): Returns: None """ - headers = ['Name', 'Refreshed', 'Disqualified By', 'Can be repoed', 'Permissions', 'Repoable', 'Repoed', - 'Services'] + headers = [ + "Name", + "Refreshed", + "Disqualified By", + "Can be repoed", + "Permissions", + "Repoable", + "Repoed", + "Services", + ] rows = list() - roles = Roles([Role(get_role_data(dynamo_table, roleID)) - for roleID in tqdm(role_ids_for_account(dynamo_table, account_number))]) + roles = Roles( + [ + Role(get_role_data(dynamo_table, roleID)) + for roleID in tqdm(role_ids_for_account(dynamo_table, account_number)) + ] + ) if not inactive: roles = roles.filter(active=True) for role in roles: - rows.append([role.role_name, - role.refreshed, - role.disqualified_by, - len(role.disqualified_by) == 0, - role.total_permissions, - role.repoable_permissions, - role.repoed, - role.repoable_services]) + rows.append( + [ + role.role_name, + role.refreshed, + role.disqualified_by, + len(role.disqualified_by) == 0, + role.total_permissions, + role.repoable_permissions, + role.repoed, + role.repoable_services, + ] + ) rows = sorted(rows, key=lambda x: (x[5], x[0], x[4])) rows.insert(0, headers) # print tabulate(rows, headers=headers) t.view(rows) - with open('table.csv', 'wb') as csvfile: + with open("table.csv", "w") as csvfile: csv_writer = csv.writer(csvfile) csv_writer.writerow(headers) for row in rows: @@ -519,7 +587,11 @@ def find_roles_with_permissions(permissions, dynamo_table, output_file): """ arns = list() for roleID in role_ids_for_all_accounts(dynamo_table): - role = Role(get_role_data(dynamo_table, roleID, fields=['Policies', 'RoleName', 'Arn', 'Active'])) + role = Role( + get_role_data( + dynamo_table, roleID, fields=["Policies", "RoleName", "Arn", "Active"] + ) + ) role_permissions, _ = roledata._get_role_permissions(role) permissions = set([p.lower() for p in permissions]) @@ -527,12 +599,16 @@ def find_roles_with_permissions(permissions, dynamo_table, output_file): if found_permissions and role.active: arns.append(role.arn) - LOGGER.info('ARN {arn} has {permissions}'.format(arn=role.arn, permissions=list(found_permissions))) + LOGGER.info( + "ARN {arn} has {permissions}".format( + arn=role.arn, permissions=list(found_permissions) + ) + ) if not output_file: return - with open(output_file, 'wb') as fd: + with open(output_file, "wb") as fd: json.dump(arns, fd) LOGGER.info('Ouput written to file "{output_file}"'.format(output_file=output_file)) @@ -557,84 +633,123 @@ def display_role(account_number, role_name, dynamo_table, config, hooks): """ role_id = find_role_in_cache(dynamo_table, account_number, role_name) if not role_id: - LOGGER.warn('Could not find role with name {}'.format(role_name)) + LOGGER.warn("Could not find role with name {}".format(role_name)) return role = Role(get_role_data(dynamo_table, role_id)) - print "\n\nRole repo data:" - headers = ['Name', 'Refreshed', 'Disqualified By', 'Can be repoed', 'Permissions', 'Repoable', 'Repoed', 'Services'] - rows = [[role.role_name, - role.refreshed, - role.disqualified_by, - len(role.disqualified_by) == 0, - role.total_permissions, - role.repoable_permissions, - role.repoed, - role.repoable_services]] - print tabulate(rows, headers=headers) + '\n\n' - - print "Policy history:" - headers = ['Number', 'Source', 'Discovered', 'Permissions', 'Services'] + print("\n\nRole repo data:") + headers = [ + "Name", + "Refreshed", + "Disqualified By", + "Can be repoed", + "Permissions", + "Repoable", + "Repoed", + "Services", + ] + rows = [ + [ + role.role_name, + role.refreshed, + role.disqualified_by, + len(role.disqualified_by) == 0, + role.total_permissions, + role.repoable_permissions, + role.repoed, + role.repoable_services, + ] + ] + print(tabulate(rows, headers=headers) + "\n\n") + + print("Policy history:") + headers = ["Number", "Source", "Discovered", "Permissions", "Services"] rows = [] for index, policies_version in enumerate(role.policies): - policy_permissions, _ = roledata._get_permissions_in_policy(policies_version['Policy']) - rows.append([index, - policies_version['Source'], - policies_version['Discovered'], - len(policy_permissions), - roledata._get_services_in_permissions(policy_permissions)]) - print tabulate(rows, headers=headers) + '\n\n' - - print "Stats:" - headers = ['Date', 'Event Type', 'Permissions Count', 'Disqualified By'] + policy_permissions, _ = roledata._get_permissions_in_policy( + policies_version["Policy"] + ) + rows.append( + [ + index, + policies_version["Source"], + policies_version["Discovered"], + len(policy_permissions), + roledata._get_services_in_permissions(policy_permissions), + ] + ) + print(tabulate(rows, headers=headers) + "\n\n") + + print("Stats:") + headers = ["Date", "Event Type", "Permissions Count", "Disqualified By"] rows = [] for stats_entry in role.stats: - rows.append([stats_entry['Date'], - stats_entry['Source'], - stats_entry['PermissionsCount'], - stats_entry.get('DisqualifiedBy', [])]) - print tabulate(rows, headers=headers) + '\n\n' + rows.append( + [ + stats_entry["Date"], + stats_entry["Source"], + stats_entry["PermissionsCount"], + stats_entry.get("DisqualifiedBy", []), + ] + ) + print(tabulate(rows, headers=headers) + "\n\n") # can't do anymore if we don't have AA data if not role.aa_data: - LOGGER.warn('ARN not found in Access Advisor: {}'.format(role.arn)) + LOGGER.warn("ARN not found in Access Advisor: {}".format(role.arn)) return - warn_unknown_permissions = config.get('warnings', {}).get('unknown_permissions', False) + warn_unknown_permissions = config.get("warnings", {}).get( + "unknown_permissions", False + ) repoable_permissions = set([]) - permissions, eligible_permissions = roledata._get_role_permissions(role, - warn_unknown_perms=warn_unknown_permissions) + permissions, eligible_permissions = roledata._get_role_permissions( + role, warn_unknown_perms=warn_unknown_permissions + ) if len(role.disqualified_by) == 0: - repoable_permissions = roledata._get_repoable_permissions(account_number, role.role_name, eligible_permissions, - role.aa_data, role.no_repo_permissions, - config['filter_config']['AgeFilter']['minimum_age'], - hooks) - - print "Repoable services and permissions" - headers = ['Service', 'Action', 'Repoable'] + repoable_permissions = roledata._get_repoable_permissions( + account_number, + role.role_name, + eligible_permissions, + role.aa_data, + role.no_repo_permissions, + config["filter_config"]["AgeFilter"]["minimum_age"], + hooks, + ) + + print("Repoable services and permissions") + headers = ["Service", "Action", "Repoable"] rows = [] for permission in permissions: - service = permission.split(':')[0] - action = permission.split(':')[1] + service = permission.split(":")[0] + action = permission.split(":")[1] repoable = permission in repoable_permissions rows.append([service, action, repoable]) rows = sorted(rows, key=lambda x: (x[2], x[0], x[1])) - print tabulate(rows, headers=headers) + '\n\n' + print(tabulate(rows, headers=headers) + "\n\n") - repoed_policies, _ = roledata._get_repoed_policy(role.policies[-1]['Policy'], repoable_permissions) + repoed_policies, _ = roledata._get_repoed_policy( + role.policies[-1]["Policy"], repoable_permissions + ) if repoed_policies: - print('Repo\'d Policies: \n{}'.format(json.dumps(repoed_policies, indent=2, sort_keys=True))) + print( + "Repo'd Policies: \n{}".format( + json.dumps(repoed_policies, indent=2, sort_keys=True) + ) + ) else: - print('All Policies Removed') + print("All Policies Removed") # need to check if all policies would be too large if _inline_policies_size_exceeds_maximum(repoed_policies): - LOGGER.warning("Policies would exceed the AWS size limit after repo for role: {}. " - "Please manually minify.".format(role_name)) + LOGGER.warning( + "Policies would exceed the AWS size limit after repo for role: {}. " + "Please manually minify.".format(role_name) + ) def schedule_repo(account_number, dynamo_table, config, hooks): @@ -644,49 +759,72 @@ def schedule_repo(account_number, dynamo_table, config, hooks): """ scheduled_roles = [] - roles = Roles([Role(get_role_data(dynamo_table, roleID)) - for roleID in tqdm(role_ids_for_account(dynamo_table, account_number))]) + roles = Roles( + [ + Role(get_role_data(dynamo_table, roleID)) + for roleID in tqdm(role_ids_for_account(dynamo_table, account_number)) + ] + ) - scheduled_time = int(time.time()) + (86400 * config.get('repo_schedule_period_days', 7)) + scheduled_time = int(time.time()) + ( + 86400 * config.get("repo_schedule_period_days", 7) + ) for role in roles: if role.repoable_permissions > 0 and not role.repo_scheduled: role.repo_scheduled = scheduled_time # freeze the scheduled perms to whatever is repoable right now - set_role_data(dynamo_table, role.role_id, - {'RepoScheduled': scheduled_time, 'ScheduledPerms': role.repoable_services}) + set_role_data( + dynamo_table, + role.role_id, + { + "RepoScheduled": scheduled_time, + "ScheduledPerms": role.repoable_services, + }, + ) scheduled_roles.append(role) - LOGGER.info("Scheduled repo for {} days from now for account {} and these roles:\n\t{}".format( - config.get('repo_schedule_period_days', 7), - account_number, - ', '.join([r.role_name for r in scheduled_roles]))) + LOGGER.info( + "Scheduled repo for {} days from now for account {} and these roles:\n\t{}".format( + config.get("repo_schedule_period_days", 7), + account_number, + ", ".join([r.role_name for r in scheduled_roles]), + ) + ) - repokid.hooks.call_hooks(hooks, 'AFTER_SCHEDULE_REPO', {'roles': scheduled_roles}) + repokid.hooks.call_hooks(hooks, "AFTER_SCHEDULE_REPO", {"roles": scheduled_roles}) def show_scheduled_roles(account_number, dynamo_table): """ Show scheduled repos for a given account. For each scheduled show whether scheduled time is elapsed or not. """ - roles = Roles([Role(get_role_data(dynamo_table, roleID)) - for roleID in tqdm(role_ids_for_account(dynamo_table, account_number))]) + roles = Roles( + [ + Role(get_role_data(dynamo_table, roleID)) + for roleID in tqdm(role_ids_for_account(dynamo_table, account_number)) + ] + ) # filter to show only roles that are scheduled roles = roles.filter(active=True) roles = [role for role in roles if (role.repo_scheduled)] - header = ['Role name', 'Scheduled', 'Scheduled Time Elapsed?'] + header = ["Role name", "Scheduled", "Scheduled Time Elapsed?"] rows = [] curtime = int(time.time()) for role in roles: - rows.append([role.role_name, - dt.fromtimestamp(role.repo_scheduled).strftime('%Y-%m-%d %H:%M'), - role.repo_scheduled < curtime]) + rows.append( + [ + role.role_name, + dt.fromtimestamp(role.repo_scheduled).strftime("%Y-%m-%d %H:%M"), + role.repo_scheduled < curtime, + ] + ) - print tabulate(rows, headers=header) + print(tabulate(rows, headers=header)) def cancel_scheduled_repo(account_number, dynamo_table, role_name=None, is_all=None): @@ -694,36 +832,59 @@ def cancel_scheduled_repo(account_number, dynamo_table, role_name=None, is_all=N Cancel scheduled repo for a role in an account """ if not is_all and not role_name: - LOGGER.error('Either a specific role to cancel or all must be provided') + LOGGER.error("Either a specific role to cancel or all must be provided") return if is_all: - roles = Roles([Role(get_role_data(dynamo_table, roleID)) - for roleID in role_ids_for_account(dynamo_table, account_number)]) + roles = Roles( + [ + Role(get_role_data(dynamo_table, roleID)) + for roleID in role_ids_for_account(dynamo_table, account_number) + ] + ) # filter to show only roles that are scheduled roles = [role for role in roles if (role.repo_scheduled)] for role in roles: - set_role_data(dynamo_table, role.role_id, {'RepoScheduled': 0, 'ScheduledPerms': []}) - - LOGGER.info('Canceled scheduled repo for roles: {}'.format(', '.join([role.role_name for role in roles]))) + set_role_data( + dynamo_table, role.role_id, {"RepoScheduled": 0, "ScheduledPerms": []} + ) + + LOGGER.info( + "Canceled scheduled repo for roles: {}".format( + ", ".join([role.role_name for role in roles]) + ) + ) return role_id = find_role_in_cache(dynamo_table, account_number, role_name) if not role_id: - LOGGER.warn('Could not find role with name {} in account {}'.format(role_name, account_number)) + LOGGER.warn( + "Could not find role with name {} in account {}".format( + role_name, account_number + ) + ) return role = Role(get_role_data(dynamo_table, role_id)) if not role.repo_scheduled: - LOGGER.warn('Repo was not scheduled for role {} in account {}'.format(role.role_name, account_number)) + LOGGER.warn( + "Repo was not scheduled for role {} in account {}".format( + role.role_name, account_number + ) + ) return - set_role_data(dynamo_table, role.role_id, {'RepoScheduled': 0, 'ScheduledPerms': []}) - LOGGER.info('Successfully cancelled scheduled repo for role {} in account {}'.format(role.role_name, - role.account)) + set_role_data( + dynamo_table, role.role_id, {"RepoScheduled": 0, "ScheduledPerms": []} + ) + LOGGER.info( + "Successfully cancelled scheduled repo for role {} in account {}".format( + role.role_name, role.account + ) + ) def _inline_policies_size_exceeds_maximum(policies): @@ -734,13 +895,15 @@ def _inline_policies_size_exceeds_maximum(policies): Returns: bool """ - exported_no_whitespace = json.dumps(policies, separators=(',', ':')) + exported_no_whitespace = json.dumps(policies, separators=(",", ":")) if len(exported_no_whitespace) > MAX_AWS_POLICY_SIZE: return True return False -def _logprint_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role_name, account_number): +def _logprint_deleted_and_repoed_policies( + deleted_policy_names, repoed_policies, role_name, account_number +): """Logs data on policies that would otherwise be modified or deleted if the commit flag were set. Args: @@ -753,16 +916,20 @@ def _logprint_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, None """ for name in deleted_policy_names: - LOGGER.info('Would delete policy from {} with name {} in account {}'.format( - role_name, - name, - account_number)) + LOGGER.info( + "Would delete policy from {} with name {} in account {}".format( + role_name, name, account_number + ) + ) if repoed_policies: - LOGGER.info('Would replace policies for role {} with: \n{} in account {}'.format( - role_name, - json.dumps(repoed_policies, indent=2, sort_keys=True), - account_number)) + LOGGER.info( + "Would replace policies for role {} with: \n{} in account {}".format( + role_name, + json.dumps(repoed_policies, indent=2, sort_keys=True), + account_number, + ) + ) def _delete_policy(name, role, account_number, conn): @@ -777,15 +944,17 @@ def _delete_policy(name, role, account_number, conn): Returns: error (string) or None """ - LOGGER.info('Deleting policy with name {} from {} in account {}'.format(name, role.role_name, account_number)) + LOGGER.info( + "Deleting policy with name {} from {} in account {}".format( + name, role.role_name, account_number + ) + ) try: delete_role_policy(RoleName=role.role_name, PolicyName=name, **conn) except botocore.exceptions.ClientError as e: - return 'Error deleting policy: {} from role: {} in account {}. Exception: {}'.format( - name, - role.role_name, - account_number, - e) + return "Error deleting policy: {} from role: {} in account {}. Exception: {}".format( + name, role.role_name, account_number, e + ) def _replace_policies(repoed_policies, role, account_number, conn): @@ -800,24 +969,36 @@ def _replace_policies(repoed_policies, role, account_number, conn): Returns: error (string) or None """ - LOGGER.info('Replacing Policies With: \n{} (role: {} account: {})'.format( - json.dumps(repoed_policies, indent=2, sort_keys=True), - role.role_name, - account_number)) + LOGGER.info( + "Replacing Policies With: \n{} (role: {} account: {})".format( + json.dumps(repoed_policies, indent=2, sort_keys=True), + role.role_name, + account_number, + ) + ) for policy_name, policy in repoed_policies.items(): try: - put_role_policy(RoleName=role.role_name, PolicyName=policy_name, - PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), - **conn) + put_role_policy( + RoleName=role.role_name, + PolicyName=policy_name, + PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), + **conn + ) except botocore.exceptions.ClientError as e: - error = 'Exception calling PutRolePolicy on {role}/{policy} in account {account}\n{e}\n'.format( - role=role.role_name, policy=policy_name, account=account_number, e=str(e)) + error = "Exception calling PutRolePolicy on {role}/{policy} in account {account}\n{e}\n".format( + role=role.role_name, + policy=policy_name, + account=account_number, + e=str(e), + ) return error -def remove_permissions_from_roles(permissions, role_filename, dynamo_table, config, hooks, commit=False): +def remove_permissions_from_roles( + permissions, role_filename, dynamo_table, config, hooks, commit=False +): """Loads roles specified in file and calls _remove_permissions_from_role() for each one. Args: @@ -829,29 +1010,45 @@ def remove_permissions_from_roles(permissions, role_filename, dynamo_table, conf None """ roles = list() - with open(role_filename, 'r') as fd: + with open(role_filename, "r") as fd: roles = json.load(fd) for role_arn in tqdm(roles): arn = ARN(role_arn) if arn.error: - LOGGER.error('INVALID ARN: {arn}'.format(arn=role_arn)) + LOGGER.error("INVALID ARN: {arn}".format(arn=role_arn)) return account_number = arn.account_number - role_name = arn.name.split('/')[-1] + role_name = arn.name.split("/")[-1] role_id = find_role_in_cache(dynamo_table, account_number, role_name) role = Role(get_role_data(dynamo_table, role_id)) - _remove_permissions_from_role(account_number, permissions, role, role_id, dynamo_table, config, hooks, - commit=commit) - - repokid.hooks.call_hooks(hooks, 'AFTER_REPO', {'role': role}) - - -def _remove_permissions_from_role(account_number, permissions, role, role_id, dynamo_table, config, hooks, - commit=False): + _remove_permissions_from_role( + account_number, + permissions, + role, + role_id, + dynamo_table, + config, + hooks, + commit=commit, + ) + + repokid.hooks.call_hooks(hooks, "AFTER_REPO", {"role": role}) + + +def _remove_permissions_from_role( + account_number, + permissions, + role, + role_id, + dynamo_table, + config, + hooks, + commit=False, +): """Remove the list of permissions from the provided role. Args: @@ -864,19 +1061,25 @@ def _remove_permissions_from_role(account_number, permissions, role, role_id, dy Returns: None """ - repoed_policies, deleted_policy_names = roledata._get_repoed_policy(role.policies[-1]['Policy'], permissions) + repoed_policies, deleted_policy_names = roledata._get_repoed_policy( + role.policies[-1]["Policy"], permissions + ) if _inline_policies_size_exceeds_maximum(repoed_policies): - LOGGER.error("Policies would exceed the AWS size limit after repo for role: {} in account {}. " - "Please manually minify.".format(role.role_name, account_number)) + LOGGER.error( + "Policies would exceed the AWS size limit after repo for role: {} in account {}. " + "Please manually minify.".format(role.role_name, account_number) + ) return if not commit: - _logprint_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role.role_name, account_number) + _logprint_deleted_and_repoed_policies( + deleted_policy_names, repoed_policies, role.role_name, account_number + ) return - conn = config['connection_iam'] - conn['account_number'] = account_number + conn = config["connection_iam"] + conn["account_number"] = account_number for name in deleted_policy_names: error = _delete_policy(name, role, account_number, conn) @@ -889,17 +1092,38 @@ def _remove_permissions_from_role(account_number, permissions, role, role_id, dy LOGGER.error(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} - roledata.add_new_policy_version(dynamo_table, role, current_policies, 'Repo') + roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") - set_role_data(dynamo_table, role.role_id, {'Repoed': datetime.datetime.utcnow().isoformat()}) + set_role_data( + dynamo_table, role.role_id, {"Repoed": datetime.datetime.utcnow().isoformat()} + ) _update_repoed_description(role.role_name, **conn) - _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source='ManualPermissionRepo', - add_no_repo=False) - LOGGER.info('Successfully removed {permissions} from role: {role} in account {account_number}'.format( - permissions=permissions, role=role.role_name, account_number=account_number)) - - -def repo_role(account_number, role_name, dynamo_table, config, hooks, commit=False, scheduled=False): + _update_role_data( + role, + dynamo_table, + account_number, + config, + conn, + hooks, + source="ManualPermissionRepo", + add_no_repo=False, + ) + LOGGER.info( + "Successfully removed {permissions} from role: {role} in account {account_number}".format( + permissions=permissions, role=role.role_name, account_number=account_number + ) + ) + + +def repo_role( + account_number, + role_name, + dynamo_table, + config, + hooks, + commit=False, + scheduled=False, +): """ Calculate what repoing can be done for a role and then actually do it if commit is set 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data @@ -917,10 +1141,13 @@ def repo_role(account_number, role_name, dynamo_table, config, hooks, commit=Fal role_id = find_role_in_cache(dynamo_table, account_number, role_name) # only load partial data that we need to determine if we should keep going - role_data = get_role_data(dynamo_table, role_id, fields=['DisqualifiedBy', 'AAData', 'RepoablePermissions', - 'RoleName']) + role_data = get_role_data( + dynamo_table, + role_id, + fields=["DisqualifiedBy", "AAData", "RepoablePermissions", "RoleName"], + ) if not role_data: - LOGGER.warn('Could not find role with name {}'.format(role_name)) + LOGGER.warn("Could not find role with name {}".format(role_name)) return else: role = Role(role_data) @@ -928,18 +1155,23 @@ def repo_role(account_number, role_name, dynamo_table, config, hooks, commit=Fal continuing = True if len(role.disqualified_by) > 0: - LOGGER.info('Cannot repo role {} in account {} because it is being disqualified by: {}'.format( - role_name, - account_number, - role.disqualified_by)) + LOGGER.info( + "Cannot repo role {} in account {} because it is being disqualified by: {}".format( + role_name, account_number, role.disqualified_by + ) + ) continuing = False if not role.aa_data: - LOGGER.warning('ARN not found in Access Advisor: {}'.format(role.arn)) + LOGGER.warning("ARN not found in Access Advisor: {}".format(role.arn)) continuing = False if not role.repoable_permissions: - LOGGER.info('No permissions to repo for role {} in account {}'.format(role_name, account_number)) + LOGGER.info( + "No permissions to repo for role {} in account {}".format( + role_name, account_number + ) + ) continuing = False # if we've gotten to this point, load the rest of the role @@ -947,48 +1179,66 @@ def repo_role(account_number, role_name, dynamo_table, config, hooks, commit=Fal old_aa_data_services = [] for aa_service in role.aa_data: - if(datetime.datetime.strptime(aa_service['lastUpdated'], '%a, %d %b %Y %H:%M:%S %Z') < - datetime.datetime.now() - datetime.timedelta(days=config['repo_requirements']['oldest_aa_data_days'])): - old_aa_data_services.append(aa_service['serviceName']) + if datetime.datetime.strptime( + aa_service["lastUpdated"], "%a, %d %b %Y %H:%M:%S %Z" + ) < datetime.datetime.now() - datetime.timedelta( + days=config["repo_requirements"]["oldest_aa_data_days"] + ): + old_aa_data_services.append(aa_service["serviceName"]) if old_aa_data_services: - LOGGER.error('AAData older than threshold for these services: {} (role: {}, account {})'.format( - old_aa_data_services, - role_name, - account_number)) + LOGGER.error( + "AAData older than threshold for these services: {} (role: {}, account {})".format( + old_aa_data_services, role_name, account_number + ) + ) continuing = False total_permissions, eligible_permissions = roledata._get_role_permissions(role) - repoable_permissions = roledata._get_repoable_permissions(account_number, role.role_name, eligible_permissions, - role.aa_data, role.no_repo_permissions, - config['filter_config']['AgeFilter']['minimum_age'], - hooks) + repoable_permissions = roledata._get_repoable_permissions( + account_number, + role.role_name, + eligible_permissions, + role.aa_data, + role.no_repo_permissions, + config["filter_config"]["AgeFilter"]["minimum_age"], + hooks, + ) # if this is a scheduled repo we need to filter out permissions that weren't previously scheduled if scheduled: - repoable_permissions = roledata._filter_scheduled_repoable_perms(repoable_permissions, role.scheduled_perms) + repoable_permissions = roledata._filter_scheduled_repoable_perms( + repoable_permissions, role.scheduled_perms + ) - repoed_policies, deleted_policy_names = roledata._get_repoed_policy(role.policies[-1]['Policy'], - repoable_permissions) + repoed_policies, deleted_policy_names = roledata._get_repoed_policy( + role.policies[-1]["Policy"], repoable_permissions + ) if _inline_policies_size_exceeds_maximum(repoed_policies): - error = ("Policies would exceed the AWS size limit after repo for role: {} in account {}. " - "Please manually minify.".format(role_name, account_number)) + error = ( + "Policies would exceed the AWS size limit after repo for role: {} in account {}. " + "Please manually minify.".format(role_name, account_number) + ) LOGGER.error(error) errors.append(error) continuing = False # if we aren't repoing for some reason, unschedule the role if not continuing: - set_role_data(dynamo_table, role.role_id, {'RepoScheduled': 0, 'ScheduledPerms': []}) + set_role_data( + dynamo_table, role.role_id, {"RepoScheduled": 0, "ScheduledPerms": []} + ) return if not commit: - _logprint_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role_name, account_number) + _logprint_deleted_and_repoed_policies( + deleted_policy_names, repoed_policies, role_name, account_number + ) return - conn = config['connection_iam'] - conn['account_number'] = account_number + conn = config["connection_iam"] + conn["account_number"] = account_number for name in deleted_policy_names: error = _delete_policy(name, role, account_number, conn) @@ -1003,23 +1253,44 @@ def repo_role(account_number, role_name, dynamo_table, config, hooks, commit=Fal errors.append(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} - roledata.add_new_policy_version(dynamo_table, role, current_policies, 'Repo') + roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") # regardless of whether we're successful we want to unschedule the repo - set_role_data(dynamo_table, role.role_id, {'RepoScheduled': 0, 'ScheduledPerms': []}) + set_role_data( + dynamo_table, role.role_id, {"RepoScheduled": 0, "ScheduledPerms": []} + ) - repokid.hooks.call_hooks(hooks, 'AFTER_REPO', {'role': role}) + repokid.hooks.call_hooks(hooks, "AFTER_REPO", {"role": role}) if not errors: # repos will stay scheduled until they are successful - set_role_data(dynamo_table, role.role_id, {'Repoed': datetime.datetime.utcnow().isoformat()}) + set_role_data( + dynamo_table, + role.role_id, + {"Repoed": datetime.datetime.utcnow().isoformat()}, + ) _update_repoed_description(role.role_name, **conn) - _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source='Repo', add_no_repo=False) - LOGGER.info('Successfully repoed role: {} in account {}'.format(role.role_name, account_number)) + _update_role_data( + role, + dynamo_table, + account_number, + config, + conn, + hooks, + source="Repo", + add_no_repo=False, + ) + LOGGER.info( + "Successfully repoed role: {} in account {}".format( + role.role_name, account_number + ) + ) return errors -def rollback_role(account_number, role_name, dynamo_table, config, hooks, selection=None, commit=None): +def rollback_role( + account_number, role_name, dynamo_table, config, hooks, selection=None, commit=None +): """ Display the historical policy versions for a roll as a numbered list. Restore to a specific version if selected. Indicate changes that will be made and then actually make them if commit is selected. @@ -1037,7 +1308,9 @@ def rollback_role(account_number, role_name, dynamo_table, config, hooks, select role_id = find_role_in_cache(dynamo_table, account_number, role_name) if not role_id: - message = 'Could not find role with name {} in account {}'.format(role_name, account_number) + message = "Could not find role with name {} in account {}".format( + role_name, account_number + ) errors.append(message) LOGGER.warning(message) return errors @@ -1046,36 +1319,48 @@ def rollback_role(account_number, role_name, dynamo_table, config, hooks, select # no option selected, display a table of options if not selection: - headers = ['Number', 'Source', 'Discovered', 'Permissions', 'Services'] + headers = ["Number", "Source", "Discovered", "Permissions", "Services"] rows = [] for index, policies_version in enumerate(role.policies): - policy_permissions, _ = roledata._get_permissions_in_policy(policies_version['Policy']) - rows.append([index, policies_version['Source'], policies_version['Discovered'], - len(policy_permissions), - roledata._get_services_in_permissions(policy_permissions)]) - print tabulate(rows, headers=headers) + policy_permissions, _ = roledata._get_permissions_in_policy( + policies_version["Policy"] + ) + rows.append( + [ + index, + policies_version["Source"], + policies_version["Discovered"], + len(policy_permissions), + roledata._get_services_in_permissions(policy_permissions), + ] + ) + print(tabulate(rows, headers=headers)) return - conn = config['connection_iam'] - conn['account_number'] = account_number + conn = config["connection_iam"] + conn["account_number"] = account_number current_policies = get_role_inline_policies(role.as_dict(), **conn) if selection: pp = pprint.PrettyPrinter() - print "Will restore the following policies:" - pp.pprint(role.policies[int(selection)]['Policy']) + print("Will restore the following policies:") + pp.pprint(role.policies[int(selection)]["Policy"]) - print "Current policies:" + print("Current policies:") pp.pprint(current_policies) - current_permissions, _ = roledata._get_permissions_in_policy(role.policies[-1]['Policy']) - selected_permissions, _ = roledata._get_permissions_in_policy(role.policies[int(selection)]['Policy']) + current_permissions, _ = roledata._get_permissions_in_policy( + role.policies[-1]["Policy"] + ) + selected_permissions, _ = roledata._get_permissions_in_policy( + role.policies[int(selection)]["Policy"] + ) restored_permissions = selected_permissions - current_permissions - print "\nResore will return these permissions:" - print '\n'.join([perm for perm in sorted(restored_permissions)]) + print("\nResore will return these permissions:") + print("\n".join([perm for perm in sorted(restored_permissions)])) if not commit: return False @@ -1085,23 +1370,25 @@ def rollback_role(account_number, role_name, dynamo_table, config, hooks, select # from the list as we update. Any policy names left need to be manually removed policies_to_remove = current_policies.keys() - for policy_name, policy in role.policies[int(selection)]['Policy'].items(): + for policy_name, policy in role.policies[int(selection)]["Policy"].items(): try: - LOGGER.info("Pushing cached policy: {} (role: {} account {})".format( - policy_name, - role.role_name, - account_number)) - - put_role_policy(RoleName=role.role_name, PolicyName=policy_name, - PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), - **conn) + LOGGER.info( + "Pushing cached policy: {} (role: {} account {})".format( + policy_name, role.role_name, account_number + ) + ) + + put_role_policy( + RoleName=role.role_name, + PolicyName=policy_name, + PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), + **conn + ) except botocore.exceptions.ClientError as e: message = "Unable to push policy {}. Error: {} (role: {} account {})".format( - policy_name, - e.message, - role.role_name, - account_number) + policy_name, e.message, role.role_name, account_number + ) LOGGER.error(message) errors.append(message) @@ -1115,27 +1402,40 @@ def rollback_role(account_number, role_name, dynamo_table, config, hooks, select if policies_to_remove: for policy_name in policies_to_remove: try: - delete_role_policy(RoleName=role.role_name, PolicyName=policy_name, **conn) + delete_role_policy( + RoleName=role.role_name, PolicyName=policy_name, **conn + ) except botocore.excpetions.ClientError as e: message = "Unable to delete policy {}. Error: {} (role: {} account {})".format( - policy_name, - e.message, - role.role_name, - account_number) + policy_name, e.message, role.role_name, account_number + ) LOGGER.error(message) errors.append(message) - _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source='Restore', add_no_repo=False) + _update_role_data( + role, + dynamo_table, + account_number, + config, + conn, + hooks, + source="Restore", + add_no_repo=False, + ) if not errors: - LOGGER.info('Successfully restored selected version of role policies (role: {} account: {})'.format( - role.role_name, - account_number)) + LOGGER.info( + "Successfully restored selected version of role policies (role: {} account: {})".format( + role.role_name, account_number + ) + ) return errors -def repo_all_roles(account_number, dynamo_table, config, hooks, commit=False, scheduled=True): +def repo_all_roles( + account_number, dynamo_table, config, hooks, commit=False, scheduled=True +): """ Repo all scheduled or eligible roles in an account. Collect any errors and display them at the end. @@ -1154,31 +1454,58 @@ def repo_all_roles(account_number, dynamo_table, config, hooks, commit=False, sc role_ids_in_account = role_ids_for_account(dynamo_table, account_number) roles = Roles([]) for role_id in role_ids_in_account: - roles.append(Role(get_role_data(dynamo_table, role_id, fields=['Active', 'RoleName', 'RepoScheduled']))) + roles.append( + Role( + get_role_data( + dynamo_table, + role_id, + fields=["Active", "RoleName", "RepoScheduled"], + ) + ) + ) roles = roles.filter(active=True) cur_time = int(time.time()) if scheduled: - roles = [role for role in roles if (role.repo_scheduled and cur_time > role.repo_scheduled)] - - LOGGER.info('Repoing these {}roles from account {}:\n\t{}'.format('scheduled ' if scheduled else '', - account_number, - ', '.join([role.role_name for role in roles]))) + roles = [ + role + for role in roles + if (role.repo_scheduled and cur_time > role.repo_scheduled) + ] + + LOGGER.info( + "Repoing these {}roles from account {}:\n\t{}".format( + "scheduled " if scheduled else "", + account_number, + ", ".join([role.role_name for role in roles]), + ) + ) - repokid.hooks.call_hooks(hooks, 'BEFORE_REPO_ROLES', {'account_number': account_number, 'roles': roles}) + repokid.hooks.call_hooks( + hooks, "BEFORE_REPO_ROLES", {"account_number": account_number, "roles": roles} + ) for role in roles: - error = repo_role(account_number, role.role_name, dynamo_table, config, hooks, commit=commit, - scheduled=scheduled) + error = repo_role( + account_number, + role.role_name, + dynamo_table, + config, + hooks, + commit=commit, + scheduled=scheduled, + ) if error: errors.append(error) if errors: - LOGGER.error('Error(s) during repo: \n{} (account: {})'.format(errors, account_number)) + LOGGER.error( + "Error(s) during repo: \n{} (account: {})".format(errors, account_number) + ) else: - LOGGER.info('Successfully repoed roles in account {}'.format(account_number)) + LOGGER.info("Successfully repoed roles in account {}".format(account_number)) def repo_stats(output_file, dynamo_table, account_number=None): @@ -1192,120 +1519,172 @@ def repo_stats(output_file, dynamo_table, account_number=None): Returns: None """ - roleIDs = (role_ids_for_account(dynamo_table, account_number) if account_number else - role_ids_for_all_accounts(dynamo_table)) - headers = ['RoleId', 'Role Name', 'Account', 'Active', 'Date', 'Source', 'Permissions Count', - 'Repoable Permissions Count', 'Disqualified By'] + roleIDs = ( + role_ids_for_account(dynamo_table, account_number) + if account_number + else role_ids_for_all_accounts(dynamo_table) + ) + headers = [ + "RoleId", + "Role Name", + "Account", + "Active", + "Date", + "Source", + "Permissions Count", + "Repoable Permissions Count", + "Disqualified By", + ] rows = [] for roleID in roleIDs: - role_data = get_role_data(dynamo_table, roleID, fields=['RoleId', 'RoleName', 'Account', 'Active', 'Stats']) - for stats_entry in role_data.get('Stats', []): - rows.append([role_data['RoleId'], role_data['RoleName'], role_data['Account'], role_data['Active'], - stats_entry['Date'], stats_entry['Source'], stats_entry['PermissionsCount'], - stats_entry.get('RepoablePermissionsCount'), stats_entry.get('DisqualifiedBy', [])]) + role_data = get_role_data( + dynamo_table, + roleID, + fields=["RoleId", "RoleName", "Account", "Active", "Stats"], + ) + for stats_entry in role_data.get("Stats", []): + rows.append( + [ + role_data["RoleId"], + role_data["RoleName"], + role_data["Account"], + role_data["Active"], + stats_entry["Date"], + stats_entry["Source"], + stats_entry["PermissionsCount"], + stats_entry.get("RepoablePermissionsCount"), + stats_entry.get("DisqualifiedBy", []), + ] + ) try: - with open(output_file, 'wb') as csvfile: + with open(output_file, "w") as csvfile: csv_writer = csv.writer(csvfile) csv_writer.writerow(headers) for row in rows: csv_writer.writerow(row) except IOError as e: - LOGGER.error('Unable to write file {}: {}'.format(output_file, e)) + LOGGER.error("Unable to write file {}: {}".format(output_file, e)) else: - LOGGER.info('Successfully wrote stats to {}'.format(output_file)) + LOGGER.info("Successfully wrote stats to {}".format(output_file)) def main(): - args = docopt(__doc__, version='Repokid {version}'.format(version=__version__)) + args = docopt(__doc__, version="Repokid {version}".format(version=__version__)) - if args.get('config'): - config_filename = args.get('') + if args.get("config"): + config_filename = args.get("") _generate_default_config(filename=config_filename) sys.exit(0) - account_number = args.get('') + account_number = args.get("") if not CONFIG: config = _generate_default_config() else: config = CONFIG - LOGGER.debug('Repokid cli called with args {}'.format(args)) + LOGGER.debug("Repokid cli called with args {}".format(args)) - hooks = _get_hooks(config.get('hooks', ['repokid.hooks.loggers'])) - dynamo_table = dynamo_get_or_create_table(**config['dynamo_db']) + hooks = _get_hooks(config.get("hooks", ["repokid.hooks.loggers"])) + dynamo_table = dynamo_get_or_create_table(**config["dynamo_db"]) - if args.get('update_role_cache'): + if args.get("update_role_cache"): return update_role_cache(account_number, dynamo_table, config, hooks) - if args.get('display_role_cache'): - inactive = args.get('--inactive') + if args.get("display_role_cache"): + inactive = args.get("--inactive") return display_roles(account_number, dynamo_table, inactive=inactive) - if args.get('find_roles_with_permissions'): - permissions = args.get('') - output_file = args.get('--output') + if args.get("find_roles_with_permissions"): + permissions = args.get("") + output_file = args.get("--output") return find_roles_with_permissions(permissions, dynamo_table, output_file) - if args.get('remove_permissions_from_roles'): - permissions = args.get('') - role_filename = args.get('--role-file') - commit = args.get('--commit') - return remove_permissions_from_roles(permissions, role_filename, dynamo_table, config, hooks, commit=commit) + if args.get("remove_permissions_from_roles"): + permissions = args.get("") + role_filename = args.get("--role-file") + commit = args.get("--commit") + return remove_permissions_from_roles( + permissions, role_filename, dynamo_table, config, hooks, commit=commit + ) - if args.get('display_role'): - role_name = args.get('') + if args.get("display_role"): + role_name = args.get("") return display_role(account_number, role_name, dynamo_table, config, hooks) - if args.get('repo_role'): - role_name = args.get('') - commit = args.get('--commit') - return repo_role(account_number, role_name, dynamo_table, config, hooks, commit=commit) - - if args.get('rollback_role'): - role_name = args.get('') - commit = args.get('--commit') - selection = args.get('--selection') - return rollback_role(account_number, role_name, dynamo_table, config, hooks, selection=selection, commit=commit) - - if args.get('repo_all_roles'): - LOGGER.info('Updating role data') + if args.get("repo_role"): + role_name = args.get("") + commit = args.get("--commit") + return repo_role( + account_number, role_name, dynamo_table, config, hooks, commit=commit + ) + + if args.get("rollback_role"): + role_name = args.get("") + commit = args.get("--commit") + selection = args.get("--selection") + return rollback_role( + account_number, + role_name, + dynamo_table, + config, + hooks, + selection=selection, + commit=commit, + ) + + if args.get("repo_all_roles"): + LOGGER.info("Updating role data") update_role_cache(account_number, dynamo_table, config, hooks) - LOGGER.info('Repoing all roles') - commit = args.get('--commit') - return repo_all_roles(account_number, dynamo_table, config, hooks, commit=commit, scheduled=False) - - if args.get('schedule_repo'): - LOGGER.info('Updating role data') + LOGGER.info("Repoing all roles") + commit = args.get("--commit") + return repo_all_roles( + account_number, dynamo_table, config, hooks, commit=commit, scheduled=False + ) + + if args.get("schedule_repo"): + LOGGER.info("Updating role data") update_role_cache(account_number, dynamo_table, config, hooks) return schedule_repo(account_number, dynamo_table, config, hooks) - if args.get('show_scheduled_roles'): - LOGGER.info('Showing scheduled roles') + if args.get("show_scheduled_roles"): + LOGGER.info("Showing scheduled roles") return show_scheduled_roles(account_number, dynamo_table) - if args.get('cancel_scheduled_repo'): - role_name = args.get('--role') - is_all = args.get('--all') + if args.get("cancel_scheduled_repo"): + role_name = args.get("--role") + is_all = args.get("--all") if not is_all: - LOGGER.info('Cancelling scheduled repo for role: {} in account {}'.format(role_name, account_number)) + LOGGER.info( + "Cancelling scheduled repo for role: {} in account {}".format( + role_name, account_number + ) + ) else: - LOGGER.info('Cancelling scheduled repo for all roles in account {}'.format(account_number)) - return cancel_scheduled_repo(account_number, dynamo_table, role_name=role_name, is_all=is_all) - - if args.get('repo_scheduled_roles'): + LOGGER.info( + "Cancelling scheduled repo for all roles in account {}".format( + account_number + ) + ) + return cancel_scheduled_repo( + account_number, dynamo_table, role_name=role_name, is_all=is_all + ) + + if args.get("repo_scheduled_roles"): update_role_cache(account_number, dynamo_table, config, hooks) - LOGGER.info('Repoing scheduled roles') - commit = args.get('--commit') - return repo_all_roles(account_number, dynamo_table, config, hooks, commit=commit, scheduled=True) - - if args.get('repo_stats'): - output_file = args.get('') - account_number = args.get('--account') + LOGGER.info("Repoing scheduled roles") + commit = args.get("--commit") + return repo_all_roles( + account_number, dynamo_table, config, hooks, commit=commit, scheduled=True + ) + + if args.get("repo_stats"): + output_file = args.get("") + account_number = args.get("--account") return repo_stats(output_file, dynamo_table, account_number=account_number) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/repokid/dispatcher/__init__.py b/repokid/dispatcher/__init__.py index 09669490e..e6bc0ede3 100644 --- a/repokid/dispatcher/__init__.py +++ b/repokid/dispatcher/__init__.py @@ -8,12 +8,12 @@ import repokid.utils.dynamo as dynamo import repokid.utils.roledata as roledata -ResponderReturn = namedtuple('ResponderReturn', 'successful, return_message') +ResponderReturn = namedtuple("ResponderReturn", "successful, return_message") if CONFIG: - hooks = _get_hooks(CONFIG.get('hooks', ['repokid.hooks.loggers'])) + hooks = _get_hooks(CONFIG.get("hooks", ["repokid.hooks.loggers"])) else: - hooks = ['repokid.hooks.loggers'] + hooks = ["repokid.hooks.loggers"] def implements_command(command): @@ -21,129 +21,199 @@ def _implements_command(func): if not hasattr(func, "_implements_command"): func._implements_command = command return func + return _implements_command -@implements_command('list_repoable_services') +@implements_command("list_repoable_services") def list_repoable_services(dynamo_table, message): - role_id = dynamo.find_role_in_cache(dynamo_table, message.account, message.role_name) + role_id = dynamo.find_role_in_cache( + dynamo_table, message.account, message.role_name + ) if not role_id: - return ResponderReturn(successful=False, - return_message='Unable to find role {} in account {}'.format(message.role_name, - message.account)) + return ResponderReturn( + successful=False, + return_message="Unable to find role {} in account {}".format( + message.role_name, message.account + ), + ) else: - role_data = dynamo.get_role_data(dynamo_table, role_id, fields=['RepoableServices']) + role_data = dynamo.get_role_data( + dynamo_table, role_id, fields=["RepoableServices"] + ) - (repoable_permissions, repoable_services) = roledata._convert_repoed_service_to_sorted_perms_and_services( - role_data['RepoableServices']) + ( + repoable_permissions, + repoable_services, + ) = roledata._convert_repoed_service_to_sorted_perms_and_services( + role_data["RepoableServices"] + ) - repoable_services = role_data['RepoableServices'] + repoable_services = role_data["RepoableServices"] return ResponderReturn( successful=True, return_message=( - 'Role {} in account {} has:\n Repoable Services: \n{}\n\n Repoable Permissions: \n{}'.format( - message.role_name, message.account, '\n'.join([service for service in repoable_services]), - '\n'.join([perm for perm in repoable_permissions]) + "Role {} in account {} has:\n Repoable Services: \n{}\n\n Repoable Permissions: \n{}".format( + message.role_name, + message.account, + "\n".join([service for service in repoable_services]), + "\n".join([perm for perm in repoable_permissions]), ) - ) + ), ) -@implements_command('list_role_rollbacks') +@implements_command("list_role_rollbacks") def list_role_rollbacks(dynamo_table, message): - role_id = dynamo.find_role_in_cache(dynamo_table, message.account, message.role_name) + role_id = dynamo.find_role_in_cache( + dynamo_table, message.account, message.role_name + ) if not role_id: - return ResponderReturn(successful=False, - return_message='Unable to find role {} in account {}'.format(message.role_name, - message.account)) + return ResponderReturn( + successful=False, + return_message="Unable to find role {} in account {}".format( + message.role_name, message.account + ), + ) else: - role_data = dynamo.get_role_data(dynamo_table, role_id, fields=['Policies']) - return_val = 'Restorable versions for role {} in account {}\n'.format(message.role_name, message.account) - for index, policy_version in enumerate(role_data['Policies']): - total_permissions, _ = roledata._get_permissions_in_policy(policy_version['Policy']) - return_val += '({:>3}): {:<5} {:<15} {}\n'.format(index, len(total_permissions), - policy_version['Discovered'], - policy_version['Source']) + role_data = dynamo.get_role_data(dynamo_table, role_id, fields=["Policies"]) + return_val = "Restorable versions for role {} in account {}\n".format( + message.role_name, message.account + ) + for index, policy_version in enumerate(role_data["Policies"]): + total_permissions, _ = roledata._get_permissions_in_policy( + policy_version["Policy"] + ) + return_val += "({:>3}): {:<5} {:<15} {}\n".format( + index, + len(total_permissions), + policy_version["Discovered"], + policy_version["Source"], + ) return ResponderReturn(successful=True, return_message=return_val) -@implements_command('opt_out') +@implements_command("opt_out") def opt_out(dynamo_table, message): if CONFIG: - opt_out_period = CONFIG.get('opt_out_period_days', 90) + opt_out_period = CONFIG.get("opt_out_period_days", 90) else: opt_out_period = 90 if not message.reason or not message.requestor: - return ResponderReturn(successful=False, return_message='Reason and requestor must be specified') + return ResponderReturn( + successful=False, return_message="Reason and requestor must be specified" + ) - role_id = dynamo.find_role_in_cache(dynamo_table, message.account, message.role_name) + role_id = dynamo.find_role_in_cache( + dynamo_table, message.account, message.role_name + ) if not role_id: - return ResponderReturn(successful=False, - return_message='Unable to find role {} in account {}'.format(message.role_name, - message.account)) + return ResponderReturn( + successful=False, + return_message="Unable to find role {} in account {}".format( + message.role_name, message.account + ), + ) - role_data = dynamo.get_role_data(dynamo_table, role_id, fields=['OptOut']) - if 'OptOut' in role_data and role_data['OptOut']: + role_data = dynamo.get_role_data(dynamo_table, role_id, fields=["OptOut"]) + if "OptOut" in role_data and role_data["OptOut"]: - timestr = time.strftime('%m/%d/%y', time.localtime(role_data['OptOut']['expire'])) - return ResponderReturn(successful=False, - return_message=('Role {} in account {} is already opted out by {} for reason {} ' - 'until {}'.format(message.role_name, message.account, - role_data['OptOut']['owner'], - role_data['OptOut']['reason'], - timestr))) + timestr = time.strftime( + "%m/%d/%y", time.localtime(role_data["OptOut"]["expire"]) + ) + return ResponderReturn( + successful=False, + return_message=( + "Role {} in account {} is already opted out by {} for reason {} " + "until {}".format( + message.role_name, + message.account, + role_data["OptOut"]["owner"], + role_data["OptOut"]["reason"], + timestr, + ) + ), + ) else: current_dt = datetime.datetime.fromtimestamp(time.time()) expire_dt = current_dt + datetime.timedelta(opt_out_period) expire_epoch = int((expire_dt - datetime.datetime(1970, 1, 1)).total_seconds()) - new_opt_out = {'owner': message.requestor, 'reason': message.reason, 'expire': expire_epoch} - dynamo.set_role_data(dynamo_table, role_id, {'OptOut': new_opt_out}) + new_opt_out = { + "owner": message.requestor, + "reason": message.reason, + "expire": expire_epoch, + } + dynamo.set_role_data(dynamo_table, role_id, {"OptOut": new_opt_out}) return ResponderReturn( successful=True, - return_message='Role {} in account {} opted-out until {}'.format( - message.role_name, message.account, expire_dt.strftime('%m/%d/%y')) + return_message="Role {} in account {} opted-out until {}".format( + message.role_name, message.account, expire_dt.strftime("%m/%d/%y") + ), ) -@implements_command('remove_opt_out') +@implements_command("remove_opt_out") def remove_opt_out(dynamo_table, message): - role_id = dynamo.find_role_in_cache(dynamo_table, message.account, message.role_name) + role_id = dynamo.find_role_in_cache( + dynamo_table, message.account, message.role_name + ) if not role_id: - return ResponderReturn(successful=False, - return_message='Unable to find role {} in account {}'.format(message.role_name, - message.account)) + return ResponderReturn( + successful=False, + return_message="Unable to find role {} in account {}".format( + message.role_name, message.account + ), + ) - role_data = dynamo.get_role_data(dynamo_table, role_id, fields=['OptOut']) + role_data = dynamo.get_role_data(dynamo_table, role_id, fields=["OptOut"]) - if 'OptOut' not in role_data or not role_data['OptOut']: - return ResponderReturn(successful=False, - return_message='Role {} in account {} wasn\'t opted out'.format(message.role_name, - message.account)) + if "OptOut" not in role_data or not role_data["OptOut"]: + return ResponderReturn( + successful=False, + return_message="Role {} in account {} wasn't opted out".format( + message.role_name, message.account + ), + ) else: - dynamo.set_role_data(dynamo_table, role_id, {'OptOut': {}}) - return ResponderReturn(successful=True, - return_message='Cancelled opt-out for role {} in account {}'.format(message.role_name, - message.account)) + dynamo.set_role_data(dynamo_table, role_id, {"OptOut": {}}) + return ResponderReturn( + successful=True, + return_message="Cancelled opt-out for role {} in account {}".format( + message.role_name, message.account + ), + ) -@implements_command('rollback_role') +@implements_command("rollback_role") def rollback_role(dynamo_table, message): if not message.selection: - return ResponderReturn(successful=False, return_message='Rollback must contain a selection number') + return ResponderReturn( + successful=False, return_message="Rollback must contain a selection number" + ) - errors = cli.rollback_role(message.account, message.role_name, dynamo_table, CONFIG, hooks, - selection=message.selection, commit=True) + errors = cli.rollback_role( + message.account, + message.role_name, + dynamo_table, + CONFIG, + hooks, + selection=message.selection, + commit=True, + ) if errors: - return ResponderReturn(successful=False, return_message='Errors during rollback: {}'.format(errors)) + return ResponderReturn( + successful=False, return_message="Errors during rollback: {}".format(errors) + ) else: return ResponderReturn( successful=True, - return_message='Successfully rolled back role {} in account {}'.format( - message.role_name, message.account) + return_message="Successfully rolled back role {} in account {}".format( + message.role_name, message.account + ), ) diff --git a/repokid/filters/age/__init__.py b/repokid/filters/age/__init__.py index 8c94a904c..6ac4ce287 100644 --- a/repokid/filters/age/__init__.py +++ b/repokid/filters/age/__init__.py @@ -9,9 +9,9 @@ class AgeFilter(Filter): def apply(self, input_list): now = datetime.datetime.now(tzlocal()) try: - days_delta = self.config['minimum_age'] + days_delta = self.config["minimum_age"] except KeyError: - LOGGER.info('Minimum age not set in config, using default 90 days') + LOGGER.info("Minimum age not set in config, using default 90 days") days_delta = 90 ago = datetime.timedelta(days=days_delta) @@ -19,7 +19,10 @@ def apply(self, input_list): too_young = [] for role in input_list: if role.create_date > now - ago: - LOGGER.info('Role {name} created too recently to cleanup. ({date})'.format( - name=role.role_name, date=role.create_date)) + LOGGER.info( + "Role {name} created too recently to cleanup. ({date})".format( + name=role.role_name, date=role.create_date + ) + ) too_young.append(role) return too_young diff --git a/repokid/filters/blocklist/__init__.py b/repokid/filters/blocklist/__init__.py index f9c6e80d9..21f45d1b7 100644 --- a/repokid/filters/blocklist/__init__.py +++ b/repokid/filters/blocklist/__init__.py @@ -9,23 +9,32 @@ def get_blocklist_from_bucket(bucket_config): try: - s3_resource = boto3_cached_conn('s3', service_type='resource', - account_number=bucket_config.get('account_number'), - assume_role=bucket_config.get('assume_role', None), - session_name='repokid', - region=bucket_config.get('region', 'us-west-2')) - - s3_obj = s3_resource.Object(bucket_name=bucket_config['bucket_name'], key=bucket_config['key']) - blocklist = s3_obj.get()['Body'].read().decode("utf-8") + s3_resource = boto3_cached_conn( + "s3", + service_type="resource", + account_number=bucket_config.get("account_number"), + assume_role=bucket_config.get("assume_role", None), + session_name="repokid", + region=bucket_config.get("region", "us-west-2"), + ) + + s3_obj = s3_resource.Object( + bucket_name=bucket_config["bucket_name"], key=bucket_config["key"] + ) + blocklist = s3_obj.get()["Body"].read().decode("utf-8") blocklist_json = json.loads(blocklist) # Blocklist problems are really bad and we should quit rather than silently continue except (botocore.exceptions.ClientError, AttributeError): - LOGGER.error("S3 blocklist config was set but unable to connect retrieve object, quitting") + LOGGER.error( + "S3 blocklist config was set but unable to connect retrieve object, quitting" + ) sys.exit(1) except ValueError: - LOGGER.error("S3 blocklist config was set but the returned file is bad, quitting") + LOGGER.error( + "S3 blocklist config was set but the returned file is bad, quitting" + ) sys.exit(1) - if set(blocklist_json.keys()) != set(['arns', 'names']): + if set(blocklist_json.keys()) != set(["arns", "names"]): LOGGER.error("S3 blocklist file is malformed, quitting") sys.exit(1) return blocklist_json @@ -34,29 +43,45 @@ def get_blocklist_from_bucket(bucket_config): class BlocklistFilter(Filter): def __init__(self, config=None): blocklist_json = None - bucket_config = config.get('blocklist_bucket', config.get('blacklist_bucket', None)) + bucket_config = config.get( + "blocklist_bucket", config.get("blacklist_bucket", None) + ) if bucket_config: blocklist_json = get_blocklist_from_bucket(bucket_config) - current_account = config.get('current_account') or None + current_account = config.get("current_account") or None if not current_account: - LOGGER.error('Unable to get current account for Blocklist Filter') + LOGGER.error("Unable to get current account for Blocklist Filter") blocklisted_role_names = set() - blocklisted_role_names.update([rolename.lower() for rolename in config.get(current_account, [])]) - blocklisted_role_names.update([rolename.lower() for rolename in config.get('all', [])]) + blocklisted_role_names.update( + [rolename.lower() for rolename in config.get(current_account, [])] + ) + blocklisted_role_names.update( + [rolename.lower() for rolename in config.get("all", [])] + ) if blocklist_json: - blocklisted_role_names.update([name.lower() for name, accounts in blocklist_json['names'].items() if - ('all' in accounts or config.get('current_account') in accounts)]) + blocklisted_role_names.update( + [ + name.lower() + for name, accounts in blocklist_json["names"].items() + if ("all" in accounts or config.get("current_account") in accounts) + ] + ) - self.blocklisted_arns = set() if not blocklist_json else blocklist_json.get('arns', []) + self.blocklisted_arns = ( + set() if not blocklist_json else blocklist_json.get("arns", []) + ) self.blocklisted_role_names = blocklisted_role_names def apply(self, input_list): blocklisted_roles = [] for role in input_list: - if(role.role_name.lower() in self.blocklisted_role_names or role.arn in self.blocklisted_arns): + if ( + role.role_name.lower() in self.blocklisted_role_names + or role.arn in self.blocklisted_arns + ): blocklisted_roles.append(role) return blocklisted_roles diff --git a/repokid/filters/exclusive/__init__.py b/repokid/filters/exclusive/__init__.py index a6784ef52..0722def2d 100644 --- a/repokid/filters/exclusive/__init__.py +++ b/repokid/filters/exclusive/__init__.py @@ -6,13 +6,17 @@ class ExclusiveFilter(Filter): def __init__(self, config=None): - current_account = config.get('current_account') or None + current_account = config.get("current_account") or None if not current_account: - LOGGER.error('Unable to get current account for Exclusive Filter') + LOGGER.error("Unable to get current account for Exclusive Filter") exclusive_role_globs = set() - exclusive_role_globs.update([role_glob.lower() for role_glob in config.get(current_account, [])]) - exclusive_role_globs.update([role_glob.lower() for role_glob in config.get('all', [])]) + exclusive_role_globs.update( + [role_glob.lower() for role_glob in config.get(current_account, [])] + ) + exclusive_role_globs.update( + [role_glob.lower() for role_glob in config.get("all", [])] + ) self.exclusive_role_globs = exclusive_role_globs @@ -21,6 +25,10 @@ def apply(self, input_list): filtered_roles = [] for role_glob in self.exclusive_role_globs: - exclusive_roles += [role for role in input_list if fnmatch.fnmatch(role.role_name.lower(), role_glob)] + exclusive_roles += [ + role + for role in input_list + if fnmatch.fnmatch(role.role_name.lower(), role_glob) + ] filtered_roles = list(set(input_list) - set(exclusive_roles)) return filtered_roles diff --git a/repokid/filters/lambda/__init__.py b/repokid/filters/lambda/__init__.py index 205b8321b..b6e234dd2 100644 --- a/repokid/filters/lambda/__init__.py +++ b/repokid/filters/lambda/__init__.py @@ -6,6 +6,6 @@ def apply(self, input_list): lambda_roles = [] for role in input_list: - if 'lambda' in str(role.assume_role_policy_document).lower(): + if "lambda" in str(role.assume_role_policy_document).lower(): lambda_roles.append(role) return list(lambda_roles) diff --git a/repokid/filters/optout/__init__.py b/repokid/filters/optout/__init__.py index 6f049769f..d59c48999 100644 --- a/repokid/filters/optout/__init__.py +++ b/repokid/filters/optout/__init__.py @@ -11,6 +11,6 @@ def apply(self, input_list): opt_out_roles = [] for role in input_list: - if role.opt_out and role.opt_out['expire'] > self.current_time_epoch: + if role.opt_out and role.opt_out["expire"] > self.current_time_epoch: opt_out_roles.append(role) return list(opt_out_roles) diff --git a/repokid/hooks/__init__.py b/repokid/hooks/__init__.py index a3b2ccd51..4307ef3d0 100644 --- a/repokid/hooks/__init__.py +++ b/repokid/hooks/__init__.py @@ -23,6 +23,7 @@ def _implements_hook(func): if not hasattr(func, "_implements_hook"): func._implements_hook = {"hook_name": hook_name, "priority": priority} return func + return _implements_hook diff --git a/repokid/hooks/loggers/__init__.py b/repokid/hooks/loggers/__init__.py index 2c0634260..787a08d59 100644 --- a/repokid/hooks/loggers/__init__.py +++ b/repokid/hooks/loggers/__init__.py @@ -3,48 +3,72 @@ from repokid.role import Role -@hooks.implements_hook('BEFORE_REPO_ROLES', 1) +@hooks.implements_hook("BEFORE_REPO_ROLES", 1) def log_before_repo_roles(input_dict): LOGGER.debug("Calling DURING_REPOABLE_CALCULATION hooks") - if not all(required in input_dict for required in ['account_number', 'roles']): - raise hooks.MissingHookParamaeter("Did not get all required parameters for BEFORE_REPO_ROLES hook") + if not all(required in input_dict for required in ["account_number", "roles"]): + raise hooks.MissingHookParamaeter( + "Did not get all required parameters for BEFORE_REPO_ROLES hook" + ) return input_dict -@hooks.implements_hook('DURING_REPOABLE_CALCULATION', 1) +@hooks.implements_hook("DURING_REPOABLE_CALCULATION", 1) def log_during_repoable_calculation_hooks(input_dict): LOGGER.debug("Calling DURING_REPOABLE_CALCULATION hooks") - if not all(required in input_dict for required in['account_number', 'role_name', 'potentially_repoable_permissions', - 'minimum_age']): - raise hooks.MissingHookParamaeter("Did not get all required parameters for DURING_REPOABLE_CALCULATION hook") + if not all( + required in input_dict + for required in [ + "account_number", + "role_name", + "potentially_repoable_permissions", + "minimum_age", + ] + ): + raise hooks.MissingHookParamaeter( + "Did not get all required parameters for DURING_REPOABLE_CALCULATION hook" + ) return input_dict -@hooks.implements_hook('DURING_REPOABLE_CALCULATION_BATCH', 1) +@hooks.implements_hook("DURING_REPOABLE_CALCULATION_BATCH", 1) def log_during_repoable_calculation_batch_hooks(input_dict): LOGGER.debug("Calling DURING_REPOABLE_CALCULATION_BATCH hooks") - if not all(required in input_dict for required in['role_batch', 'potentially_repoable_permissions', 'minimum_age']): + if not all( + required in input_dict + for required in [ + "role_batch", + "potentially_repoable_permissions", + "minimum_age", + ] + ): raise hooks.MissingHookParamaeter( - "Did not get all required parameters for DURING_REPOABLE_CALCULATION_BATCH hook") - for role in input_dict['role_batch']: + "Did not get all required parameters for DURING_REPOABLE_CALCULATION_BATCH hook" + ) + for role in input_dict["role_batch"]: if not isinstance(role, Role): raise hooks.MissingHookParamaeter( - "Role_batch needs to be a series of Role objects in DURING_REPOABLE_CALCULATION_BATCH hook") + "Role_batch needs to be a series of Role objects in DURING_REPOABLE_CALCULATION_BATCH hook" + ) return input_dict -@hooks.implements_hook('AFTER_SCHEDULE_REPO', 1) +@hooks.implements_hook("AFTER_SCHEDULE_REPO", 1) def log_after_schedule_repo_hooks(input_dict): LOGGER.debug("Calling AFTER_SCHEDULE_REPO hooks") - if 'roles' not in input_dict: - raise hooks.MissingHookParamaeter("Required key 'roles' not passed to AFTER_SCHEDULE_REPO") + if "roles" not in input_dict: + raise hooks.MissingHookParamaeter( + "Required key 'roles' not passed to AFTER_SCHEDULE_REPO" + ) return input_dict -@hooks.implements_hook('AFTER_REPO', 1) +@hooks.implements_hook("AFTER_REPO", 1) def log_after_repo_hooks(input_dict): LOGGER.debug("Calling AFTER_REPO hooks") - if 'role' not in input_dict: - raise hooks.MissingHookParamaeter("Required key 'role' not passed to AFTER_REPO") + if "role" not in input_dict: + raise hooks.MissingHookParamaeter( + "Required key 'role' not passed to AFTER_REPO" + ) return input_dict diff --git a/repokid/role.py b/repokid/role.py index 72bd73c57..86a836786 100644 --- a/repokid/role.py +++ b/repokid/role.py @@ -13,40 +13,52 @@ # limitations under the License. import copy -dict_to_attr = {'AAData': {'attribute': 'aa_data', 'default': dict()}, - 'Account': {'attribute': 'account', 'default': None}, - 'Active': {'attribute': 'active', 'default': True}, - 'Arn': {'attribute': 'arn', 'default': None}, - 'AssumeRolePolicyDocument': {'attribute': 'assume_role_policy_document', 'default': None}, - 'CreateDate': {'attribute': 'create_date', 'default': None}, - 'DisqualifiedBy': {'attribute': 'disqualified_by', 'default': list()}, - 'NoRepoPermissions': {'attribute': 'no_repo_permissions', 'default': dict()}, - 'OptOut': {'attribute': 'opt_out', 'default': dict()}, - 'Policies': {'attribute': 'policies', 'default': list()}, - 'Refreshed': {'attribute': 'refreshed', 'default': str()}, - 'RepoablePermissions': {'attribute': 'repoable_permissions', 'default': int()}, - 'RepoableServices': {'attribute': 'repoable_services', 'default': list()}, - 'Repoed': {'attribute': 'repoed', 'default': str()}, - 'RepoScheduled': {'attribute': 'repo_scheduled', 'default': int()}, - 'RoleId': {'attribute': 'role_id', 'default': None}, - 'RoleName': {'attribute': 'role_name', 'default': None}, - 'ScheduledPerms': {'attribute': 'scheduled_perms', 'default': dict()}, - 'Stats': {'attribute': 'stats', 'default': list()}, - 'Tags': {'attribute': 'tags', 'default': list()}, - 'TotalPermissions': {'attribute': 'total_permissions', 'default': int()}} +dict_to_attr = { + "AAData": {"attribute": "aa_data", "default": dict()}, + "Account": {"attribute": "account", "default": None}, + "Active": {"attribute": "active", "default": True}, + "Arn": {"attribute": "arn", "default": None}, + "AssumeRolePolicyDocument": { + "attribute": "assume_role_policy_document", + "default": None, + }, + "CreateDate": {"attribute": "create_date", "default": None}, + "DisqualifiedBy": {"attribute": "disqualified_by", "default": list()}, + "NoRepoPermissions": {"attribute": "no_repo_permissions", "default": dict()}, + "OptOut": {"attribute": "opt_out", "default": dict()}, + "Policies": {"attribute": "policies", "default": list()}, + "Refreshed": {"attribute": "refreshed", "default": str()}, + "RepoablePermissions": {"attribute": "repoable_permissions", "default": int()}, + "RepoableServices": {"attribute": "repoable_services", "default": list()}, + "Repoed": {"attribute": "repoed", "default": str()}, + "RepoScheduled": {"attribute": "repo_scheduled", "default": int()}, + "RoleId": {"attribute": "role_id", "default": None}, + "RoleName": {"attribute": "role_name", "default": None}, + "ScheduledPerms": {"attribute": "scheduled_perms", "default": dict()}, + "Stats": {"attribute": "stats", "default": list()}, + "Tags": {"attribute": "tags", "default": list()}, + "TotalPermissions": {"attribute": "total_permissions", "default": int()}, +} class Role(object): def __init__(self, role_dict): - for key, value in dict_to_attr.items(): - setattr(self, value['attribute'], role_dict[key] if key in role_dict else copy.copy(value['default'])) + for key, value in list(dict_to_attr.items()): + setattr( + self, + value["attribute"], + role_dict[key] if key in role_dict else copy.copy(value["default"]), + ) def as_dict(self): - return {key: getattr(self, value['attribute']) for key, value in dict_to_attr.items()} + return { + key: getattr(self, value["attribute"]) + for key, value in dict_to_attr.items() + } def set_attributes(self, attributes_dict): for key, value in attributes_dict.items(): - setattr(self, dict_to_attr[key]['attribute'], value) + setattr(self, dict_to_attr[key]["attribute"], value) def __eq__(self, other): return self.role_id == other @@ -72,8 +84,9 @@ def __repr__(self): return str([role.role_id for role in self.roles]) def __eq__(self, other): - return (all(role.role_id in other for role in self.roles) and - all(role.role_id in self.roles for role in other)) + return all(role.role_id in other for role in self.roles) and all( + role.role_id in self.roles for role in other + ) def append(self, role): self.roles.append(role) diff --git a/repokid/tests/artifacts/hook/__init__.py b/repokid/tests/artifacts/hook/__init__.py index fa5711b82..9d1418fb2 100644 --- a/repokid/tests/artifacts/hook/__init__.py +++ b/repokid/tests/artifacts/hook/__init__.py @@ -1,11 +1,11 @@ import repokid.hooks as hooks -@hooks.implements_hook('TEST_HOOK', 2) +@hooks.implements_hook("TEST_HOOK", 2) def function_2(): pass -@hooks.implements_hook('TEST_HOOK', 1) +@hooks.implements_hook("TEST_HOOK", 1) def function_1(): pass diff --git a/repokid/tests/test_dispatcher_cli.py b/repokid/tests/test_dispatcher_cli.py index 33ccab0fd..35d87b2be 100644 --- a/repokid/tests/test_dispatcher_cli.py +++ b/repokid/tests/test_dispatcher_cli.py @@ -7,40 +7,58 @@ DYNAMO_TABLE = None -MESSAGE = dispatcher_cli.Message('command', 'account', 'role', 'respond_channel', respond_user='some_user', - requestor='a_requestor', reason='some_reason', selection='some_selection') +MESSAGE = dispatcher_cli.Message( + "command", + "account", + "role", + "respond_channel", + respond_user="some_user", + requestor="a_requestor", + reason="some_reason", + selection="some_selection", +) class TestDispatcherCLI(object): def test_message_creation(self): test_message = MESSAGE - assert test_message.command == 'command' - assert test_message.account == 'account' - assert test_message.role_name == 'role' - assert test_message.respond_channel == 'respond_channel' - assert test_message.respond_user == 'some_user' - assert test_message.requestor == 'a_requestor' - assert test_message.reason == 'some_reason' - assert test_message.selection == 'some_selection' + assert test_message.command == "command" + assert test_message.account == "account" + assert test_message.role_name == "role" + assert test_message.respond_channel == "respond_channel" + assert test_message.respond_user == "some_user" + assert test_message.requestor == "a_requestor" + assert test_message.reason == "some_reason" + assert test_message.selection == "some_selection" def test_schema(self): schema = dispatcher_cli.MessageSchema() # happy path - test_message = {'command': 'list_repoable_services', 'account': '123', 'role_name': 'abc', - 'respond_channel': 'channel', 'respond_user': 'user'} + test_message = { + "command": "list_repoable_services", + "account": "123", + "role_name": "abc", + "respond_channel": "channel", + "respond_user": "user", + } result = schema.load(test_message) assert not result.errors # missing required field command - test_message = {'account': '123', 'role_name': 'abc', 'respond_channel': 'channel', 'respond_user': 'user'} + test_message = { + "account": "123", + "role_name": "abc", + "respond_channel": "channel", + "respond_user": "user", + } result = schema.load(test_message) assert result.errors - @patch('repokid.utils.dynamo.find_role_in_cache') - @patch('repokid.utils.dynamo.get_role_data') + @patch("repokid.utils.dynamo.find_role_in_cache") + @patch("repokid.utils.dynamo.get_role_data") def test_list_repoable_services(self, mock_get_role_data, mock_find_role_in_cache): - mock_find_role_in_cache.side_effect = [None, 'ROLE_ID_A'] + mock_find_role_in_cache.side_effect = [None, "ROLE_ID_A"] (success, _) = dispatcher.list_repoable_services(DYNAMO_TABLE, MESSAGE) assert not success @@ -48,10 +66,10 @@ def test_list_repoable_services(self, mock_get_role_data, mock_find_role_in_cach (success, _) = dispatcher.list_repoable_services(DYNAMO_TABLE, MESSAGE) assert success - @patch('repokid.utils.dynamo.find_role_in_cache') - @patch('repokid.utils.dynamo.get_role_data') + @patch("repokid.utils.dynamo.find_role_in_cache") + @patch("repokid.utils.dynamo.get_role_data") def test_list_role_rollbacks(self, mock_get_role_data, mock_find_role_in_cache): - mock_find_role_in_cache.side_effect = [None, 'ROLE_ID_A'] + mock_find_role_in_cache.side_effect = [None, "ROLE_ID_A"] (success, _) = dispatcher.list_role_rollbacks(DYNAMO_TABLE, MESSAGE) assert not success @@ -59,19 +77,22 @@ def test_list_role_rollbacks(self, mock_get_role_data, mock_find_role_in_cache): (success, _) = dispatcher.list_repoable_services(DYNAMO_TABLE, MESSAGE) assert success - @patch('time.time') - @patch('repokid.utils.dynamo.find_role_in_cache') - @patch('repokid.utils.dynamo.get_role_data') - @patch('repokid.utils.dynamo.set_role_data') - def test_opt_out(self, mock_set_role_data, mock_get_role_data, mock_find_role_in_cache, mock_time): - mock_find_role_in_cache.side_effect = [None, 'ROLE_ID_A'] + @patch("time.time") + @patch("repokid.utils.dynamo.find_role_in_cache") + @patch("repokid.utils.dynamo.get_role_data") + @patch("repokid.utils.dynamo.set_role_data") + def test_opt_out( + self, mock_set_role_data, mock_get_role_data, mock_find_role_in_cache, mock_time + ): + mock_find_role_in_cache.side_effect = [None, "ROLE_ID_A"] GET_ROLE_DATA_FOR_FIND = {} - mock_get_role_data.side_effect = [GET_ROLE_DATA_FOR_FIND, # role not found - GET_ROLE_DATA_FOR_FIND, # opt out exists - {'OptOut': {'owner': 'somebody', 'reason': 'because'}}, - GET_ROLE_DATA_FOR_FIND, - {'OptOut': {}} # success - ] + mock_get_role_data.side_effect = [ + GET_ROLE_DATA_FOR_FIND, # role not found + GET_ROLE_DATA_FOR_FIND, # opt out exists + {"OptOut": {"owner": "somebody", "reason": "because"}}, + GET_ROLE_DATA_FOR_FIND, + {"OptOut": {}}, # success + ] mock_time.return_value = 0 @@ -91,7 +112,16 @@ def test_opt_out(self, mock_set_role_data, mock_get_role_data, mock_find_role_in (success, msg) = dispatcher.opt_out(DYNAMO_TABLE, MESSAGE) assert success - assert mock_set_role_data.mock_calls == [call(DYNAMO_TABLE, 'ROLE_ID_A', - {'OptOut': {'owner': MESSAGE.requestor, - 'reason': MESSAGE.reason, - 'expire': expire_epoch}})] + assert mock_set_role_data.mock_calls == [ + call( + DYNAMO_TABLE, + "ROLE_ID_A", + { + "OptOut": { + "owner": MESSAGE.requestor, + "reason": MESSAGE.reason, + "expire": expire_epoch, + } + }, + ) + ] diff --git a/repokid/tests/test_hooks.py b/repokid/tests/test_hooks.py index d28b6909a..8f5c00860 100644 --- a/repokid/tests/test_hooks.py +++ b/repokid/tests/test_hooks.py @@ -8,22 +8,22 @@ def func_a(input_dict): - input_dict['value'] += 1 + input_dict["value"] += 1 return input_dict def func_b(input_dict): - input_dict['value'] += 1 + input_dict["value"] += 1 return input_dict def func_c(input_dict): - input_dict['value'] += 10 + input_dict["value"] += 10 return input_dict def func_d(input_value): - required_vals = ['a', 'b'] + required_vals = ["a", "b"] if not all(val in input_value for val in required_vals): raise repokid.hooks.MissingHookParamaeter @@ -34,48 +34,66 @@ def func_e(input_value): class TestHooks(object): def test_call_hooks(self): - hooks = {'TEST_HOOK': [func_a, func_b], 'NOT_CALLED': [func_c], 'MISSING_PARAMETER': [func_d], - 'MISSING_OUTPUT': [func_e]} - hook_args = {'value': 0} - output_value = repokid.hooks.call_hooks(hooks, 'TEST_HOOK', hook_args) + hooks = { + "TEST_HOOK": [func_a, func_b], + "NOT_CALLED": [func_c], + "MISSING_PARAMETER": [func_d], + "MISSING_OUTPUT": [func_e], + } + hook_args = {"value": 0} + output_value = repokid.hooks.call_hooks(hooks, "TEST_HOOK", hook_args) # func_a and func_b are called to increment 0 --> 2, func_c is not called - assert output_value['value'] == 2 + assert output_value["value"] == 2 # missing required parameter b with pytest.raises(repokid.hooks.MissingHookParamaeter): - output_value = repokid.hooks.call_hooks(hooks, 'MISSING_PARAMETER', {'a': '1'}) + output_value = repokid.hooks.call_hooks( + hooks, "MISSING_PARAMETER", {"a": "1"} + ) with pytest.raises(repokid.hooks.MissingOutputInHook): - output_value = repokid.hooks.call_hooks(hooks, 'MISSING_OUTPUT', {'a': 1}) + output_value = repokid.hooks.call_hooks(hooks, "MISSING_OUTPUT", {"a": 1}) def test_get_hooks(self): - hooks_config = ['repokid.tests.artifacts.hook'] + hooks_config = ["repokid.tests.artifacts.hook"] hooks = repokid.cli.repokid_cli._get_hooks(hooks_config) # key is correct, both functions are loaded and in correct priority order - assert hooks == {'TEST_HOOK': [function_1, function_2]} + assert hooks == {"TEST_HOOK": [function_1, function_2]} def test_implements_hook(self): def func_a(): pass - @repokid.hooks.implements_hook('DECORATOR_TEST', 1) + @repokid.hooks.implements_hook("DECORATOR_TEST", 1) def func_b(): pass assert not hasattr(func_a, "_implements_hook") assert hasattr(func_b, "_implements_hook") - assert func_b._implements_hook == {"hook_name": 'DECORATOR_TEST', "priority": 1} + assert func_b._implements_hook == {"hook_name": "DECORATOR_TEST", "priority": 1} def test_log_during_repoable_calculation_batch_hooks(self): - hooks = {'DURING_REPOABLE_CALCULATION_BATCH': [log_during_repoable_calculation_batch_hooks]} - - input_dict = {'role_batch': [Role(ROLES[0]), "def"], 'potentially_repoable_permissions': [], 'minimum_age': 1} + hooks = { + "DURING_REPOABLE_CALCULATION_BATCH": [ + log_during_repoable_calculation_batch_hooks + ] + } + + input_dict = { + "role_batch": [Role(ROLES[0]), "def"], + "potentially_repoable_permissions": [], + "minimum_age": 1, + } with pytest.raises(repokid.hooks.MissingHookParamaeter): # role_batch', 'potentially_repoable_permissions', 'minimum_age' - repokid.hooks.call_hooks(hooks, 'DURING_REPOABLE_CALCULATION_BATCH', input_dict) - - input_dict['role_batch'] = [Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2])] - assert input_dict == repokid.hooks.call_hooks(hooks, 'DURING_REPOABLE_CALCULATION_BATCH', input_dict) + repokid.hooks.call_hooks( + hooks, "DURING_REPOABLE_CALCULATION_BATCH", input_dict + ) + + input_dict["role_batch"] = [Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2])] + assert input_dict == repokid.hooks.call_hooks( + hooks, "DURING_REPOABLE_CALCULATION_BATCH", input_dict + ) diff --git a/repokid/tests/test_repokid_cli.py b/repokid/tests/test_repokid_cli.py index e16ddd9b0..fbbc01bda 100644 --- a/repokid/tests/test_repokid_cli.py +++ b/repokid/tests/test_repokid_cli.py @@ -24,134 +24,118 @@ AARDVARK_DATA = { "arn:aws:iam::123456789012:role/all_services_used": [ - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "iam"}, - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "s3"}], - + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "iam"}, + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "s3"}, + ], "arn:aws:iam::123456789012:role/unused_ec2": [ - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "iam"}, - {"lastAuthenticated": 0, - "serviceNamespace": "ec2"}], - + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "iam"}, + {"lastAuthenticated": 0, "serviceNamespace": "ec2"}, + ], "arn:aws:iam::123456789012:role/young_role": [ - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "iam"}, - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "s3"}], - + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "iam"}, + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "s3"}, + ], "arn:aws:iam::123456789012:role/additional_unused_ec2": [ - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "iam"}, - {"lastAuthenticated": 0, - "serviceNamespace": "ec2"}, - {"lastAuthenticated": 0, - "serviceNamespace": "unsupported_service"}, - {"lastAuthenticated": 0, - "serviceNamespace": "supported_service"}, + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "iam"}, + {"lastAuthenticated": 0, "serviceNamespace": "ec2"}, + {"lastAuthenticated": 0, "serviceNamespace": "unsupported_service"}, + {"lastAuthenticated": 0, "serviceNamespace": "supported_service"}, ], - "arn:aws:iam::123456789012:role/unused_iam": [ - {"lastAuthenticated": 0, - "serviceNamespace": "iam"}, - {"lastAuthenticated": int(time.time()) * 1000, - "serviceNamespace": "ec2"}], + {"lastAuthenticated": 0, "serviceNamespace": "iam"}, + {"lastAuthenticated": int(time.time()) * 1000, "serviceNamespace": "ec2"}, + ], } ROLE_POLICIES = { - 'all_services_used': { - 'iam_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "all_services_used": { + "iam_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + "Resource": ["*"], + "Effect": "Allow", } - ] + ], }, - - 's3_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "s3_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['s3:CreateBucket', 's3:DeleteBucket'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["s3:CreateBucket", "s3:DeleteBucket"], + "Resource": ["*"], + "Effect": "Allow", } - ] - } + ], + }, }, - 'unused_ec2': { - 'iam_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "unused_ec2": { + "iam_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + "Resource": ["*"], + "Effect": "Allow", } - ] + ], }, - - 'ec2_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "ec2_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['ec2:AllocateHosts', 'ec2:AssociateAddress'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["ec2:AllocateHosts", "ec2:AssociateAddress"], + "Resource": ["*"], + "Effect": "Allow", } - ] - } + ], + }, }, - 'additional_unused_ec2': { - 'iam_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "additional_unused_ec2": { + "iam_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + "Resource": ["*"], + "Effect": "Allow", } - ] + ], }, - - 'ec2_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "ec2_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['ec2:AllocateHosts', 'ec2:AssociateAddress'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["ec2:AllocateHosts", "ec2:AssociateAddress"], + "Resource": ["*"], + "Effect": "Allow", } - ] - } + ], + }, }, - 'unused_iam': { - 'iam_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "unused_iam": { + "iam_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + "Resource": ["*"], + "Effect": "Allow", } - ] + ], }, - - 'ec2_perms': { - 'Version': '2012-10-17', - 'Statement': [ + "ec2_perms": { + "Version": "2012-10-17", + "Statement": [ { - 'Action': ['ec2:AllocateHosts', 'ec2:AssociateAddress'], - 'Resource': ['*'], - 'Effect': 'Allow' + "Action": ["ec2:AllocateHosts", "ec2:AssociateAddress"], + "Resource": ["*"], + "Effect": "Allow", } - ] - } - } + ], + }, + }, } ROLES = [ @@ -160,7 +144,7 @@ "CreateDate": datetime.datetime(2017, 1, 31, 12, 0, 0, tzinfo=tzlocal()), "RoleId": "AROAABCDEFGHIJKLMNOPA", "RoleName": "all_services_used", - "Active": True + "Active": True, }, { "Arn": "arn:aws:iam::123456789012:role/unused_ec2", @@ -205,196 +189,325 @@ "RepoablePermissions": 0, "Repoed": "Never", "RepoableServices": [], - "Refreshed": "Someday" + "Refreshed": "Someday", }, { "TotalPermissions": 4, "RepoablePermissions": 2, "Repoed": "Never", "RepoableServices": ["ec2"], - "Refreshed": "Someday" + "Refreshed": "Someday", }, { "TotalPermissions": 4, "RepoablePermissions": 0, "Repoed": "Never", "RepoableServices": [], - "Refreshed": "Someday" + "Refreshed": "Someday", }, { "TotalPermissions": 4, "RepoablePermissions": 0, "Repoed": "Never", "RepoableServices": [], - "Refreshed": "Someday" + "Refreshed": "Someday", }, { "TotalPermissions": 4, "RepoablePermissions": 2, "Repoed": "Never", "RepoableServices": ["ec2"], - "Refreshed": "Someday" + "Refreshed": "Someday", }, { "TotalPermissions": 4, "RepoablePermissions": 2, "Repoed": "Never", "RepoableServices": ["ec2"], - "Refreshed": "Someday" - } + "Refreshed": "Someday", + }, ] class TestRepokidCLI(object): - @patch('repokid.utils.roledata.update_stats') - @patch('repokid.utils.roledata.find_and_mark_inactive') - @patch('repokid.utils.roledata.update_role_data') - @patch('repokid.utils.roledata._calculate_repo_scores') - @patch('repokid.cli.repokid_cli.set_role_data') - @patch('repokid.cli.repokid_cli._get_aardvark_data') - @patch('repokid.cli.repokid_cli.get_account_authorization_details') - def test_repokid_update_role_cache(self, mock_get_account_authorization_details, mock_get_aardvark_data, - mock_set_role_data, mock_calculate_repo_scores, mock_update_role_data, - mock_find_and_mark_inactive, mock_update_stats): + @patch("repokid.utils.roledata.update_stats") + @patch("repokid.utils.roledata.find_and_mark_inactive") + @patch("repokid.utils.roledata.update_role_data") + @patch("repokid.utils.roledata._calculate_repo_scores") + @patch("repokid.cli.repokid_cli.set_role_data") + @patch("repokid.cli.repokid_cli._get_aardvark_data") + @patch("repokid.cli.repokid_cli.get_account_authorization_details") + def test_repokid_update_role_cache( + self, + mock_get_account_authorization_details, + mock_get_aardvark_data, + mock_set_role_data, + mock_calculate_repo_scores, + mock_update_role_data, + mock_find_and_mark_inactive, + mock_update_stats, + ): hooks = {} role_data = ROLES[:3] - role_data[0]['RolePolicyList'] = [{'PolicyName': 'all_services_used', - 'PolicyDocument': ROLE_POLICIES['all_services_used']}] - role_data[1]['RolePolicyList'] = [{'PolicyName': 'unused_ec2', - 'PolicyDocument': ROLE_POLICIES['unused_ec2']}] - role_data[2]['RolePolicyList'] = [{'PolicyName': 'all_services_used', - 'PolicyDocument': ROLE_POLICIES['all_services_used']}] + role_data[0]["RolePolicyList"] = [ + { + "PolicyName": "all_services_used", + "PolicyDocument": ROLE_POLICIES["all_services_used"], + } + ] + role_data[1]["RolePolicyList"] = [ + {"PolicyName": "unused_ec2", "PolicyDocument": ROLE_POLICIES["unused_ec2"]} + ] + role_data[2]["RolePolicyList"] = [ + { + "PolicyName": "all_services_used", + "PolicyDocument": ROLE_POLICIES["all_services_used"], + } + ] mock_get_account_authorization_details.side_effect = [role_data] mock_get_aardvark_data.return_value = AARDVARK_DATA def update_role_data(dynamo_table, account_number, role, current_policies): - role.policies = [{'Policy': current_policies}] + role.policies = [{"Policy": current_policies}] mock_update_role_data.side_effect = update_role_data - config = {"aardvark_api_location": "", "connection_iam": {}, - "active_filters": ["repokid.filters.age:AgeFilter"], "filter_config": - {"AgeFilter": {"minimum_age": 90}, "BlocklistFilter": {}}} + config = { + "aardvark_api_location": "", + "connection_iam": {}, + "active_filters": ["repokid.filters.age:AgeFilter"], + "filter_config": {"AgeFilter": {"minimum_age": 90}, "BlocklistFilter": {}}, + } console_logger = logging.StreamHandler() console_logger.setLevel(logging.WARNING) - repokid.cli.repokid_cli.LOGGER = logging.getLogger('test') + repokid.cli.repokid_cli.LOGGER = logging.getLogger("test") repokid.cli.repokid_cli.LOGGER.addHandler(console_logger) dynamo_table = None - account_number = '123456789012' + account_number = "123456789012" - repokid.cli.repokid_cli.update_role_cache(account_number, dynamo_table, config, hooks) + repokid.cli.repokid_cli.update_role_cache( + account_number, dynamo_table, config, hooks + ) roles = Roles([Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2])]) - assert mock_calculate_repo_scores.mock_calls == [call(roles, 90, hooks, False, 100)] + assert mock_calculate_repo_scores.mock_calls == [ + call(roles, 90, hooks, False, 100) + ] # validate update data called for each role assert mock_update_role_data.mock_calls == [ - call(dynamo_table, account_number, Role(ROLES[0]), {'all_services_used': - ROLE_POLICIES['all_services_used']}), - call(dynamo_table, account_number, Role(ROLES[1]), {'unused_ec2': ROLE_POLICIES['unused_ec2']}), - call(dynamo_table, account_number, Role(ROLES[2]), {'all_services_used': - ROLE_POLICIES['all_services_used']})] + call( + dynamo_table, + account_number, + Role(ROLES[0]), + {"all_services_used": ROLE_POLICIES["all_services_used"]}, + ), + call( + dynamo_table, + account_number, + Role(ROLES[1]), + {"unused_ec2": ROLE_POLICIES["unused_ec2"]}, + ), + call( + dynamo_table, + account_number, + Role(ROLES[2]), + {"all_services_used": ROLE_POLICIES["all_services_used"]}, + ), + ] # all roles active - assert mock_find_and_mark_inactive.mock_calls == [call(dynamo_table, account_number, - [Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2])])] + assert mock_find_and_mark_inactive.mock_calls == [ + call( + dynamo_table, + account_number, + [Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2])], + ) + ] # TODO: validate total permission, repoable, etc are getting updated properly - assert mock_update_stats.mock_calls == [call(dynamo_table, roles, source='Scan')] + assert mock_update_stats.mock_calls == [ + call(dynamo_table, roles, source="Scan") + ] # TODO: set_role_data called with - @patch('tabview.view') - @patch('repokid.cli.repokid_cli.get_role_data') - @patch('repokid.cli.repokid_cli.role_ids_for_account') - def test_repokid_display_roles(self, mock_role_ids_for_account, mock_get_role_data, mock_tabview): + @patch("tabview.view") + @patch("repokid.cli.repokid_cli.get_role_data") + @patch("repokid.cli.repokid_cli.role_ids_for_account") + def test_repokid_display_roles( + self, mock_role_ids_for_account, mock_get_role_data, mock_tabview + ): console_logger = logging.StreamHandler() console_logger.setLevel(logging.WARNING) - repokid.cli.repokid_cli.LOGGER = logging.getLogger('test') + repokid.cli.repokid_cli.LOGGER = logging.getLogger("test") repokid.cli.repokid_cli.LOGGER.addHandler(console_logger) - mock_role_ids_for_account.return_value = ['AROAABCDEFGHIJKLMNOPA', 'AROAABCDEFGHIJKLMNOPB', - 'AROAABCDEFGHIJKLMNOPC', 'AROAABCDEFGHIJKLMNOPD'] + mock_role_ids_for_account.return_value = [ + "AROAABCDEFGHIJKLMNOPA", + "AROAABCDEFGHIJKLMNOPB", + "AROAABCDEFGHIJKLMNOPC", + "AROAABCDEFGHIJKLMNOPD", + ] for x, role in enumerate(ROLES_FOR_DISPLAY): role.update(ROLES[x]) # loop over all roles twice (one for each call below) - mock_get_role_data.side_effect = [ROLES_FOR_DISPLAY[0], ROLES_FOR_DISPLAY[1], ROLES_FOR_DISPLAY[2], - ROLES_FOR_DISPLAY[3], ROLES_FOR_DISPLAY[0], ROLES_FOR_DISPLAY[1], - ROLES_FOR_DISPLAY[2], ROLES_FOR_DISPLAY[3]] + mock_get_role_data.side_effect = [ + ROLES_FOR_DISPLAY[0], + ROLES_FOR_DISPLAY[1], + ROLES_FOR_DISPLAY[2], + ROLES_FOR_DISPLAY[3], + ROLES_FOR_DISPLAY[0], + ROLES_FOR_DISPLAY[1], + ROLES_FOR_DISPLAY[2], + ROLES_FOR_DISPLAY[3], + ] - repokid.cli.repokid_cli.display_roles('123456789012', None, inactive=True) - repokid.cli.repokid_cli.display_roles('123456789012', None, inactive=False) + repokid.cli.repokid_cli.display_roles("123456789012", None, inactive=True) + repokid.cli.repokid_cli.display_roles("123456789012", None, inactive=False) # first call has inactive role, second doesn't because it's filtered assert mock_tabview.mock_calls == [ - call([['Name', 'Refreshed', 'Disqualified By', 'Can be repoed', 'Permissions', 'Repoable', 'Repoed', - 'Services'], - ['all_services_used', "Someday", [], True, 4, 0, 'Never', []], - ['inactive_role', "Someday", [], True, 4, 0, 'Never', []], - ['young_role', "Someday", [], True, 4, 0, 'Never', []], - ['unused_ec2', "Someday", [], True, 4, 2, 'Never', ['ec2']]]), - - call([['Name', 'Refreshed', 'Disqualified By', 'Can be repoed', 'Permissions', 'Repoable', 'Repoed', - 'Services'], - ['all_services_used', "Someday", [], True, 4, 0, 'Never', []], - ['young_role', "Someday", [], True, 4, 0, 'Never', []], - ['unused_ec2', "Someday", [], True, 4, 2, 'Never', ['ec2']]])] - - @patch('repokid.hooks.call_hooks') - @patch('repokid.cli.repokid_cli.get_role_data') - @patch('repokid.cli.repokid_cli.set_role_data') - @patch('repokid.cli.repokid_cli.role_ids_for_account') - @patch('time.time') - def test_schedule_repo(self, mock_time, mock_role_ids_for_account, mock_set_role_data, mock_get_role_data, - mock_call_hooks): + call( + [ + [ + "Name", + "Refreshed", + "Disqualified By", + "Can be repoed", + "Permissions", + "Repoable", + "Repoed", + "Services", + ], + ["all_services_used", "Someday", [], True, 4, 0, "Never", []], + ["inactive_role", "Someday", [], True, 4, 0, "Never", []], + ["young_role", "Someday", [], True, 4, 0, "Never", []], + ["unused_ec2", "Someday", [], True, 4, 2, "Never", ["ec2"]], + ] + ), + call( + [ + [ + "Name", + "Refreshed", + "Disqualified By", + "Can be repoed", + "Permissions", + "Repoable", + "Repoed", + "Services", + ], + ["all_services_used", "Someday", [], True, 4, 0, "Never", []], + ["young_role", "Someday", [], True, 4, 0, "Never", []], + ["unused_ec2", "Someday", [], True, 4, 2, "Never", ["ec2"]], + ] + ), + ] + + @patch("repokid.hooks.call_hooks") + @patch("repokid.cli.repokid_cli.get_role_data") + @patch("repokid.cli.repokid_cli.set_role_data") + @patch("repokid.cli.repokid_cli.role_ids_for_account") + @patch("time.time") + def test_schedule_repo( + self, + mock_time, + mock_role_ids_for_account, + mock_set_role_data, + mock_get_role_data, + mock_call_hooks, + ): hooks = {} - mock_role_ids_for_account.return_value = ['AROAABCDEFGHIJKLMNOPA', 'AROAABCDEFGHIJKLMNOPB'] + mock_role_ids_for_account.return_value = [ + "AROAABCDEFGHIJKLMNOPA", + "AROAABCDEFGHIJKLMNOPB", + ] # first role is not repoable, second role is repoable - ROLES_FOR_DISPLAY[0].update({'RoleId': 'AROAABCDEFGHIJKLMNOPA'}) - ROLES_FOR_DISPLAY[1].update({'RoleId': 'AROAABCDEFGHIJKLMNOPB'}) + ROLES_FOR_DISPLAY[0].update({"RoleId": "AROAABCDEFGHIJKLMNOPA"}) + ROLES_FOR_DISPLAY[1].update({"RoleId": "AROAABCDEFGHIJKLMNOPB"}) mock_get_role_data.side_effect = [ROLES_FOR_DISPLAY[0], ROLES_FOR_DISPLAY[1]] mock_time.return_value = 1 - config = {'repo_schedule_period_days': 1} + config = {"repo_schedule_period_days": 1} - repokid.cli.repokid_cli.schedule_repo('1234567890', None, config, hooks) + repokid.cli.repokid_cli.schedule_repo("1234567890", None, config, hooks) - assert mock_set_role_data.mock_calls == [call(None, 'AROAABCDEFGHIJKLMNOPB', - {'RepoScheduled': 86401, 'ScheduledPerms': ['ec2']})] - assert mock_call_hooks.mock_calls == [call(hooks, 'AFTER_SCHEDULE_REPO', - {'roles': [Role(ROLES_FOR_DISPLAY[1])]})] + assert mock_set_role_data.mock_calls == [ + call( + None, + "AROAABCDEFGHIJKLMNOPB", + {"RepoScheduled": 86401, "ScheduledPerms": ["ec2"]}, + ) + ] + assert mock_call_hooks.mock_calls == [ + call(hooks, "AFTER_SCHEDULE_REPO", {"roles": [Role(ROLES_FOR_DISPLAY[1])]}) + ] - @patch('repokid.hooks.call_hooks') - @patch('repokid.cli.repokid_cli.get_role_data') - @patch('repokid.cli.repokid_cli.role_ids_for_account') - @patch('repokid.cli.repokid_cli.repo_role') - @patch('time.time') - def test_repo_all_roles(self, mock_time, mock_repo_role, mock_role_ids_for_account, mock_get_role_data, - mock_call_hooks): + @patch("repokid.hooks.call_hooks") + @patch("repokid.cli.repokid_cli.get_role_data") + @patch("repokid.cli.repokid_cli.role_ids_for_account") + @patch("repokid.cli.repokid_cli.repo_role") + @patch("time.time") + def test_repo_all_roles( + self, + mock_time, + mock_repo_role, + mock_role_ids_for_account, + mock_get_role_data, + mock_call_hooks, + ): hooks = {} - mock_role_ids_for_account.return_value = ['AROAABCDEFGHIJKLMNOPA', 'AROAABCDEFGHIJKLMNOPB', - 'AROAABCDEFGHIJKLMNOPC'] - roles = [{'RoleId': 'AROAABCDEFGHIJKLMNOPA', 'Active': True, 'RoleName': 'ROLE_A', 'RepoScheduled': 100}, - {'RoleId': 'AROAABCDEFGHIJKLMNOPB', 'Active': True, 'RoleName': 'ROLE_B', 'RepoScheduled': 0}, - {'RoleId': 'AROAABCDEFGHIJKLMNOPC', 'Active': True, 'RoleName': 'ROLE_C', 'RepoScheduled': 5}] + mock_role_ids_for_account.return_value = [ + "AROAABCDEFGHIJKLMNOPA", + "AROAABCDEFGHIJKLMNOPB", + "AROAABCDEFGHIJKLMNOPC", + ] + roles = [ + { + "RoleId": "AROAABCDEFGHIJKLMNOPA", + "Active": True, + "RoleName": "ROLE_A", + "RepoScheduled": 100, + }, + { + "RoleId": "AROAABCDEFGHIJKLMNOPB", + "Active": True, + "RoleName": "ROLE_B", + "RepoScheduled": 0, + }, + { + "RoleId": "AROAABCDEFGHIJKLMNOPC", + "Active": True, + "RoleName": "ROLE_C", + "RepoScheduled": 5, + }, + ] # time is past ROLE_C but before ROLE_A mock_time.return_value = 10 # return both roles each time - mock_get_role_data.side_effect = [roles[0], roles[1], roles[2], roles[0], roles[1], roles[2]] + mock_get_role_data.side_effect = [ + roles[0], + roles[1], + roles[2], + roles[0], + roles[1], + roles[2], + ] mock_repo_role.return_value = None # repo all roles in the account, should call repo with all roles @@ -402,66 +515,141 @@ def test_repo_all_roles(self, mock_time, mock_repo_role, mock_role_ids_for_accou # repo only scheduled, should only call repo role with role C repokid.cli.repokid_cli.repo_all_roles(None, None, None, hooks, scheduled=True) - assert mock_repo_role.mock_calls == [call(None, 'ROLE_A', None, None, hooks, commit=False, scheduled=False), - call(None, 'ROLE_B', None, None, hooks, commit=False, scheduled=False), - call(None, 'ROLE_C', None, None, hooks, commit=False, scheduled=False), - call(None, 'ROLE_C', None, None, hooks, commit=False, scheduled=True)] + assert mock_repo_role.mock_calls == [ + call(None, "ROLE_A", None, None, hooks, commit=False, scheduled=False), + call(None, "ROLE_B", None, None, hooks, commit=False, scheduled=False), + call(None, "ROLE_C", None, None, hooks, commit=False, scheduled=False), + call(None, "ROLE_C", None, None, hooks, commit=False, scheduled=True), + ] roles_items = [Role(roles[0]), Role(roles[1]), Role(roles[2])] assert mock_call_hooks.mock_calls == [ - call(hooks, 'BEFORE_REPO_ROLES', {'account_number': None, 'roles': roles_items}), - call(hooks, 'BEFORE_REPO_ROLES', {'account_number': None, 'roles': [roles_items[2]]}), + call( + hooks, + "BEFORE_REPO_ROLES", + {"account_number": None, "roles": roles_items}, + ), + call( + hooks, + "BEFORE_REPO_ROLES", + {"account_number": None, "roles": [roles_items[2]]}, + ), ] - @patch('repokid.cli.repokid_cli.find_role_in_cache') - @patch('repokid.cli.repokid_cli.get_role_data') - @patch('repokid.cli.repokid_cli.role_ids_for_account') - @patch('repokid.cli.repokid_cli.set_role_data') - def test_cancel_scheduled_repo(self, mock_set_role_data, mock_role_ids_for_account, mock_get_role_data, - mock_find_role_in_cache): - - mock_role_ids_for_account.return_value = ['AROAABCDEFGHIJKLMNOPA', 'AROAABCDEFGHIJKLMNOPB'] - roles = [{'RoleId': 'AROAABCDEFGHIJKLMNOPA', 'Active': True, 'RoleName': 'ROLE_A', 'RepoScheduled': 100}, - {'RoleId': 'AROAABCDEFGHIJKLMNOPB', 'Active': True, 'RoleName': 'ROLE_B', 'RepoScheduled': 0}, - {'RoleId': 'AROAABCDEFGHIJKLMNOPC', 'Active': True, 'RoleName': 'ROLE_C', 'RepoScheduled': 5}] + @patch("repokid.cli.repokid_cli.find_role_in_cache") + @patch("repokid.cli.repokid_cli.get_role_data") + @patch("repokid.cli.repokid_cli.role_ids_for_account") + @patch("repokid.cli.repokid_cli.set_role_data") + def test_cancel_scheduled_repo( + self, + mock_set_role_data, + mock_role_ids_for_account, + mock_get_role_data, + mock_find_role_in_cache, + ): + + mock_role_ids_for_account.return_value = [ + "AROAABCDEFGHIJKLMNOPA", + "AROAABCDEFGHIJKLMNOPB", + ] + roles = [ + { + "RoleId": "AROAABCDEFGHIJKLMNOPA", + "Active": True, + "RoleName": "ROLE_A", + "RepoScheduled": 100, + }, + { + "RoleId": "AROAABCDEFGHIJKLMNOPB", + "Active": True, + "RoleName": "ROLE_B", + "RepoScheduled": 0, + }, + { + "RoleId": "AROAABCDEFGHIJKLMNOPC", + "Active": True, + "RoleName": "ROLE_C", + "RepoScheduled": 5, + }, + ] mock_get_role_data.side_effect = [roles[0], roles[2], roles[0]] # first check all - repokid.cli.repokid_cli.cancel_scheduled_repo(None, None, role_name=None, is_all=True) + repokid.cli.repokid_cli.cancel_scheduled_repo( + None, None, role_name=None, is_all=True + ) # ensure all are cancelled - mock_find_role_in_cache.return_value = ['AROAABCDEFGHIJKLMNOPA'] - - repokid.cli.repokid_cli.cancel_scheduled_repo(None, None, role_name='ROLE_A', is_all=False) - - assert mock_set_role_data.mock_calls == [call(None, 'AROAABCDEFGHIJKLMNOPA', - {'RepoScheduled': 0, 'ScheduledPerms': []}), - call(None, 'AROAABCDEFGHIJKLMNOPC', - {'RepoScheduled': 0, 'ScheduledPerms': []}), - call(None, 'AROAABCDEFGHIJKLMNOPA', - {'RepoScheduled': 0, 'ScheduledPerms': []})] + mock_find_role_in_cache.return_value = ["AROAABCDEFGHIJKLMNOPA"] + + repokid.cli.repokid_cli.cancel_scheduled_repo( + None, None, role_name="ROLE_A", is_all=False + ) + + assert mock_set_role_data.mock_calls == [ + call( + None, + "AROAABCDEFGHIJKLMNOPA", + {"RepoScheduled": 0, "ScheduledPerms": []}, + ), + call( + None, + "AROAABCDEFGHIJKLMNOPC", + {"RepoScheduled": 0, "ScheduledPerms": []}, + ), + call( + None, + "AROAABCDEFGHIJKLMNOPA", + {"RepoScheduled": 0, "ScheduledPerms": []}, + ), + ] def test_generate_default_config(self): generated_config = repokid.cli.repokid_cli._generate_default_config() - required_config_fields = ['filter_config', 'active_filters', 'aardvark_api_location', 'connection_iam', - 'dynamo_db', 'logging', 'repo_requirements'] + required_config_fields = [ + "filter_config", + "active_filters", + "aardvark_api_location", + "connection_iam", + "dynamo_db", + "logging", + "repo_requirements", + ] - required_filter_configs = ['AgeFilter', 'BlocklistFilter'] + required_filter_configs = ["AgeFilter", "BlocklistFilter"] - required_dynamo_config = ['account_number', 'endpoint', 'region', 'session_name'] + required_dynamo_config = [ + "account_number", + "endpoint", + "region", + "session_name", + ] - required_iam_config = ['assume_role', 'session_name', 'region'] + required_iam_config = ["assume_role", "session_name", "region"] - required_repo_requirements = ['oldest_aa_data_days', 'exclude_new_permissions_for_days'] + required_repo_requirements = [ + "oldest_aa_data_days", + "exclude_new_permissions_for_days", + ] assert all(field in generated_config for field in required_config_fields) - assert all(field in generated_config['filter_config'] for field in required_filter_configs) - assert all(field in generated_config['dynamo_db'] for field in required_dynamo_config) - assert all(field in generated_config['connection_iam'] for field in required_iam_config) - assert all(field in generated_config['repo_requirements'] for field in required_repo_requirements) - assert 'warnings' in generated_config + assert all( + field in generated_config["filter_config"] + for field in required_filter_configs + ) + assert all( + field in generated_config["dynamo_db"] for field in required_dynamo_config + ) + assert all( + field in generated_config["connection_iam"] for field in required_iam_config + ) + assert all( + field in generated_config["repo_requirements"] + for field in required_repo_requirements + ) + assert "warnings" in generated_config def test_inline_policies_size_exceeds_maximum(self): cli = repokid.cli.repokid_cli @@ -471,7 +659,9 @@ def test_inline_policies_size_exceeds_maximum(self): backup_size = cli.MAX_AWS_POLICY_SIZE cli.MAX_AWS_POLICY_SIZE = 10 - assert cli._inline_policies_size_exceeds_maximum(ROLE_POLICIES['all_services_used']) + assert cli._inline_policies_size_exceeds_maximum( + ROLE_POLICIES["all_services_used"] + ) cli.MAX_AWS_POLICY_SIZE = backup_size def test_logprint_deleted_and_repoed_policies(self): @@ -491,31 +681,36 @@ def emit(self, record): def reset(self): self.messages = { - 'debug': [], - 'info': [], - 'warning': [], - 'error': [], - 'critical': [], + "debug": [], + "info": [], + "warning": [], + "error": [], + "critical": [], } - cli.LOGGER = logging.getLogger('test') + cli.LOGGER = logging.getLogger("test") mock_logger = MockLoggingHandler() cli.LOGGER.addHandler(mock_logger) - policy_names = ['policy1', 'policy2'] + policy_names = ["policy1", "policy2"] repoed_policies = [ROLE_POLICIES] - cli._logprint_deleted_and_repoed_policies(policy_names, repoed_policies, 'MyRoleName', '123456789012') - assert len(mock_logger.messages['info']) == 3 - assert 'policy1' in mock_logger.messages['info'][0] - assert 'policy2' in mock_logger.messages['info'][1] - assert 'all_services_used' in mock_logger.messages['info'][2] + cli._logprint_deleted_and_repoed_policies( + policy_names, repoed_policies, "MyRoleName", "123456789012" + ) + assert len(mock_logger.messages["info"]) == 3 + assert "policy1" in mock_logger.messages["info"][0] + assert "policy2" in mock_logger.messages["info"][1] + assert "all_services_used" in mock_logger.messages["info"][2] def test_delete_policy(self): cli = repokid.cli.repokid_cli def mock_delete_role_policy(RoleName, PolicyName, **conn): import botocore - raise botocore.exceptions.ClientError(dict(Error=dict(Code='TESTING')), 'TESTING') + + raise botocore.exceptions.ClientError( + dict(Error=dict(Code="TESTING")), "TESTING" + ) class MockRole: role_name = "role_name" @@ -523,15 +718,18 @@ class MockRole: cli.delete_role_policy = mock_delete_role_policy mock_role = MockRole() - error = cli._delete_policy('PolicyName', mock_role, '123456789012', dict()) - assert 'Error deleting policy:' in error + error = cli._delete_policy("PolicyName", mock_role, "123456789012", dict()) + assert "Error deleting policy:" in error def test_replace_policies(self): cli = repokid.cli.repokid_cli def mock_put_role_policy(RoleName, PolicyName, PolicyDocument, **conn): import botocore - raise botocore.exceptions.ClientError(dict(Error=dict(Code='TESTING')), 'TESTING') + + raise botocore.exceptions.ClientError( + dict(Error=dict(Code="TESTING")), "TESTING" + ) class MockRole: role_name = "role_name" @@ -539,23 +737,33 @@ class MockRole: cli.put_role_policy = mock_put_role_policy mock_role = MockRole() - error = cli._replace_policies(ROLE_POLICIES, mock_role, '123456789012', {}) - assert 'Exception calling PutRolePolicy' in error - - @patch('repokid.cli.repokid_cli._delete_policy', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli._replace_policies', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli.get_role_inline_policies', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli.roledata.add_new_policy_version', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli.set_role_data', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli._update_repoed_description', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli._update_role_data', MagicMock(return_value=None)) + error = cli._replace_policies(ROLE_POLICIES, mock_role, "123456789012", {}) + assert "Exception calling PutRolePolicy" in error + + @patch("repokid.cli.repokid_cli._delete_policy", MagicMock(return_value=None)) + @patch("repokid.cli.repokid_cli._replace_policies", MagicMock(return_value=None)) + @patch( + "repokid.cli.repokid_cli.get_role_inline_policies", MagicMock(return_value=None) + ) + @patch( + "repokid.cli.repokid_cli.roledata.add_new_policy_version", + MagicMock(return_value=None), + ) + @patch("repokid.cli.repokid_cli.set_role_data", MagicMock(return_value=None)) + @patch( + "repokid.cli.repokid_cli._update_repoed_description", + MagicMock(return_value=None), + ) + @patch("repokid.cli.repokid_cli._update_role_data", MagicMock(return_value=None)) def test_remove_permissions_from_role(self): cli = repokid.cli.repokid_cli class MockRole: role_name = "role_name" - role_id = '12345-roleid' - policies = [dict(Policy=policy) for _, policy in ROLE_POLICIES.items()] + role_id = "12345-roleid" + policies = [ + dict(Policy=policy) for _, policy in list(ROLE_POLICIES.items()) + ] def as_dict(self): return dict(role_name=self.role_name, policies=self.policies) @@ -563,45 +771,60 @@ def as_dict(self): mock_role = MockRole() cli._remove_permissions_from_role( - '123456789012', - ['s3:putobjectacl'], + "123456789012", + ["s3:putobjectacl"], mock_role, - '12345-roleid', + "12345-roleid", None, None, None, - commit=False) + commit=False, + ) cli._remove_permissions_from_role( - '123456789012', - ['s3:putobjectacl'], + "123456789012", + ["s3:putobjectacl"], mock_role, - '12345-roleid', + "12345-roleid", None, {"connection_iam": dict()}, None, - commit=True) - - @patch('repokid.cli.repokid_cli.find_role_in_cache', MagicMock(return_value='12345-roleid')) - @patch('repokid.cli.repokid_cli.get_role_data', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli.Role', MagicMock(return_value='IAMROLE')) - @patch('repokid.cli.repokid_cli._remove_permissions_from_role', MagicMock(return_value=None)) - @patch('repokid.cli.repokid_cli.repokid.hooks') + commit=True, + ) + + @patch( + "repokid.cli.repokid_cli.find_role_in_cache", + MagicMock(return_value="12345-roleid"), + ) + @patch("repokid.cli.repokid_cli.get_role_data", MagicMock(return_value=None)) + @patch("repokid.cli.repokid_cli.Role", MagicMock(return_value="IAMROLE")) + @patch( + "repokid.cli.repokid_cli._remove_permissions_from_role", + MagicMock(return_value=None), + ) + @patch("repokid.cli.repokid_cli.repokid.hooks") def test_remove_permissions_from_roles(self, mock_hooks): import json + cli = repokid.cli.repokid_cli - arns = [role['Arn'] for role in ROLES] + arns = [role["Arn"] for role in ROLES] arns = json.dumps(arns) class Hooks: def call_hooks(hooks, tag, role_dict): - assert tag == 'AFTER_REPO' + assert tag == "AFTER_REPO" mock_hooks = Hooks() - with patch("__builtin__.open", mock_open(read_data=arns)) as mock_file: + with patch("builtins.open", mock_open(read_data=arns)) as mock_file: assert open("somefile.json").read() == arns mock_file.assert_called_with("somefile.json") - cli.remove_permissions_from_roles(['s3:putobjectacl'], 'somefile.json', None, None, mock_hooks, - commit=False) + cli.remove_permissions_from_roles( + ["s3:putobjectacl"], + "somefile.json", + None, + None, + mock_hooks, + commit=False, + ) diff --git a/repokid/tests/test_roledata.py b/repokid/tests/test_roledata.py index 789018939..a411fd468 100644 --- a/repokid/tests/test_roledata.py +++ b/repokid/tests/test_roledata.py @@ -22,44 +22,80 @@ class TestRoledata(object): - @patch('repokid.utils.roledata.expand_policy') - @patch('repokid.utils.roledata.get_actions_from_statement') - @patch('repokid.utils.roledata.all_permissions') - def test_get_role_permissions(self, mock_all_permissions, mock_get_actions_from_statement, mock_expand_policy): + @patch("repokid.utils.roledata.expand_policy") + @patch("repokid.utils.roledata.get_actions_from_statement") + @patch("repokid.utils.roledata.all_permissions") + def test_get_role_permissions( + self, mock_all_permissions, mock_get_actions_from_statement, mock_expand_policy + ): test_role = Role(ROLES[0]) - all_permissions = ['ec2:associateaddress', 'ec2:attachvolume', 'ec2:createsnapshot', 's3:createbucket', - 's3:getobject'] + all_permissions = [ + "ec2:associateaddress", + "ec2:attachvolume", + "ec2:createsnapshot", + "s3:createbucket", + "s3:getobject", + ] # empty policy to make sure we get the latest - test_role.policies = [{'Policy': ROLE_POLICIES['all_services_used']}, {'Policy': ROLE_POLICIES['unused_ec2']}] + test_role.policies = [ + {"Policy": ROLE_POLICIES["all_services_used"]}, + {"Policy": ROLE_POLICIES["unused_ec2"]}, + ] mock_all_permissions.return_value = all_permissions - mock_get_actions_from_statement.return_value = ROLE_POLICIES['unused_ec2']['ec2_perms'] - mock_expand_policy.return_value = ROLE_POLICIES['unused_ec2']['ec2_perms'] + mock_get_actions_from_statement.return_value = ROLE_POLICIES["unused_ec2"][ + "ec2_perms" + ] + mock_expand_policy.return_value = ROLE_POLICIES["unused_ec2"]["ec2_perms"] - total_permissions, eligible_permissions = repokid.utils.roledata._get_role_permissions(test_role) - assert total_permissions == set(ROLE_POLICIES['unused_ec2']['ec2_perms']) - assert eligible_permissions == set(ROLE_POLICIES['unused_ec2']['ec2_perms']) + total_permissions, eligible_permissions = repokid.utils.roledata._get_role_permissions( + test_role + ) + assert total_permissions == set(ROLE_POLICIES["unused_ec2"]["ec2_perms"]) + assert eligible_permissions == set(ROLE_POLICIES["unused_ec2"]["ec2_perms"]) - @patch('repokid.hooks.call_hooks') + @patch("repokid.hooks.call_hooks") def test_get_repoable_permissions(self, mock_call_hooks): minimum_age = 1 - repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES = ['service_2'] - repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS = ['service_1:action_3', 'service_1:action_4'] + repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES = ["service_2"] + repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS = [ + "service_1:action_3", + "service_1:action_4", + ] hooks = {} - permissions = ['service_1:action_1', 'service_1:action_2', 'service_1:action_3', 'service_1:action_4', - 'service_2:action_1', 'service_3:action_1', 'service_3:action_2', 'service_4:action_1', - 'service_4:action_2'] + permissions = [ + "service_1:action_1", + "service_1:action_2", + "service_1:action_3", + "service_1:action_4", + "service_2:action_1", + "service_3:action_1", + "service_3:action_2", + "service_4:action_1", + "service_4:action_2", + ] # service_1 and service_2 both used more than a day ago, which is outside of our test filter for age - aa_data = [{'serviceNamespace': 'service_1', 'lastAuthenticated': (time.time() - 90000) * 1000}, - {'serviceNamespace': 'service_2', 'lastAuthenticated': (time.time() - 90000) * 1000}, - {'serviceNamespace': 'service_3', 'lastAuthenticated': time.time() * 1000}] + aa_data = [ + { + "serviceNamespace": "service_1", + "lastAuthenticated": (time.time() - 90000) * 1000, + }, + { + "serviceNamespace": "service_2", + "lastAuthenticated": (time.time() - 90000) * 1000, + }, + {"serviceNamespace": "service_3", "lastAuthenticated": time.time() * 1000}, + ] - no_repo_permissions = {'service_4:action_1': time.time() - 1, 'service_4:action_2': time.time() + 1000} + no_repo_permissions = { + "service_4:action_1": time.time() - 1, + "service_4:action_2": time.time() + 1000, + } true_repoable_decision = repokid.utils.roledata.RepoablePermissionDecision() true_repoable_decision.repoable = True @@ -67,52 +103,82 @@ def test_get_repoable_permissions(self, mock_call_hooks): false_repoable_decision = repokid.utils.roledata.RepoablePermissionDecision() false_repoable_decision.repoable = False - mock_call_hooks.return_value = {'potentially_repoable_permissions': - {'service_1:action_1': true_repoable_decision, - 'service_1:action_2': true_repoable_decision, - 'service_4:action_1': true_repoable_decision, - 'service_1:action_3': false_repoable_decision, - 'service_1:action_4': false_repoable_decision, - 'service_2:action_1': false_repoable_decision, - 'service_3:action_1': false_repoable_decision, - 'service_3:action_2': false_repoable_decision, - 'service_4:action_2': false_repoable_decision - }} - - repoable_permissions = repokid.utils.roledata._get_repoable_permissions(None, 'test_name', permissions, aa_data, - no_repo_permissions, minimum_age, - hooks) + mock_call_hooks.return_value = { + "potentially_repoable_permissions": { + "service_1:action_1": true_repoable_decision, + "service_1:action_2": true_repoable_decision, + "service_4:action_1": true_repoable_decision, + "service_1:action_3": false_repoable_decision, + "service_1:action_4": false_repoable_decision, + "service_2:action_1": false_repoable_decision, + "service_3:action_1": false_repoable_decision, + "service_3:action_2": false_repoable_decision, + "service_4:action_2": false_repoable_decision, + } + } + + repoable_permissions = repokid.utils.roledata._get_repoable_permissions( + None, + "test_name", + permissions, + aa_data, + no_repo_permissions, + minimum_age, + hooks, + ) # service_1:action_3 and action_4 are unsupported actions, service_2 is an unsupported service, service_3 # was used too recently, service_4 action 2 is in no_repo_permissions and not expired - assert repoable_permissions == set(['service_1:action_1', 'service_1:action_2', 'service_4:action_1']) + assert repoable_permissions == set( + ["service_1:action_1", "service_1:action_2", "service_4:action_1"] + ) - @patch('repokid.hooks.call_hooks') + @patch("repokid.hooks.call_hooks") def test_get_repoable_permissions_batch(self, mock_call_hooks): roles = [Role(ROLES[0]), Role(ROLES[4]), Role(ROLES[5])] roles[0].aa_data = AARDVARK_DATA[roles[0].arn] - roles[1].no_repo_permissions = {'ec2:AllocateHosts': time.time() - 1, - 'ec2:AssociateAddress': time.time() + 1000} + roles[1].no_repo_permissions = { + "ec2:AllocateHosts": time.time() - 1, + "ec2:AssociateAddress": time.time() + 1000, + } roles[1].aa_data = AARDVARK_DATA[roles[1].arn] roles[2].aa_data = AARDVARK_DATA[roles[2].arn] minimum_age = 1 - repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES = ['unsupported_service'] - repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS = ['supported_service:unsupported_action'] + repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES = [ + "unsupported_service" + ] + repokid.utils.roledata.IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS = [ + "supported_service:unsupported_action" + ] hooks = {} permissions_dict = { - "arn:aws:iam::123456789012:role/all_services_used": ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy', - 'ec2:AllocateHosts', 'ec2:AssociateAddress'], - "arn:aws:iam::123456789012:role/unused_ec2": ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy', - 'ec2:AllocateHosts', 'ec2:AssociateAddress', - 'unsupported_service:action', - 'supported_service:unsuported_action'], - "arn:aws:iam::123456789012:role/additional_unused_ec2": ['iam:AddRoleToInstanceProfile', - 'iam:AttachRolePolicy', 'ec2:AllocateHosts', - 'ec2:AssociateAddress'], - "arn:aws:iam::123456789012:role/unused_iam": ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'] + "arn:aws:iam::123456789012:role/all_services_used": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + ], + "arn:aws:iam::123456789012:role/unused_ec2": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + "unsupported_service:action", + "supported_service:unsuported_action", + ], + "arn:aws:iam::123456789012:role/additional_unused_ec2": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + ], + "arn:aws:iam::123456789012:role/unused_iam": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + ], } true_repoable_decision = repokid.utils.roledata.RepoablePermissionDecision() @@ -124,207 +190,311 @@ def test_get_repoable_permissions_batch(self, mock_call_hooks): # The new hook should return a dict mapping role arn's to a json of their repoable permissions with decisions mock_call_hooks.return_value = { "arn:aws:iam::123456789012:role/all_services_used": { - 'potentially_repoable_permissions': { - 'iam:AddRoleToInstanceProfile': false_repoable_decision, - 'iam:AttachRolePolicy': false_repoable_decision, - 'ec2:AllocateHosts': false_repoable_decision, - 'ec2:AssociateAddress': false_repoable_decision + "potentially_repoable_permissions": { + "iam:AddRoleToInstanceProfile": false_repoable_decision, + "iam:AttachRolePolicy": false_repoable_decision, + "ec2:AllocateHosts": false_repoable_decision, + "ec2:AssociateAddress": false_repoable_decision, } }, "arn:aws:iam::123456789012:role/unused_ec2": { - 'potentially_repoable_permissions': { - 'iam:AddRoleToInstanceProfile': false_repoable_decision, - 'iam:AttachRolePolicy': false_repoable_decision, - 'ec2:AllocateHosts': true_repoable_decision, - 'ec2:AssociateAddress': true_repoable_decision, + "potentially_repoable_permissions": { + "iam:AddRoleToInstanceProfile": false_repoable_decision, + "iam:AttachRolePolicy": false_repoable_decision, + "ec2:AllocateHosts": true_repoable_decision, + "ec2:AssociateAddress": true_repoable_decision, } }, "arn:aws:iam::123456789012:role/additional_unused_ec2": { - 'potentially_repoable_permissions': { - 'iam:AddRoleToInstanceProfile': false_repoable_decision, - 'iam:AttachRolePolicy': false_repoable_decision, - 'ec2:AllocateHosts': false_repoable_decision, - 'ec2:AssociateAddress': false_repoable_decision, - 'unsupported_service:action': false_repoable_decision, - 'supported_service:unsupported_action': false_repoable_decision + "potentially_repoable_permissions": { + "iam:AddRoleToInstanceProfile": false_repoable_decision, + "iam:AttachRolePolicy": false_repoable_decision, + "ec2:AllocateHosts": false_repoable_decision, + "ec2:AssociateAddress": false_repoable_decision, + "unsupported_service:action": false_repoable_decision, + "supported_service:unsupported_action": false_repoable_decision, } }, "arn:aws:iam::123456789012:role/unused_iam": { - 'potentially_repoable_permissions': { - 'iam:AddRoleToInstanceProfile': true_repoable_decision, - 'iam:AttachRolePolicy': true_repoable_decision, - 'ec2:AllocateHosts': false_repoable_decision, - 'ec2:AssociateAddress': false_repoable_decision + "potentially_repoable_permissions": { + "iam:AddRoleToInstanceProfile": true_repoable_decision, + "iam:AttachRolePolicy": true_repoable_decision, + "ec2:AllocateHosts": false_repoable_decision, + "ec2:AssociateAddress": false_repoable_decision, } }, } repoable_permissions_dict = { "arn:aws:iam::123456789012:role/all_services_used": set(), - "arn:aws:iam::123456789012:role/unused_ec2": set(['ec2:AllocateHosts', 'ec2:AssociateAddress']), + "arn:aws:iam::123456789012:role/unused_ec2": set( + ["ec2:AllocateHosts", "ec2:AssociateAddress"] + ), "arn:aws:iam::123456789012:role/additional_unused_ec2": set(), - "arn:aws:iam::123456789012:role/unused_iam": set(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), + "arn:aws:iam::123456789012:role/unused_iam": set( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"] + ), } - assert repoable_permissions_dict == repokid.utils.roledata._get_repoable_permissions_batch( - roles, permissions_dict, minimum_age, hooks, 2) - assert repoable_permissions_dict == repokid.utils.roledata._get_repoable_permissions_batch( - roles, permissions_dict, minimum_age, hooks, 4) + assert ( + repoable_permissions_dict + == repokid.utils.roledata._get_repoable_permissions_batch( + roles, permissions_dict, minimum_age, hooks, 2 + ) + ) + assert ( + repoable_permissions_dict + == repokid.utils.roledata._get_repoable_permissions_batch( + roles, permissions_dict, minimum_age, hooks, 4 + ) + ) - @patch('repokid.utils.roledata._get_role_permissions') - @patch('repokid.utils.roledata._get_repoable_permissions') - @patch('repokid.hooks.call_hooks') - def test_calculate_repo_scores(self, mock_call_hooks, mock_get_repoable_permissions, mock_get_role_permissions): + @patch("repokid.utils.roledata._get_role_permissions") + @patch("repokid.utils.roledata._get_repoable_permissions") + @patch("repokid.hooks.call_hooks") + def test_calculate_repo_scores( + self, mock_call_hooks, mock_get_repoable_permissions, mock_get_role_permissions + ): roles = [Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2])] roles[0].disqualified_by = [] - roles[0].aa_data = 'some_aa_data' + roles[0].aa_data = "some_aa_data" # disqualified by a filter - roles[1].policies = [{'Policy': ROLE_POLICIES['unused_ec2']}] - roles[1].disqualified_by = ['some_filter'] - roles[1].aa_data = 'some_aa_data' + roles[1].policies = [{"Policy": ROLE_POLICIES["unused_ec2"]}] + roles[1].disqualified_by = ["some_filter"] + roles[1].aa_data = "some_aa_data" # no AA data - roles[2].policies = [{'Policy': ROLE_POLICIES['all_services_used']}] + roles[2].policies = [{"Policy": ROLE_POLICIES["all_services_used"]}] roles[2].disqualified_by = [] roles[2].aa_data = None hooks = {} - mock_get_role_permissions.side_effect = [(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy', - 'ec2:AllocateHosts', 'ec2:AssociateAddress'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - (['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - (['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'])] - - mock_call_hooks.return_value = set(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']) - mock_get_repoable_permissions.side_effect = [set(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'])] + mock_get_role_permissions.side_effect = [ + ( + [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + ], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ] + + mock_call_hooks.return_value = set( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"] + ) + mock_get_repoable_permissions.side_effect = [ + set(["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"]) + ] minimum_age = 90 repokid.utils.roledata._calculate_repo_scores(roles, minimum_age, hooks) assert roles[0].repoable_permissions == 2 - assert roles[0].repoable_services == ['iam'] + assert roles[0].repoable_services == ["iam"] assert roles[1].repoable_permissions == 0 assert roles[1].repoable_services == [] assert roles[2].repoable_permissions == 0 assert roles[2].repoable_services == [] - @patch('repokid.utils.roledata._get_role_permissions') - @patch('repokid.utils.roledata._get_repoable_permissions_batch') - @patch('repokid.hooks.call_hooks') - def test_calculate_repo_scores_batch(self, mock_call_hooks, mock_get_repoable_permissions_batch, - mock_get_role_permissions): - roles = [Role(ROLES[0]), Role(ROLES[1]), Role(ROLES[2]), Role(ROLES[4]), Role(ROLES[5])] + @patch("repokid.utils.roledata._get_role_permissions") + @patch("repokid.utils.roledata._get_repoable_permissions_batch") + @patch("repokid.hooks.call_hooks") + def test_calculate_repo_scores_batch( + self, + mock_call_hooks, + mock_get_repoable_permissions_batch, + mock_get_role_permissions, + ): + roles = [ + Role(ROLES[0]), + Role(ROLES[1]), + Role(ROLES[2]), + Role(ROLES[4]), + Role(ROLES[5]), + ] roles[0].disqualified_by = [] - roles[0].aa_data = 'some_aa_data' + roles[0].aa_data = "some_aa_data" # disqualified by a filter - roles[1].policies = [{'Policy': ROLE_POLICIES['unused_ec2']}] - roles[1].disqualified_by = ['some_filter'] - roles[1].aa_data = 'some_aa_data' + roles[1].policies = [{"Policy": ROLE_POLICIES["unused_ec2"]}] + roles[1].disqualified_by = ["some_filter"] + roles[1].aa_data = "some_aa_data" # no AA data - roles[2].policies = [{'Policy': ROLE_POLICIES['all_services_used']}] + roles[2].policies = [{"Policy": ROLE_POLICIES["all_services_used"]}] roles[2].disqualified_by = [] roles[2].aa_data = None roles[3].disqualified_by = [] - roles[3].aa_data = 'some_aa_data' + roles[3].aa_data = "some_aa_data" roles[4].disqualified_by = [] - roles[4].aa_data = 'some_aa_data' + roles[4].aa_data = "some_aa_data" hooks = {} - mock_get_role_permissions.side_effect = [(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy', - 'ec2:AllocateHosts', 'ec2:AssociateAddress'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - (['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - (['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - (['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy', - 'ec2:AllocateHosts', 'ec2:AssociateAddress'], - ['ec2:AllocateHosts', 'ec2:AssociateAddress']), - (['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy', - 'ec2:AllocateHosts', 'ec2:AssociateAddress'], - ['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - ] - - mock_call_hooks.return_value = set(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']) + mock_get_role_permissions.side_effect = [ + ( + [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + ], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ( + [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + ], + ["ec2:AllocateHosts", "ec2:AssociateAddress"], + ), + ( + [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "ec2:AllocateHosts", + "ec2:AssociateAddress", + ], + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"], + ), + ] + + mock_call_hooks.return_value = set( + ["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"] + ) batch_perms_dict = { - roles[0].arn: set(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), - roles[3].arn: set(['ec2:AllocateHosts', 'ec2:AssociateAddress']), - roles[4].arn: set(['iam:AddRoleToInstanceProfile', 'iam:AttachRolePolicy']), + roles[0].arn: set(["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"]), + roles[3].arn: set(["ec2:AllocateHosts", "ec2:AssociateAddress"]), + roles[4].arn: set(["iam:AddRoleToInstanceProfile", "iam:AttachRolePolicy"]), } mock_get_repoable_permissions_batch.side_effect = [batch_perms_dict] minimum_age = 90 - repokid.utils.roledata._calculate_repo_scores(roles, minimum_age, hooks, batch=True, batch_size=100) + repokid.utils.roledata._calculate_repo_scores( + roles, minimum_age, hooks, batch=True, batch_size=100 + ) assert roles[0].repoable_permissions == 2 - assert roles[0].repoable_services == ['iam'] + assert roles[0].repoable_services == ["iam"] assert roles[1].repoable_permissions == 0 assert roles[1].repoable_services == [] assert roles[2].repoable_permissions == 0 assert roles[2].repoable_services == [] assert roles[3].repoable_permissions == 2 - assert roles[3].repoable_services == ['ec2'] + assert roles[3].repoable_services == ["ec2"] assert roles[4].repoable_permissions == 2 - assert roles[4].repoable_services == ['iam'] + assert roles[4].repoable_services == ["iam"] def test_get_repoed_policy(self): - policies = ROLE_POLICIES['all_services_used'] - repoable_permissions = set(['iam:addroletoinstanceprofile', 'iam:attachrolepolicy', 's3:createbucket']) + policies = ROLE_POLICIES["all_services_used"] + repoable_permissions = set( + ["iam:addroletoinstanceprofile", "iam:attachrolepolicy", "s3:createbucket"] + ) - rewritten_policies, empty_policies = repokid.utils.roledata._get_repoed_policy(policies, repoable_permissions) + rewritten_policies, empty_policies = repokid.utils.roledata._get_repoed_policy( + policies, repoable_permissions + ) - assert rewritten_policies == {'s3_perms': {'Version': '2012-10-17', - 'Statement': [{'Action': ['s3:deletebucket'], - 'Resource': ['*'], - 'Effect': 'Allow'}]}} - assert empty_policies == ['iam_perms'] + assert rewritten_policies == { + "s3_perms": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": ["s3:deletebucket"], + "Resource": ["*"], + "Effect": "Allow", + } + ], + } + } + assert empty_policies == ["iam_perms"] def test_find_newly_added_permissions(self): - old_policy = ROLE_POLICIES['all_services_used'] - new_policy = ROLE_POLICIES['unused_ec2'] + old_policy = ROLE_POLICIES["all_services_used"] + new_policy = ROLE_POLICIES["unused_ec2"] - new_perms = repokid.utils.roledata.find_newly_added_permissions(old_policy, new_policy) - assert new_perms == set(['ec2:allocatehosts', 'ec2:associateaddress']) + new_perms = repokid.utils.roledata.find_newly_added_permissions( + old_policy, new_policy + ) + assert new_perms == set(["ec2:allocatehosts", "ec2:associateaddress"]) def test_convert_repoable_perms_to_perms_and_services(self): - all_perms = ['a:j', 'a:k', 'b:l', 'c:m', 'c:n'] - repoable_perms = ['b:l', 'c:m'] - expected_repoed_services = ['b'] - expected_repoed_permissions = ['c:m'] - assert (repokid.utils.roledata._convert_repoable_perms_to_perms_and_services(all_perms, repoable_perms) == - (expected_repoed_permissions, expected_repoed_services)) + all_perms = ["a:j", "a:k", "b:l", "c:m", "c:n"] + repoable_perms = ["b:l", "c:m"] + expected_repoed_services = ["b"] + expected_repoed_permissions = ["c:m"] + assert repokid.utils.roledata._convert_repoable_perms_to_perms_and_services( + all_perms, repoable_perms + ) == (expected_repoed_permissions, expected_repoed_services) def test_convert_repoed_service_to_sorted_perms_and_services(self): - repoed_services = ['route53', 'ec2', 's3:abc', 'dynamodb:def', 'ses:ghi', 'ses:jkl'] - expected_services = ['ec2', 'route53'] - expected_permissions = ['dynamodb:def', 's3:abc', 'ses:ghi', 'ses:jkl'] - assert repokid.utils.roledata._convert_repoed_service_to_sorted_perms_and_services(repoed_services) == ( - expected_permissions, expected_services + repoed_services = [ + "route53", + "ec2", + "s3:abc", + "dynamodb:def", + "ses:ghi", + "ses:jkl", + ] + expected_services = ["ec2", "route53"] + expected_permissions = ["dynamodb:def", "s3:abc", "ses:ghi", "ses:jkl"] + assert repokid.utils.roledata._convert_repoed_service_to_sorted_perms_and_services( + repoed_services + ) == ( + expected_permissions, + expected_services, ) def test_get_epoch_authenticated(self): - assert (repokid.utils.roledata._get_epoch_authenticated(1545787620000) == (1545787620, True)) - assert (repokid.utils.roledata._get_epoch_authenticated(1545787620) == (1545787620, True)) - assert (repokid.utils.roledata._get_epoch_authenticated(154578762) == (None, False)) + assert repokid.utils.roledata._get_epoch_authenticated(1545787620000) == ( + 1545787620, + True, + ) + assert repokid.utils.roledata._get_epoch_authenticated(1545787620) == ( + 1545787620, + True, + ) + assert repokid.utils.roledata._get_epoch_authenticated(154578762) == ( + None, + False, + ) def test_filter_scheduled_repoable_perms(self): assert repokid.utils.roledata._filter_scheduled_repoable_perms( - ['a:b', 'a:c', 'b:a'], ['a:c', 'b']) == ['a:c', 'b:a'] + ["a:b", "a:c", "b:a"], ["a:c", "b"] + ) == ["a:c", "b:a"] assert repokid.utils.roledata._filter_scheduled_repoable_perms( - ['a:b', 'a:c', 'b:a'], ['a', 'b']) == ['a:b', 'a:c', 'b:a'] + ["a:b", "a:c", "b:a"], ["a", "b"] + ) == ["a:b", "a:c", "b:a"] assert repokid.utils.roledata._filter_scheduled_repoable_perms( - ['a:b', 'a:c', 'b:a'], ['a:b', 'a:c']) == ['a:b', 'a:c'] + ["a:b", "a:c", "b:a"], ["a:b", "a:c"] + ) == ["a:b", "a:c"] def test_get_repoed_policy_sid(self): """ roledata._get_repoed_policy(policies, repoable_permissions) @@ -345,35 +515,43 @@ def __init__(self, sid=None, actions=None): def to_dict(self): policy = { - 'Version': '2012-10-17', - 'Statement': [ - { - 'Action': self.actions, - 'Resource': ['*'], - 'Effect': 'Allow', - } - ] + "Version": "2012-10-17", + "Statement": [ + {"Action": self.actions, "Resource": ["*"], "Effect": "Allow"} + ], } if self.sid: - policy['Statement'][0]['Sid'] = self.sid + policy["Statement"][0]["Sid"] = self.sid return policy sid = "{}-jira1234".format(repokid.utils.roledata.STATEMENT_SKIP_SID) policies = { - 'norepo_sid': TestPolicy(sid=sid, actions=['s3:getobject', 'iam:get*', 'sqs:createqueue']).to_dict(), - 'norepo_used_permissions': TestPolicy(actions=['iam:get*']).to_dict(), - 'repo_some': TestPolicy(actions=['iam:getaccesskeylastused', 'sqs:createqueue']).to_dict(), - 'repo_all': TestPolicy(actions=['sqs:createqueue']).to_dict() + "norepo_sid": TestPolicy( + sid=sid, actions=["s3:getobject", "iam:get*", "sqs:createqueue"] + ).to_dict(), + "norepo_used_permissions": TestPolicy(actions=["iam:get*"]).to_dict(), + "repo_some": TestPolicy( + actions=["iam:getaccesskeylastused", "sqs:createqueue"] + ).to_dict(), + "repo_all": TestPolicy(actions=["sqs:createqueue"]).to_dict(), } - repoable_permissions = ['sqs:createqueue'] - repoed_policies, empty_policies = repokid.utils.roledata._get_repoed_policy(policies, repoable_permissions) + repoable_permissions = ["sqs:createqueue"] + repoed_policies, empty_policies = repokid.utils.roledata._get_repoed_policy( + policies, repoable_permissions + ) - expected_repo_some = TestPolicy(actions=['iam:getaccesskeylastused']).to_dict() - assert ['repo_all'] == empty_policies - assert json.dumps(repoed_policies['norepo_sid']) == json.dumps(policies['norepo_sid']) - assert json.dumps(repoed_policies['norepo_used_permissions']) == json.dumps(policies['norepo_used_permissions']) - assert json.dumps(repoed_policies['repo_some']) == json.dumps(expected_repo_some) + expected_repo_some = TestPolicy(actions=["iam:getaccesskeylastused"]).to_dict() + assert ["repo_all"] == empty_policies + assert json.dumps(repoed_policies["norepo_sid"]) == json.dumps( + policies["norepo_sid"] + ) + assert json.dumps(repoed_policies["norepo_used_permissions"]) == json.dumps( + policies["norepo_used_permissions"] + ) + assert json.dumps(repoed_policies["repo_some"]) == json.dumps( + expected_repo_some + ) def test_get_permissions_in_policy_sid(self): """ roledata._get_permissions_in_policy(policy_dict, warn_unkown_perms=False) @@ -399,31 +577,29 @@ def __init__(self, sid=None, actions=None): def to_dict(self): policy = { - 'Version': '2012-10-17', - 'Statement': [ - { - 'Action': self.actions, - 'Resource': ['*'], - 'Effect': 'Allow', - } - ] + "Version": "2012-10-17", + "Statement": [ + {"Action": self.actions, "Resource": ["*"], "Effect": "Allow"} + ], } if self.sid: - policy['Statement'][0]['Sid'] = self.sid + policy["Statement"][0]["Sid"] = self.sid return policy sid = "{}-jira1234".format(repokid.utils.roledata.STATEMENT_SKIP_SID) policies = { - 'no_sid': TestPolicy(actions=['ec2:getregions']).to_dict(), - 'other_sid': TestPolicy(sid='jira-1234', actions=['sqs:createqueue']).to_dict(), - 'norepo_sid': TestPolicy(sid=sid, actions=['sns:createtopic']).to_dict() + "no_sid": TestPolicy(actions=["ec2:getregions"]).to_dict(), + "other_sid": TestPolicy( + sid="jira-1234", actions=["sqs:createqueue"] + ).to_dict(), + "norepo_sid": TestPolicy(sid=sid, actions=["sns:createtopic"]).to_dict(), } total, eligible = repokid.utils.roledata._get_permissions_in_policy(policies) # eligible is a subset of total assert eligible < total - assert 'ec2:getregions' in eligible - assert 'sqs:createqueue' in eligible - assert 'sns:createtopic' not in eligible + assert "ec2:getregions" in eligible + assert "sqs:createqueue" in eligible + assert "sns:createtopic" not in eligible diff --git a/repokid/utils/dynamo.py b/repokid/utils/dynamo.py index f12c6418f..ebd3f9190 100644 --- a/repokid/utils/dynamo.py +++ b/repokid/utils/dynamo.py @@ -16,20 +16,26 @@ def decorated_func(*args, **kwargs): try: return func(*args, **kwargs) except BotoClientError as e: - LOGGER.error('Dynamo table error: {}'.format(e)) + LOGGER.error("Dynamo table error: {}".format(e)) sys.exit(1) + return decorated_func @catch_boto_error def add_to_end_of_list(dynamo_table, role_id, field_name, object_to_add): - dynamo_table.update_item(Key={'RoleId': role_id}, - UpdateExpression=("SET #updatelist = list_append(if_not_exists(#updatelist," - ":empty_list), :object_to_add)"), - ExpressionAttributeNames={"#updatelist": field_name}, - ExpressionAttributeValues={":empty_list": [], - ":object_to_add": [_empty_string_to_dynamo_replace( - object_to_add)]}) + dynamo_table.update_item( + Key={"RoleId": role_id}, + UpdateExpression=( + "SET #updatelist = list_append(if_not_exists(#updatelist," + ":empty_list), :object_to_add)" + ), + ExpressionAttributeNames={"#updatelist": field_name}, + ExpressionAttributeValues={ + ":empty_list": [], + ":object_to_add": [_empty_string_to_dynamo_replace(object_to_add)], + }, + ) def dynamo_get_or_create_table(**dynamo_config): @@ -48,85 +54,56 @@ def dynamo_get_or_create_table(**dynamo_config): Returns: dynamo_table object """ - if 'localhost' in dynamo_config['endpoint']: - resource = boto3.resource('dynamodb', - region_name='us-east-1', - endpoint_url=dynamo_config['endpoint']) + if "localhost" in dynamo_config["endpoint"]: + resource = boto3.resource( + "dynamodb", region_name="us-east-1", endpoint_url=dynamo_config["endpoint"] + ) else: resource = boto3_cached_conn( - 'dynamodb', - service_type='resource', - account_number=dynamo_config['account_number'], - assume_role=dynamo_config.get('assume_role', None), - session_name=dynamo_config['session_name'], - region=dynamo_config['region']) + "dynamodb", + service_type="resource", + account_number=dynamo_config["account_number"], + assume_role=dynamo_config.get("assume_role", None), + session_name=dynamo_config["session_name"], + region=dynamo_config["region"], + ) for table in resource.tables.all(): - if table.name == 'repokid_roles': + if table.name == "repokid_roles": return table table = None try: table = resource.create_table( - TableName='repokid_roles', - KeySchema=[ - { - 'AttributeName': 'RoleId', - 'KeyType': 'HASH' # Partition key - } - ], + TableName="repokid_roles", + KeySchema=[{"AttributeName": "RoleId", "KeyType": "HASH"}], # Partition key AttributeDefinitions=[ - { - 'AttributeName': 'RoleId', - 'AttributeType': 'S' - }, - { - 'AttributeName': 'RoleName', - 'AttributeType': 'S' - }, - { - 'AttributeName': 'Account', - 'AttributeType': 'S' - } + {"AttributeName": "RoleId", "AttributeType": "S"}, + {"AttributeName": "RoleName", "AttributeType": "S"}, + {"AttributeName": "Account", "AttributeType": "S"}, ], - ProvisionedThroughput={ - 'ReadCapacityUnits': 50, - 'WriteCapacityUnits': 50 - }, + ProvisionedThroughput={"ReadCapacityUnits": 50, "WriteCapacityUnits": 50}, GlobalSecondaryIndexes=[ { - 'IndexName': 'Account', - 'KeySchema': [ - { - 'AttributeName': 'Account', - 'KeyType': 'HASH' - } - ], - 'Projection': { - 'ProjectionType': 'KEYS_ONLY', + "IndexName": "Account", + "KeySchema": [{"AttributeName": "Account", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 10, + "WriteCapacityUnits": 10, }, - 'ProvisionedThroughput': { - 'ReadCapacityUnits': 10, - 'WriteCapacityUnits': 10 - } }, { - 'IndexName': 'RoleName', - 'KeySchema': [ - { - 'AttributeName': 'RoleName', - 'KeyType': 'HASH' - } - ], - 'Projection': - { - 'ProjectionType': 'KEYS_ONLY', + "IndexName": "RoleName", + "KeySchema": [{"AttributeName": "RoleName", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 10, + "WriteCapacityUnits": 10, }, - 'ProvisionedThroughput': { - 'ReadCapacityUnits': 10, - 'WriteCapacityUnits': 10 - } - }]) + }, + ], + ) except BotoClientError as e: LOGGER.error(e) @@ -143,15 +120,19 @@ def find_role_in_cache(dynamo_table, account_number, role_name): Returns: string: RoleID for active role with name in given account, else None """ - results = dynamo_table.query(IndexName='RoleName', - KeyConditionExpression='RoleName = :rn', - ExpressionAttributeValues={':rn': role_name}) - role_id_candidates = [return_dict['RoleId'] for return_dict in results.get('Items')] + results = dynamo_table.query( + IndexName="RoleName", + KeyConditionExpression="RoleName = :rn", + ExpressionAttributeValues={":rn": role_name}, + ) + role_id_candidates = [return_dict["RoleId"] for return_dict in results.get("Items")] if len(role_id_candidates) > 1: for role_id in role_id_candidates: - role_data = get_role_data(dynamo_table, role_id, fields=['Account', 'Active']) - if role_data['Account'] == account_number and role_data['Active']: + role_data = get_role_data( + dynamo_table, role_id, fields=["Account", "Active"] + ) + if role_data["Account"] == account_number and role_data["Active"]: return role_id elif len(role_id_candidates) == 1: return role_id_candidates[0] @@ -171,12 +152,12 @@ def get_role_data(dynamo_table, roleID, fields=None): dict: data for the role if it exists, else None """ if fields: - response = dynamo_table.get_item(Key={'RoleId': roleID}, AttributesToGet=fields) + response = dynamo_table.get_item(Key={"RoleId": roleID}, AttributesToGet=fields) else: - response = dynamo_table.get_item(Key={'RoleId': roleID}) + response = dynamo_table.get_item(Key={"RoleId": roleID}) - if response and 'Item' in response: - return _empty_string_from_dynamo_replace(response['Item']) + if response and "Item" in response: + return _empty_string_from_dynamo_replace(response["Item"]) @catch_boto_error @@ -192,17 +173,21 @@ def role_ids_for_account(dynamo_table, account_number): """ role_ids = set() - results = dynamo_table.query(IndexName='Account', - KeyConditionExpression='Account = :act', - ExpressionAttributeValues={':act': account_number}) - role_ids.update([return_dict['RoleId'] for return_dict in results.get('Items')]) - - while 'LastEvaluatedKey' in results: - results = dynamo_table.query(IndexName='Account', - KeyConditionExpression='Account = :act', - ExpressionAttributeValues={':act': account_number}, - ExclusiveStartKey=results.get('LastEvaluatedKey')) - role_ids.update([return_dict['RoleId'] for return_dict in results.get('Items')]) + results = dynamo_table.query( + IndexName="Account", + KeyConditionExpression="Account = :act", + ExpressionAttributeValues={":act": account_number}, + ) + role_ids.update([return_dict["RoleId"] for return_dict in results.get("Items")]) + + while "LastEvaluatedKey" in results: + results = dynamo_table.query( + IndexName="Account", + KeyConditionExpression="Account = :act", + ExpressionAttributeValues={":act": account_number}, + ExclusiveStartKey=results.get("LastEvaluatedKey"), + ) + role_ids.update([return_dict["RoleId"] for return_dict in results.get("Items")]) return role_ids @@ -219,13 +204,15 @@ def role_ids_for_all_accounts(dynamo_table): """ role_ids = [] - response = dynamo_table.scan(ProjectionExpression='RoleId') - role_ids.extend([role_dict['RoleId'] for role_dict in response['Items']]) + response = dynamo_table.scan(ProjectionExpression="RoleId") + role_ids.extend([role_dict["RoleId"] for role_dict in response["Items"]]) - while 'LastEvaluatedKey' in response: - response = dynamo_table.scan(ProjectionExpression='RoleId', - ExclusiveStartKey=response['LastEvaluatedKey']) - role_ids.extend([role_dict['RoleId'] for role_dict in response['Items']]) + while "LastEvaluatedKey" in response: + response = dynamo_table.scan( + ProjectionExpression="RoleId", + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + role_ids.extend([role_dict["RoleId"] for role_dict in response["Items"]]) return role_ids @@ -238,25 +225,36 @@ def set_role_data(dynamo_table, role_id, update_keys): expression_attribute_names = {} expression_attribute_values = {} count = 0 - for key, value in update_keys.iteritems(): + for key, value in update_keys.items(): count += 1 if count > 1: - update_expression += ', ' + update_expression += ", " value = _empty_string_to_dynamo_replace(value) - update_expression += '#expr{} = :val{}'.format(count, count) - expression_attribute_names['#expr{}'.format(count)] = key - expression_attribute_values[':val{}'.format(count)] = value - - dynamo_table.update_item(Key={'RoleId': role_id}, - UpdateExpression=update_expression, - ExpressionAttributeNames=expression_attribute_names, - ExpressionAttributeValues=expression_attribute_values) - - -def store_initial_role_data(dynamo_table, arn, create_date, role_id, role_name, account_number, current_policy, tags): + update_expression += "#expr{} = :val{}".format(count, count) + expression_attribute_names["#expr{}".format(count)] = key + expression_attribute_values[":val{}".format(count)] = value + + dynamo_table.update_item( + Key={"RoleId": role_id}, + UpdateExpression=update_expression, + ExpressionAttributeNames=expression_attribute_names, + ExpressionAttributeValues=expression_attribute_values, + ) + + +def store_initial_role_data( + dynamo_table, + arn, + create_date, + role_id, + role_name, + account_number, + current_policy, + tags, +): """ Store the initial version of a role in Dynamo @@ -267,17 +265,30 @@ def store_initial_role_data(dynamo_table, arn, create_date, role_id, role_name, Returns: None """ - policy_entry = {'Source': 'Scan', 'Discovered': datetime.datetime.utcnow().isoformat(), 'Policy': current_policy} - - role_dict = {'Arn': arn, 'CreateDate': create_date.isoformat(), 'RoleId': role_id, 'RoleName': role_name, - 'Account': account_number, 'Policies': [policy_entry], - 'Refreshed': datetime.datetime.utcnow().isoformat(), 'Active': True, 'Repoed': 'Never', 'Tags': tags} + policy_entry = { + "Source": "Scan", + "Discovered": datetime.datetime.utcnow().isoformat(), + "Policy": current_policy, + } + + role_dict = { + "Arn": arn, + "CreateDate": create_date.isoformat(), + "RoleId": role_id, + "RoleName": role_name, + "Account": account_number, + "Policies": [policy_entry], + "Refreshed": datetime.datetime.utcnow().isoformat(), + "Active": True, + "Repoed": "Never", + "Tags": tags, + } store_dynamo = copy.copy(role_dict) dynamo_table.put_item(Item=_empty_string_to_dynamo_replace(store_dynamo)) # we want to store CreateDate as a string but keep it as a datetime, so put it back here - role_dict['CreateDate'] = create_date + role_dict["CreateDate"] = create_date return role_dict @@ -292,12 +303,12 @@ def _empty_string_from_dynamo_replace(obj): object: Object with original empty strings """ if isinstance(obj, dict): - return {k: _empty_string_from_dynamo_replace(v) for k, v in obj.items()} + return {k: _empty_string_from_dynamo_replace(v) for k, v in list(obj.items())} elif isinstance(obj, list): return [_empty_string_from_dynamo_replace(elem) for elem in obj] else: if str(obj) == DYNAMO_EMPTY_STRING: - obj = '' + obj = "" return obj @@ -312,12 +323,12 @@ def _empty_string_to_dynamo_replace(obj): object: Object with Dynamo friendly empty strings """ if isinstance(obj, dict): - return {k: _empty_string_to_dynamo_replace(v) for k, v in obj.items()} + return {k: _empty_string_to_dynamo_replace(v) for k, v in list(obj.items())} elif isinstance(obj, list): return [_empty_string_to_dynamo_replace(elem) for elem in obj] else: try: - if str(obj) == '': + if str(obj) == "": obj = DYNAMO_EMPTY_STRING except UnicodeEncodeError: obj = DYNAMO_EMPTY_STRING diff --git a/repokid/utils/roledata.py b/repokid/utils/roledata.py index f442fbcce..feea484fa 100644 --- a/repokid/utils/roledata.py +++ b/repokid/utils/roledata.py @@ -23,24 +23,29 @@ from repokid import LOGGER as LOGGER import repokid.hooks from repokid.role import Role -from repokid.utils.dynamo import (add_to_end_of_list, get_role_data, role_ids_for_account, set_role_data, - store_initial_role_data) +from repokid.utils.dynamo import ( + add_to_end_of_list, + get_role_data, + role_ids_for_account, + set_role_data, + store_initial_role_data, +) BEGINNING_OF_2015_MILLI_EPOCH = 1420113600000 -IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES = frozenset(['']) -IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS = frozenset(['iam:passrole']) +IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES = frozenset([""]) +IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS = frozenset(["iam:passrole"]) -STATEMENT_SKIP_SID = 'NOREPO' +STATEMENT_SKIP_SID = "NOREPO" # permission decisions have the form repoable - boolean, and decider - string class RepoablePermissionDecision(object): def __init__(self): self.repoable = None - self.decider = '' + self.decider = "" def __repr__(self): - return('Is repoable: {}, Decider: {}'.format(self.repoable, self.decider)) + return "Is repoable: {}, Decider: {}".format(self.repoable, self.decider) def add_new_policy_version(dynamo_table, role, current_policy, update_source): @@ -57,11 +62,16 @@ def add_new_policy_version(dynamo_table, role, current_policy, update_source): Returns: None """ - policy_entry = {'Source': update_source, 'Discovered': datetime.datetime.utcnow().isoformat(), - 'Policy': current_policy} + policy_entry = { + "Source": update_source, + "Discovered": datetime.datetime.utcnow().isoformat(), + "Policy": current_policy, + } - add_to_end_of_list(dynamo_table, role.role_id, 'Policies', policy_entry) - role.policies = get_role_data(dynamo_table, role.role_id, fields=['Policies'])['Policies'] + add_to_end_of_list(dynamo_table, role.role_id, "Policies", policy_entry) + role.policies = get_role_data(dynamo_table, role.role_id, fields=["Policies"])[ + "Policies" + ] def find_and_mark_inactive(dynamo_table, account_number, active_roles): @@ -82,9 +92,9 @@ def find_and_mark_inactive(dynamo_table, account_number, active_roles): inactive_roles = known_roles - active_roles for roleID in inactive_roles: - role_dict = get_role_data(dynamo_table, roleID, fields=['Active', 'Arn']) - if role_dict.get('Active'): - set_role_data(dynamo_table, roleID, {'Active': False}) + role_dict = get_role_data(dynamo_table, roleID, fields=["Active", "Arn"]) + if role_dict.get("Active"): + set_role_data(dynamo_table, roleID, {"Active": False}) def find_newly_added_permissions(old_policy, new_policy): @@ -99,8 +109,12 @@ def find_newly_added_permissions(old_policy, new_policy): Returns: set: Exapnded set of permissions that are in the new policy and not the old one """ - old_permissions, _ = _get_role_permissions(Role({'Policies': [{'Policy': old_policy}]})) - new_permissions, _ = _get_role_permissions(Role({'Policies': [{'Policy': new_policy}]})) + old_permissions, _ = _get_role_permissions( + Role({"Policies": [{"Policy": old_policy}]}) + ) + new_permissions, _ = _get_role_permissions( + Role({"Policies": [{"Policy": new_policy}]}) + ) return new_permissions - old_permissions @@ -118,23 +132,32 @@ def update_no_repo_permissions(dynamo_table, role, newly_added_permissions): None """ current_ignored_permissions = get_role_data( - dynamo_table, role.role_id, fields=['NoRepoPermissions']).get('NoRepoPermissions', {}) + dynamo_table, role.role_id, fields=["NoRepoPermissions"] + ).get("NoRepoPermissions", {}) new_ignored_permissions = {} current_time = int(time.time()) new_perms_expire_time = current_time + ( - 24 * 60 * 60 * CONFIG['repo_requirements'].get('exclude_new_permissions_for_days', 14)) + 24 + * 60 + * 60 + * CONFIG["repo_requirements"].get("exclude_new_permissions_for_days", 14) + ) # only copy non-expired items to the new dictionary - for permission, expire_time in current_ignored_permissions.items(): + for permission, expire_time in list(current_ignored_permissions.items()): if expire_time > current_time: - new_ignored_permissions[permission] = current_ignored_permissions[permission] + new_ignored_permissions[permission] = current_ignored_permissions[ + permission + ] for permission in newly_added_permissions: new_ignored_permissions[permission] = new_perms_expire_time role.no_repo_permissions = new_ignored_permissions - set_role_data(dynamo_table, role.role_id, {'NoRepoPermissions': role.no_repo_permissions}) + set_role_data( + dynamo_table, role.role_id, {"NoRepoPermissions": role.no_repo_permissions} + ) def update_opt_out(dynamo_table, role): @@ -148,11 +171,13 @@ def update_opt_out(dynamo_table, role): Returns: None """ - if role.opt_out and int(role.opt_out['expire']) < int(time.time()): - set_role_data(dynamo_table, role.role_id, {'OptOut': {}}) + if role.opt_out and int(role.opt_out["expire"]) < int(time.time()): + set_role_data(dynamo_table, role.role_id, {"OptOut": {}}) -def update_role_data(dynamo_table, account_number, role, current_policy, source='Scan', add_no_repo=True): +def update_role_data( + dynamo_table, account_number, role, current_policy, source="Scan", add_no_repo=True +): """ Compare the current version of a policy for a role and what has been previously stored in Dynamo. - If current and new policy versions are different store the new version in Dynamo. Add any newly added @@ -173,40 +198,60 @@ def update_role_data(dynamo_table, account_number, role, current_policy, source= """ # policy_entry: source, discovered, policy - stored_role = get_role_data(dynamo_table, role.role_id, fields=['OptOut', 'Policies', 'Tags']) + stored_role = get_role_data( + dynamo_table, role.role_id, fields=["OptOut", "Policies", "Tags"] + ) if not stored_role: - role_dict = store_initial_role_data(dynamo_table, role.arn, role.create_date, role.role_id, role.role_name, - account_number, current_policy, role.tags) + role_dict = store_initial_role_data( + dynamo_table, + role.arn, + role.create_date, + role.role_id, + role.role_name, + account_number, + current_policy, + role.tags, + ) role.set_attributes(role_dict) - LOGGER.info('Added new role ({}): {}'.format(role.role_id, role.arn)) + LOGGER.info("Added new role ({}): {}".format(role.role_id, role.arn)) else: # is the policy list the same as the last we had? - old_policy = stored_role['Policies'][-1]['Policy'] + old_policy = stored_role["Policies"][-1]["Policy"] if current_policy != old_policy: add_new_policy_version(dynamo_table, role, current_policy, source) - LOGGER.info('{} has different inline policies than last time, adding to role store'.format(role.arn)) - - newly_added_permissions = find_newly_added_permissions(old_policy, current_policy) + LOGGER.info( + "{} has different inline policies than last time, adding to role store".format( + role.arn + ) + ) + + newly_added_permissions = find_newly_added_permissions( + old_policy, current_policy + ) else: newly_added_permissions = set() # update tags if needed - if role.tags != stored_role.get('Tags', []): - set_role_data(dynamo_table, role.role_id, {'Tags': role.tags}) + if role.tags != stored_role.get("Tags", []): + set_role_data(dynamo_table, role.role_id, {"Tags": role.tags}) if add_no_repo: update_no_repo_permissions(dynamo_table, role, newly_added_permissions) update_opt_out(dynamo_table, role) - set_role_data(dynamo_table, role.role_id, {'Refreshed': datetime.datetime.utcnow().isoformat()}) + set_role_data( + dynamo_table, + role.role_id, + {"Refreshed": datetime.datetime.utcnow().isoformat()}, + ) # Update all data from Dynamo except CreateDate (it's in the wrong format) and DQ_by (we're going to recalc) current_role_data = get_role_data(dynamo_table, role.role_id) - current_role_data.pop('CreateDate', None) - current_role_data.pop('DisqualifiedBy', None) + current_role_data.pop("CreateDate", None) + current_role_data.pop("DisqualifiedBy", None) role.set_attributes(current_role_data) -def update_stats(dynamo_table, roles, source='Scan'): +def update_stats(dynamo_table, roles, source="Scan"): """ Create a new stats entry for each role in a set of roles and add it to Dynamo @@ -218,24 +263,34 @@ def update_stats(dynamo_table, roles, source='Scan'): None """ for role in roles: - new_stats = {'Date': datetime.datetime.utcnow().isoformat(), - 'DisqualifiedBy': role.disqualified_by, - 'PermissionsCount': role.total_permissions, - 'RepoablePermissionsCount': role.repoable_permissions, - 'Source': source} + new_stats = { + "Date": datetime.datetime.utcnow().isoformat(), + "DisqualifiedBy": role.disqualified_by, + "PermissionsCount": role.total_permissions, + "RepoablePermissionsCount": role.repoable_permissions, + "Source": source, + } try: cur_stats = role.stats[-1] except IndexError: - cur_stats = {'DisqualifiedBy': [], 'PermissionsCount': 0, 'RepoablePermissionsCount': 0} + cur_stats = { + "DisqualifiedBy": [], + "PermissionsCount": 0, + "RepoablePermissionsCount": 0, + } - for item in ['DisqualifiedBy', 'PermissionsCount', 'RepoablePermissionsCount']: + for item in ["DisqualifiedBy", "PermissionsCount", "RepoablePermissionsCount"]: if new_stats.get(item) != cur_stats.get(item): - add_to_end_of_list(dynamo_table, role.role_id, 'Stats', new_stats) + add_to_end_of_list(dynamo_table, role.role_id, "Stats", new_stats) def _update_repoable_services(role, repoable_permissions, eligible_permissions): - (repoable_permissions_set, repoable_services_set) = _convert_repoable_perms_to_perms_and_services( - eligible_permissions, repoable_permissions) + ( + repoable_permissions_set, + repoable_services_set, + ) = _convert_repoable_perms_to_perms_and_services( + eligible_permissions, repoable_permissions + ) # we're going to store both repoable permissions and repoable services in the field "RepoableServices" role.repoable_services = repoable_services_set + repoable_permissions_set @@ -268,7 +323,7 @@ def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=10 # if we don't have any access advisor data for a service than nothing is repoable if not role.aa_data: - LOGGER.info('No data found in access advisor for {}'.format(role.role_id)) + LOGGER.info("No data found in access advisor for {}".format(role.role_id)) role.repoable_permissions = 0 role.repoable_services = [] continue @@ -284,13 +339,19 @@ def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=10 repoable_permissions_dict = {} if batch: repoable_permissions_dict = _get_repoable_permissions_batch( - repo_able_roles, eligible_permissions_dict, minimum_age, hooks, batch_size) + repo_able_roles, eligible_permissions_dict, minimum_age, hooks, batch_size + ) else: for role in repo_able_roles: - repoable_permissions_dict[role.arn] = _get_repoable_permissions(role.account, role.role_name, - eligible_permissions_dict[role.arn], - role.aa_data, role.no_repo_permissions, - minimum_age, hooks) + repoable_permissions_dict[role.arn] = _get_repoable_permissions( + role.account, + role.role_name, + eligible_permissions_dict[role.arn], + role.aa_data, + role.no_repo_permissions, + minimum_age, + hooks, + ) for role in repo_able_roles: eligible_permissions = eligible_permissions_dict[role.arn] @@ -298,7 +359,9 @@ def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=10 _update_repoable_services(role, repoable_permissions, eligible_permissions) -def _convert_repoable_perms_to_perms_and_services(total_permissions, repoable_permissions): +def _convert_repoable_perms_to_perms_and_services( + total_permissions, repoable_permissions +): """ Take a list of total permissions and repoable permissions and determine whether only a few permissions are being repoed or if the entire service (all permissions from that service) are being removed. @@ -320,16 +383,21 @@ def _convert_repoable_perms_to_perms_and_services(total_permissions, repoable_pe # group total permissions and repoable permissions by service for perm in total_permissions: - total_perms_by_service[perm.split(':')[0]].append(perm) + total_perms_by_service[perm.split(":")[0]].append(perm) for perm in repoable_permissions: - repoable_perms_by_service[perm.split(':')[0]].append(perm) + repoable_perms_by_service[perm.split(":")[0]].append(perm) for service in repoable_perms_by_service: - if all(perm in repoable_perms_by_service[service] for perm in total_perms_by_service[service]): + if all( + perm in repoable_perms_by_service[service] + for perm in total_perms_by_service[service] + ): repoed_services.add(service) else: - repoed_permissions.update(perm for perm in repoable_perms_by_service[service]) + repoed_permissions.update( + perm for perm in repoable_perms_by_service[service] + ) return (sorted(repoed_permissions), sorted(repoed_services)) @@ -352,7 +420,7 @@ def _convert_repoed_service_to_sorted_perms_and_services(repoed_services): repoable_services = set() for entry in repoed_services: - if len(entry.split(':')) == 2: + if len(entry.split(":")) == 2: repoable_permissions.add(entry) else: repoable_services.add(entry) @@ -370,9 +438,15 @@ def _filter_scheduled_repoable_perms(repoable_permissions, scheduled_perms): Returns: list: New (filtered) repoable permissions """ - (scheduled_permissions, scheduled_services) = _convert_repoed_service_to_sorted_perms_and_services(scheduled_perms) - return([perm for perm in repoable_permissions - if(perm in scheduled_permissions or perm.split(':')[0] in scheduled_services)]) + ( + scheduled_permissions, + scheduled_services, + ) = _convert_repoed_service_to_sorted_perms_and_services(scheduled_perms) + return [ + perm + for perm in repoable_permissions + if (perm in scheduled_permissions or perm.split(":")[0] in scheduled_services) + ] def _get_epoch_authenticated(service_authenticated): @@ -401,58 +475,80 @@ def _get_epoch_authenticated(service_authenticated): return (None, False) -def _get_potentially_repoable_permissions(role_name, account_number, aa_data, permissions, no_repo_permissions, - minimum_age): +def _get_potentially_repoable_permissions( + role_name, account_number, aa_data, permissions, no_repo_permissions, minimum_age +): ago = datetime.timedelta(minimum_age) now = datetime.datetime.now(tzlocal()) current_time = time.time() - no_repo_list = [perm.lower() for perm in no_repo_permissions if no_repo_permissions[perm] > current_time] + no_repo_list = [ + perm.lower() + for perm in no_repo_permissions + if no_repo_permissions[perm] > current_time + ] # cast all permissions to lowercase permissions = [permission.lower() for permission in permissions] - potentially_repoable_permissions = {permission: RepoablePermissionDecision() - for permission in permissions if permission not in no_repo_list} + potentially_repoable_permissions = { + permission: RepoablePermissionDecision() + for permission in permissions + if permission not in no_repo_list + } used_services = set() for service in aa_data: - (accessed, valid_authenticated) = _get_epoch_authenticated(service['lastAuthenticated']) + (accessed, valid_authenticated) = _get_epoch_authenticated( + service["lastAuthenticated"] + ) if not accessed: continue if not valid_authenticated: - LOGGER.error("Got malformed Access Advisor data for {role_name} in {account_number} for service {service}" - ": {last_authenticated}".format( - role_name=role_name, - account_number=account_number, - service=service.get('serviceNamespace'), - last_authenticated=service['lastAuthenticated'])) - used_services.add(service['serviceNamespace']) + LOGGER.error( + "Got malformed Access Advisor data for {role_name} in {account_number} for service {service}" + ": {last_authenticated}".format( + role_name=role_name, + account_number=account_number, + service=service.get("serviceNamespace"), + last_authenticated=service["lastAuthenticated"], + ) + ) + used_services.add(service["serviceNamespace"]) accessed = datetime.datetime.fromtimestamp(accessed, tzlocal()) if accessed > now - ago: - used_services.add(service['serviceNamespace']) + used_services.add(service["serviceNamespace"]) - for permission_name, permission_decision in potentially_repoable_permissions.items(): - if permission_name.split(':')[0] in IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES: - LOGGER.warn('skipping {}'.format(permission_name)) + for permission_name, permission_decision in list( + potentially_repoable_permissions.items() + ): + if permission_name.split(":")[0] in IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES: + LOGGER.warn("skipping {}".format(permission_name)) continue # we have an unused service but need to make sure it's repoable - if permission_name.split(':')[0] not in used_services: + if permission_name.split(":")[0] not in used_services: if permission_name in IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS: - LOGGER.warn('skipping {}'.format(permission_name)) + LOGGER.warn("skipping {}".format(permission_name)) continue permission_decision.repoable = True - permission_decision.decider = 'Access Advisor' + permission_decision.decider = "Access Advisor" return potentially_repoable_permissions -def _get_repoable_permissions(account_number, role_name, permissions, aa_data, no_repo_permissions, minimum_age, - hooks): +def _get_repoable_permissions( + account_number, + role_name, + permissions, + aa_data, + no_repo_permissions, + minimum_age, + hooks, +): """ Generate a list of repoable permissions for a role based on the list of all permissions the role's policies currently allow and Access Advisor data for the services included in the role's policies. @@ -477,25 +573,52 @@ def _get_repoable_permissions(account_number, role_name, permissions, aa_data, n set: Permissions that are 'repoable' (not used within the time threshold) """ potentially_repoable_permissions = _get_potentially_repoable_permissions( - role_name, account_number, aa_data, permissions, no_repo_permissions, minimum_age) - - hooks_output = repokid.hooks.call_hooks(hooks, 'DURING_REPOABLE_CALCULATION', - {'account_number': account_number, - 'role_name': role_name, - 'potentially_repoable_permissions': potentially_repoable_permissions, - 'minimum_age': minimum_age}) - - LOGGER.debug('Repoable permissions for role {role_name} in {account_number}:\n{repoable}'.format( - role_name=role_name, - account_number=account_number, - repoable=''.join('{}: {}\n'.format(perm, decision.decider) - for perm, decision in hooks_output['potentially_repoable_permissions'].items()))) + role_name, + account_number, + aa_data, + permissions, + no_repo_permissions, + minimum_age, + ) + + hooks_output = repokid.hooks.call_hooks( + hooks, + "DURING_REPOABLE_CALCULATION", + { + "account_number": account_number, + "role_name": role_name, + "potentially_repoable_permissions": potentially_repoable_permissions, + "minimum_age": minimum_age, + }, + ) + + LOGGER.debug( + "Repoable permissions for role {role_name} in {account_number}:\n{repoable}".format( + role_name=role_name, + account_number=account_number, + repoable="".join( + "{}: {}\n".format(perm, decision.decider) + for perm, decision in list( + hooks_output["potentially_repoable_permissions"].items() + ) + ), + ) + ) - return set([permission_name for permission_name, permission_value in - hooks_output['potentially_repoable_permissions'].items() if permission_value.repoable]) + return set( + [ + permission_name + for permission_name, permission_value in list( + hooks_output["potentially_repoable_permissions"].items() + ) + if permission_value.repoable + ] + ) -def _get_repoable_permissions_batch(repo_able_roles, permissions_dict, minimum_age, hooks, batch_size): +def _get_repoable_permissions_batch( + repo_able_roles, permissions_dict, minimum_age, hooks, batch_size +): """ Generate a dictionary mapping of role arns to their repoable permissions based on the list of all permissions the role's policies currently allow and Access Advisor data for the services included in the role's policies. @@ -524,33 +647,56 @@ def _get_repoable_permissions_batch(repo_able_roles, permissions_dict, minimum_a repoable_log_dict = {} for role in repo_able_roles: - potentially_repoable_permissions_dict[role.arn] = ( - _get_potentially_repoable_permissions(role.role_name, role.account, role.aa_data, - permissions_dict[role.arn], role.no_repo_permissions, minimum_age) + potentially_repoable_permissions_dict[ + role.arn + ] = _get_potentially_repoable_permissions( + role.role_name, + role.account, + role.aa_data, + permissions_dict[role.arn], + role.no_repo_permissions, + minimum_age, ) while len(repo_able_roles_batches) > 0: role_batch = repo_able_roles_batches[:batch_size] repo_able_roles_batches = repo_able_roles_batches[batch_size:] - hooks_output = repokid.hooks.call_hooks(hooks, 'DURING_REPOABLE_CALCULATION_BATCH', - {'role_batch': role_batch, - 'potentially_repoable_permissions': - potentially_repoable_permissions_dict, - 'minimum_age': minimum_age}) - for role_arn, output in hooks_output.items(): - repoable_set = set([permission_name for permission_name, permission_value in - output['potentially_repoable_permissions'].items() if permission_value.repoable]) + hooks_output = repokid.hooks.call_hooks( + hooks, + "DURING_REPOABLE_CALCULATION_BATCH", + { + "role_batch": role_batch, + "potentially_repoable_permissions": potentially_repoable_permissions_dict, + "minimum_age": minimum_age, + }, + ) + for role_arn, output in list(hooks_output.items()): + repoable_set = set( + [ + permission_name + for permission_name, permission_value in list( + output["potentially_repoable_permissions"].items() + ) + if permission_value.repoable + ] + ) repoable_set_dict[role_arn] = repoable_set - repoable_log_dict[role_arn] = ''.join('{}: {}\n'.format(perm, decision.decider) - for perm, decision in - output['potentially_repoable_permissions'].items()) + repoable_log_dict[role_arn] = "".join( + "{}: {}\n".format(perm, decision.decider) + for perm, decision in list( + output["potentially_repoable_permissions"].items() + ) + ) for role in repo_able_roles: - LOGGER.debug('Repoable permissions for role {role_name} in {account_number}:\n{repoable}'.format( - role_name=role.role_name, - account_number=role.account, - repoable=repoable_log_dict[role.arn])) + LOGGER.debug( + "Repoable permissions for role {role_name} in {account_number}:\n{repoable}".format( + role_name=role.role_name, + account_number=role.account, + repoable=repoable_log_dict[role.arn], + ) + ) return repoable_set_dict @@ -576,16 +722,18 @@ def _get_repoed_policy(policies, repoable_permissions): role_policies = copy.deepcopy(policies) empty_policies = [] - for policy_name, policy in role_policies.items(): + for policy_name, policy in list(role_policies.items()): # list of indexes in the policy that are empty empty_statements = [] - if type(policy['Statement']) is dict: - policy['Statement'] = [policy['Statement']] + if type(policy["Statement"]) is dict: + policy["Statement"] = [policy["Statement"]] - for idx, statement in enumerate(policy['Statement']): - if statement['Effect'].lower() == 'allow': - if 'Sid' in statement and statement['Sid'].startswith(STATEMENT_SKIP_SID): + for idx, statement in enumerate(policy["Statement"]): + if statement["Effect"].lower() == "allow": + if "Sid" in statement and statement["Sid"].startswith( + STATEMENT_SKIP_SID + ): continue statement_actions = get_actions_from_statement(statement) @@ -597,22 +745,22 @@ def _get_repoed_policy(policies, repoable_permissions): statement_actions = statement_actions.difference(repoable_permissions) # get_actions_from_statement has already inverted this so our new statement should be 'Action' - if 'NotAction' in statement: - del statement['NotAction'] + if "NotAction" in statement: + del statement["NotAction"] # by putting this into a set, we lose order, which may be confusing to someone. - statement['Action'] = sorted(list(statement_actions)) + statement["Action"] = sorted(list(statement_actions)) # mark empty statements to be removed - if len(statement['Action']) == 0: + if len(statement["Action"]) == 0: empty_statements.append(idx) # do the actual removal of empty statements for idx in sorted(empty_statements, reverse=True): - del policy['Statement'][idx] + del policy["Statement"][idx] # mark empty policies to be removed - if len(policy['Statement']) == 0: + if len(policy["Statement"]) == 0: empty_policies.append(policy_name) # do the actual removal of empty policies. @@ -638,19 +786,26 @@ def _get_permissions_in_policy(policy_dict, warn_unknown_perms=False): total_permissions = set() eligible_permissions = set() - for policy_name, policy in policy_dict.items(): + for policy_name, policy in list(policy_dict.items()): policy = expand_policy(policy=policy, expand_deny=False) - for statement in policy.get('Statement'): - if statement['Effect'].lower() == 'allow': - total_permissions = total_permissions.union(get_actions_from_statement(statement)) - if not ('Sid' in statement and statement['Sid'].startswith(STATEMENT_SKIP_SID)): + for statement in policy.get("Statement"): + if statement["Effect"].lower() == "allow": + total_permissions = total_permissions.union( + get_actions_from_statement(statement) + ) + if not ( + "Sid" in statement + and statement["Sid"].startswith(STATEMENT_SKIP_SID) + ): # No Sid # Sid exists, but doesn't start with STATEMENT_SKIP_SID - eligible_permissions = eligible_permissions.union(get_actions_from_statement(statement)) + eligible_permissions = eligible_permissions.union( + get_actions_from_statement(statement) + ) weird_permissions = total_permissions.difference(all_permissions) if weird_permissions and warn_unknown_perms: - LOGGER.warn('Unknown permissions found: {}'.format(weird_permissions)) + LOGGER.warn("Unknown permissions found: {}".format(weird_permissions)) return total_permissions, eligible_permissions @@ -671,7 +826,7 @@ def _get_role_permissions(role, warn_unknown_perms=False): set - all permissions allowed by the policies set - all permisisons allowed by the policies not marked with STATEMENT_SKIP_SID """ - return _get_permissions_in_policy(role.policies[-1]['Policy']) + return _get_permissions_in_policy(role.policies[-1]["Policy"]) def _get_services_in_permissions(permissions_set): @@ -687,7 +842,7 @@ def _get_services_in_permissions(permissions_set): services_set = set() for permission in permissions_set: try: - service = permission.split(':')[0] + service = permission.split(":")[0] except IndexError: pass else: diff --git a/requirements.in b/requirements.in index fef751f84..5dbcee4fc 100644 --- a/requirements.in +++ b/requirements.in @@ -2,7 +2,7 @@ boto3 cloudaux docopt import_string -marshmallow +marshmallow<3 policyuniverse requests tabulate @@ -10,3 +10,4 @@ tabview tqdm pip-tools twine +raven diff --git a/requirements.txt b/requirements.txt index 836b5ebb6..c7878147f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,38 +4,40 @@ # # pip-compile --output-file requirements.txt requirements.in # -bleach==3.0.2 # via readme-renderer -boto3==1.9.71 +--index-url https://artifacts.netflix.com/api/pypi/pypi-netflix/simple + +bleach==3.1.0 # via readme-renderer +boto3==1.9.216 boto==2.49.0 # via cloudaux -botocore==1.12.71 # via boto3, cloudaux, s3transfer -certifi==2018.11.29 # via requests +botocore==1.12.216 # via boto3, cloudaux, s3transfer +certifi==2019.6.16 # via requests chardet==3.0.4 # via requests click==7.0 # via pip-tools -cloudaux==1.5.7 -defusedxml==0.5.0 # via cloudaux +cloudaux==1.6.4 +defusedxml==0.6.0 # via cloudaux docopt==0.6.2 -docutils==0.14 # via botocore, readme-renderer -flagpole==1.0.1 # via cloudaux -futures==3.2.0 # via s3transfer +docutils==0.15.2 # via botocore, readme-renderer +flagpole==1.1.1 # via cloudaux idna==2.8 # via requests import-string==0.1.0 inflection==0.3.1 # via cloudaux -jmespath==0.9.3 # via boto3, botocore -joblib==0.13.0 # via cloudaux -marshmallow==2.17.0 -pip-tools==3.2.0 -pkginfo==1.4.2 # via twine -policyuniverse==1.3.0.1 -pygments==2.3.1 # via readme-renderer -python-dateutil==2.7.5 # via botocore +jmespath==0.9.4 # via boto3, botocore +joblib==0.13.2 # via cloudaux +marshmallow==2.20.2 +pip-tools==4.1.0 +pkginfo==1.5.0.1 # via twine +policyuniverse==1.3.2.0 +pygments==2.4.2 # via readme-renderer +python-dateutil==2.8.0 # via botocore +raven==6.10.0 readme-renderer==24.0 # via twine -requests-toolbelt==0.8.0 # via twine -requests==2.21.0 -s3transfer==0.1.13 # via boto3 +requests-toolbelt==0.9.1 # via twine +requests==2.22.0 +s3transfer==0.2.1 # via boto3 six==1.12.0 # via bleach, cloudaux, import-string, pip-tools, python-dateutil, readme-renderer -tabulate==0.8.2 +tabulate==0.8.3 tabview==1.4.3 -tqdm==4.28.1 -twine==1.12.1 -urllib3==1.24.1 # via botocore, requests +tqdm==4.35.0 +twine==1.13.0 +urllib3==1.25.3 # via botocore, requests webencodings==0.5.1 # via bleach diff --git a/setup.py b/setup.py index e0601c872..9388bebb7 100644 --- a/setup.py +++ b/setup.py @@ -17,42 +17,43 @@ from setuptools import find_packages, setup -with open('requirements.txt') as f: +with open("requirements.in") as f: REQUIRED = f.read().splitlines() -_version_re = re.compile(r'__version__\s+=\s+(.*)') -with open('repokid/__init__.py', 'rb') as f: - REPOKID_VERSION = str(ast.literal_eval(_version_re.search( - f.read().decode('utf-8')).group(1))) +_version_re = re.compile(r"__version__\s+=\s+(.*)") +with open("repokid/__init__.py", "rb") as f: + REPOKID_VERSION = str( + ast.literal_eval(_version_re.search(f.read().decode("utf-8")).group(1)) + ) setup( - name='repokid', + name="repokid", version=REPOKID_VERSION, - description='AWS Least Privilege for Distributed, High-Velocity Deployment', + description="AWS Least Privilege for Distributed, High-Velocity Deployment", # removed as I think getting long_desc to work is perhaps outside the scope # of this PR, other long_desc's I've seen have used .rst to display on # Pypi, so I think that may be necessary also. # long_description=open("readme.md").read(), - url='https://github.com/Netflix/repokid', + url="https://github.com/Netflix/repokid", packages=find_packages(), install_requires=REQUIRED, - keywords=['aws', 'iam', 'access_advisor'], + keywords=["aws", "iam", "access_advisor"], entry_points={ - 'console_scripts': [ - 'repokid = repokid.cli.repokid_cli:main', - 'dispatcher = repokid.cli.dispatcher_cli:main' - ], + "console_scripts": [ + "repokid = repokid.cli.repokid_cli:main", + "dispatcher = repokid.cli.dispatcher_cli:main", + ] }, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Apache Software License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Topic :: Security', - 'Topic :: System', - 'Topic :: System :: Systems Administration' - ] + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.7", + "Topic :: Security", + "Topic :: System", + "Topic :: System :: Systems Administration", + ], )