From 670a033a84366b199673dd6b688e062e9c6da949 Mon Sep 17 00:00:00 2001 From: "maram.el-salamouny" Date: Fri, 20 Nov 2020 10:30:21 -0500 Subject: [PATCH] Add managed policies --- Dockerfile | 22 ++- repokid/commands/repo.py | 279 ++++++++++++++++++++++++--------- repokid/commands/role.py | 28 +++- repokid/commands/role_cache.py | 23 ++- repokid/role.py | 4 + repokid/utils/dynamo.py | 9 ++ repokid/utils/roledata.py | 189 ++++++++++++++++++++-- 7 files changed, 455 insertions(+), 99 deletions(-) diff --git a/Dockerfile b/Dockerfile index 679cc993d..a2ac0f55c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,24 @@ RUN pip install bandit coveralls && \ pip install . && \ pip install -r requirements-test.txt && \ python setup.py develop && \ - repokid config config.json # Generate example config + apt update -y && \ + apt install vim -y && \ + rm -rf /usr/src/app/ # delete the code from the container at build time then mount it at run time using -volume flag. + #repokid config config.json # Generate example config -ENTRYPOINT ["repokid"] +EXPOSE 5000 + +ENTRYPOINT ["bash"] + + + +# docker build . -tag repokid:latest + +# docker run --rm --name repokid -it --network aardvark_default --volume /home/ec2-user/repokid/repokid:/usr/local/lib/python3.7/site-packages/repokid repokid:1.0 bash +# docker exec -it repokid + + +main running somewhere +namespace + +python auto reload module. diff --git a/repokid/commands/repo.py b/repokid/commands/repo.py index f0b6f4d3a..c27612e01 100644 --- a/repokid/commands/repo.py +++ b/repokid/commands/repo.py @@ -47,91 +47,108 @@ LOGGER = logging.getLogger("repokid") -def _repo_role( - account_number, - role_name, - dynamo_table, - config, - hooks, - commit=False, - scheduled=False, -): - """ - Calculate what repoing can be done for a role and then actually do it if commit is set - 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data - 2) Get the role's current permissions, repoable permissions, and the new policy if it will change - 3) Make the changes if commit is set - Args: - account_number (string) - role_name (string) - commit (bool) - - Returns: - None - """ +def _deal_with_policies(role, account_number, config, hooks, scheduled, role_name, dynamo_table, commit, continuing): errors = [] - - role_id = find_role_in_cache(dynamo_table, account_number, role_name) - # only load partial data that we need to determine if we should keep going - role_data = get_role_data( - dynamo_table, - role_id, - fields=["DisqualifiedBy", "AAData", "RepoablePermissions", "RoleName"], + total_permissions, eligible_permissions = roledata._get_role_permissions(role) + repoable_permissions = roledata._get_repoable_permissions( + account_number, + role.role_name, + eligible_permissions, + role.aa_data, + role.no_repo_permissions, + config["filter_config"]["AgeFilter"]["minimum_age"], + hooks, ) - if not role_data: - LOGGER.warn("Could not find role with name {}".format(role_name)) - return - else: - role = Role(role_data) + # if this is a scheduled repo we need to filter out permissions that weren't previously scheduled + if scheduled: + repoable_permissions = roledata._filter_scheduled_repoable_perms( + repoable_permissions, role.scheduled_perms + ) - continuing = True + repoed_policies, deleted_policy_names = roledata._get_repoed_policy( + role.policies[-1]["Policy"], repoable_permissions + ) - if len(role.disqualified_by) > 0: - LOGGER.info( - "Cannot repo role {} in account {} because it is being disqualified by: {}".format( - role_name, account_number, role.disqualified_by - ) + if inline_policies_size_exceeds_maximum(repoed_policies): + error = ( + "Policies would exceed the AWS size limit after repo for role: {} in account {}. " + "Please manually minify.".format(role_name, account_number) ) + LOGGER.error(error) + errors.append(error) continuing = False - if not role.aa_data: - LOGGER.warning("ARN not found in Access Advisor: {}".format(role.arn)) - continuing = False + # if we aren't repoing for some reason, unschedule the role + if not continuing: + set_role_data( + dynamo_table, role.role_id, {"RepoScheduled": 0, "ScheduledPerms": []} + ) + return - if not role.repoable_permissions: - LOGGER.info( - "No permissions to repo for role {} in account {}".format( - role_name, account_number - ) + if not commit: + log_deleted_and_repoed_policies( + deleted_policy_names, repoed_policies, role_name, account_number ) - continuing = False + return - # if we've gotten to this point, load the rest of the role - role = Role(get_role_data(dynamo_table, role_id)) + conn = config["connection_iam"] + conn["account_number"] = account_number - old_aa_data_services = [] - for aa_service in role.aa_data: - if datetime.datetime.strptime( - aa_service["lastUpdated"], "%a, %d %b %Y %H:%M:%S %Z" - ) < datetime.datetime.now() - datetime.timedelta( - days=config["repo_requirements"]["oldest_aa_data_days"] - ): - old_aa_data_services.append(aa_service["serviceName"]) + for name in deleted_policy_names: + error = delete_policy(name, role, account_number, conn) + if error: + LOGGER.error(error) + errors.append(error) - if old_aa_data_services: - LOGGER.error( - "AAData older than threshold for these services: {} (role: {}, account {})".format( - old_aa_data_services, role_name, account_number - ), - exc_info=True, + if repoed_policies: + error = replace_policies(repoed_policies, role, account_number, conn) + if error: + LOGGER.error(error) + errors.append(error) + + current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} + roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") + + # regardless of whether we're successful we want to unschedule the repo + set_role_data( + dynamo_table, role.role_id, {"RepoScheduled": 0, "ScheduledPerms": []} + ) + + repokid.hooks.call_hooks(hooks, "AFTER_REPO", {"role": role, "errors": errors}) + + if not errors: + # repos will stay scheduled until they are successful + set_role_data( + dynamo_table, + role.role_id, + {"Repoed": datetime.datetime.utcnow().isoformat()}, ) - continuing = False + update_repoed_description(role.role_name, **conn) + partial_update_role_data( + role, + dynamo_table, + account_number, + config, + conn, + hooks, + source="Repo", + add_no_repo=False, + ) + LOGGER.info( + "Successfully repoed role: {} in account {}".format( + role.role_name, account_number + ) + ) + return errors - total_permissions, eligible_permissions = roledata._get_role_permissions(role) - repoable_permissions = roledata._get_repoable_permissions( + +def _deal_with_managed_policies(role, account_number, config, hooks, scheduled, role_name, dynamo_table, commit): + errors = [] + total_managed_permissions, eligible_managed_permissions = roledata._get_role_managed_permissions(role) + repoable_managed_permissions = roledata._get_repoable_permissions( account_number, role.role_name, - eligible_permissions, + eligible_managed_permissions, role.aa_data, role.no_repo_permissions, config["filter_config"]["AgeFilter"]["minimum_age"], @@ -140,12 +157,12 @@ def _repo_role( # if this is a scheduled repo we need to filter out permissions that weren't previously scheduled if scheduled: - repoable_permissions = roledata._filter_scheduled_repoable_perms( - repoable_permissions, role.scheduled_perms + repoable_managed_permissions = roledata._filter_scheduled_repoable_perms( + repoable_managed_permissions, role.scheduled_perms ) - repoed_policies, deleted_policy_names = roledata._get_repoed_policy( - role.policies[-1]["Policy"], repoable_permissions + repoed_policies, deleted_policy_names = roledata._get_repoed_managed_policy( # TODO + role.policies[-1]["ManagedPolicy"], repoable_managed_permissions ) if inline_policies_size_exceeds_maximum(repoed_policies): @@ -185,7 +202,7 @@ def _repo_role( LOGGER.error(error) errors.append(error) - current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} + current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} # what should this be? roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") # regardless of whether we're successful we want to unschedule the repo @@ -214,15 +231,124 @@ def _repo_role( add_no_repo=False, ) LOGGER.info( - "Successfully repoed role: {} in account {}".format( + "Successfully repoed role: {} in account {}. Included managed policies: {}".format( role.role_name, account_number ) ) return errors +def _repo_role( + account_number, + role_name, + dynamo_table, + config, + hooks, + include_managed_policies=True, + commit=False, + scheduled=False, +): + """ + Calculate what repoing can be done for a role and then actually do it if commit is set + 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data + 2) Get the role's current permissions, repoable permissions, and the new policy if it will change + 3) If include_managed_policies is set, get the role's current managed permissions, repoable managed permissions, + and the new policy if it will change + 4) Make the changes if commit is set + Args: + account_number (string) + role_name (string) + commit (bool) + + Returns: + None + """ + errors = [] + + role_id = find_role_in_cache(dynamo_table, account_number, role_name) + # only load partial data that we need to determine if we should keep going + role_data = get_role_data( + dynamo_table, + role_id, + fields=["DisqualifiedBy", "AAData", "RepoablePermissions", "RepoableManagedPermissions", "RoleName"], + ) + if not role_data: + LOGGER.warn("Could not find role with name {}".format(role_name)) + return + else: + role = Role(role_data) + + continuing = True + + if len(role.disqualified_by) > 0: + LOGGER.info( + "Cannot repo role {} in account {} because it is being disqualified by: {}".format( + role_name, account_number, role.disqualified_by + ) + ) + continuing = False + + if not role.aa_data: + LOGGER.warning("ARN not found in Access Advisor: {}".format(role.arn)) + continuing = False + + permissionsToDo = [] + if not role.repoable_permissions: + LOGGER.info( + "No permissions to repo for role {} in account {}".format( + role_name, account_number + ) + ) + else: + permissionsToDo.append("inline") + + if include_managed_policies: + if not role.repoable_managed_permissions: + LOGGER.info( + "No managed permissions to repo for role {} in account {}".format( + role_name, account_number + ) + ) + else: + permissionsToDo.append("managed") + + if not permissionsToDo: + continuing = False + + # if we've gotten to this point, load the rest of the role + role = Role(get_role_data(dynamo_table, role_id)) + + fiveDaysAgo = datetime.datetime.now() - datetime.timedelta(days=config["repo_requirements"]["oldest_aa_data_days"]) + old_aa_data_services = [] + for aa_service in role.aa_data: + if datetime.datetime.strptime(aa_service["lastUpdated"], "%a, %d %b %Y %H:%M:%S %Z") < fiveDaysAgo: + old_aa_data_services.append(aa_service["serviceName"]) + + if old_aa_data_services: + LOGGER.error( + "AAData older than threshold for these services: {} (role: {}, account {})".format( + old_aa_data_services, role_name, account_number + ), + exc_info=True, + ) + continuing = False + + if "inline" in permissionsToDo: + errors.append( + _deal_with_policies(role, account_number, config, hooks, scheduled, role_name, dynamo_table, commit, + continuing) + ) + + if "managed" in permissionsToDo: + errors.append( + _deal_with_managed_policies(role, account_number, config, hooks, scheduled, role_name, dynamo_table, commit, + continuing) + ) + return errors + + def _rollback_role( - account_number, role_name, dynamo_table, config, hooks, selection=None, commit=None + account_number, role_name, dynamo_table, config, hooks, selection=None, commit=None ): """ Display the historical policy versions for a roll as a numbered list. Restore to a specific version if selected. @@ -366,6 +492,7 @@ def _rollback_role( return errors +# Doesn't support managed policies yet def _repo_all_roles( account_number, dynamo_table, config, hooks, commit=False, scheduled=True, limit=-1 ): diff --git a/repokid/commands/role.py b/repokid/commands/role.py index ef0e12a1c..1f21394ed 100644 --- a/repokid/commands/role.py +++ b/repokid/commands/role.py @@ -47,15 +47,19 @@ def _display_roles(account_number, dynamo_table, inactive=False): Returns: None """ + headers = [ "Name", "Refreshed", "Disqualified By", "Can be repoed", "Permissions", - "Repoable", - "Repoed", + "Policies Repoable", "Services", + "Repoed", + "Managed Permissions", + "Managed Policies Repoable" + "Managed Services", ] rows = list() @@ -79,8 +83,11 @@ def _display_roles(account_number, dynamo_table, inactive=False): len(role.disqualified_by) == 0, role.total_permissions, role.repoable_permissions, - role.repoed, role.repoable_services, + role.repoed, + role.total_managed_permissions, + role.repoable_managed_permissions, + role.repoable_managed_services, ] ) @@ -166,10 +173,14 @@ def _display_role(account_number, role_name, dynamo_table, config, hooks): "Disqualified By", "Can be repoed", "Permissions", - "Repoable", - "Repoed", + "Policies Repoable", "Services", + "Repoed", + "Managed Permissions", + "Managed Policies Repoable" + "Managed Services", ] + rows = [ [ role.role_name, @@ -178,14 +189,17 @@ def _display_role(account_number, role_name, dynamo_table, config, hooks): len(role.disqualified_by) == 0, role.total_permissions, role.repoable_permissions, - role.repoed, role.repoable_services, + role.repoed, + role.total_managed_permissions, + role.repoable_managed_permissions, + role.repoable_managed_services, ] ] print(tabulate(rows, headers=headers) + "\n\n") print("Policy history:") - headers = ["Number", "Source", "Discovered", "Permissions", "Services"] + headers = ["Number", "Source", "Discovered", "Permissions", "Services", "",] rows = [] for index, policies_version in enumerate(role.policies): policy_permissions, _ = roledata._get_permissions_in_policy( diff --git a/repokid/commands/role_cache.py b/repokid/commands/role_cache.py index ca18ac9a7..9b17a916f 100644 --- a/repokid/commands/role_cache.py +++ b/repokid/commands/role_cache.py @@ -14,6 +14,7 @@ import logging from cloudaux.aws.iam import get_account_authorization_details +from cloudaux.aws.iam import get_managed_policy_document from repokid.filters import FilterPlugins from repokid.role import Role, Roles from repokid.utils import roledata as roledata @@ -52,12 +53,13 @@ def _update_role_cache(account_number, dynamo_table, config, hooks): conn["account_number"] = account_number LOGGER.info( - "Getting current role data for account {} (this may take a while for large accounts)".format( + "FUNKY MONKEY CHUNKY Getting current role data for account {} (this may take a while for large accounts)".format( account_number ) ) role_data = get_account_authorization_details(filter="Role", **conn) + role_data = role_data[:10] role_data_by_id = {item["RoleId"]: item for item in role_data} # convert policies list to dictionary to maintain consistency with old call which returned a dict @@ -66,6 +68,14 @@ def _update_role_cache(account_number, dynamo_table, config, hooks): item["PolicyName"]: item["PolicyDocument"] for item in data["RolePolicyList"] } + LOGGER.info("DONE INLINE POLICIES!") + # get managed policies in the same format as inline policies + for _, data in role_data_by_id.items(): + data["AttachedManagedPolicies"] = { + managed_policy["PolicyName"]: get_managed_policy_document(managed_policy["PolicyArn"]) + for managed_policy in data["AttachedManagedPolicies"] + } + LOGGER.info("DONE MANAGED POLICIES!") roles = Roles([Role(rd) for rd in role_data]) @@ -74,8 +84,11 @@ def _update_role_cache(account_number, dynamo_table, config, hooks): for role in tqdm(roles): role.account = account_number current_policies = role_data_by_id[role.role_id]["RolePolicyList"] + LOGGER.info(current_policies) + current_managed_policies = role_data_by_id[role.role_id]["AttachedManagedPolicies"] + LOGGER.info(current_managed_policies) active_roles.append(role.role_id) - roledata.update_role_data(dynamo_table, account_number, role, current_policies) + roledata.update_role_data(dynamo_table, account_number, role, current_policies, current_managed_policies) LOGGER.info("Finding inactive roles in account {}".format(account_number)) roledata.find_and_mark_inactive(dynamo_table, account_number, active_roles) @@ -156,6 +169,8 @@ def _update_role_cache(account_number, dynamo_table, config, hooks): account_number, role.repoable_permissions, role.repoable_services, + role.repoable_managed_permissions, + role.repoable_managed_services, ) ) set_role_data( @@ -165,8 +180,10 @@ def _update_role_cache(account_number, dynamo_table, config, hooks): "TotalPermissions": role.total_permissions, "RepoablePermissions": role.repoable_permissions, "RepoableServices": role.repoable_services, + "RepoableManagedPermissions": role.repoable_managed_permissions, + "RepoableManagedServices": role.repoable_managed_services, }, ) LOGGER.info("Updating stats in account {}".format(account_number)) - roledata.update_stats(dynamo_table, roles, source="Scan") + roledata.update_stats(dynamo_table, roles, source="Scan") # TODO diff --git a/repokid/role.py b/repokid/role.py index 3199199d7..c4651ffed 100644 --- a/repokid/role.py +++ b/repokid/role.py @@ -27,9 +27,12 @@ "NoRepoPermissions": {"attribute": "no_repo_permissions", "default": dict()}, "OptOut": {"attribute": "opt_out", "default": dict()}, "Policies": {"attribute": "policies", "default": list()}, + "ManagedPolicies": {"attribute": "managed_policies", "default": list()}, "Refreshed": {"attribute": "refreshed", "default": str()}, "RepoablePermissions": {"attribute": "repoable_permissions", "default": int()}, + "RepoableManagedPermissions": {"attribute": "repoable_managed_permissions", "default": int()}, "RepoableServices": {"attribute": "repoable_services", "default": list()}, + "RepoableManagedServices": {"attribute": "repoable_managed_services", "default": list()}, "Repoed": {"attribute": "repoed", "default": str()}, "RepoScheduled": {"attribute": "repo_scheduled", "default": int()}, "RoleId": {"attribute": "role_id", "default": None}, @@ -38,6 +41,7 @@ "Stats": {"attribute": "stats", "default": list()}, "Tags": {"attribute": "tags", "default": list()}, "TotalPermissions": {"attribute": "total_permissions", "default": int()}, + "TotalManagedPermissions": {"attribute": "total_managed_permissions", "default": int()}, } diff --git a/repokid/utils/dynamo.py b/repokid/utils/dynamo.py index 3cb2b2845..0d54ee1ff 100644 --- a/repokid/utils/dynamo.py +++ b/repokid/utils/dynamo.py @@ -285,6 +285,7 @@ def store_initial_role_data( role_name, account_number, current_policy, + current_managed_policy, tags, ): """ @@ -293,6 +294,7 @@ def store_initial_role_data( Args: role (Role) current_policy (dict) + current_managed_policy (dict) Returns: None @@ -303,6 +305,12 @@ def store_initial_role_data( "Policy": current_policy, } + managed_policy_entry = { + "Source": "Scan", + "Discovered": datetime.datetime.utcnow().isoformat(), + "Policy": current_managed_policy, + } + role_dict = { "Arn": arn, "CreateDate": create_date.isoformat(), @@ -310,6 +318,7 @@ def store_initial_role_data( "RoleName": role_name, "Account": account_number, "Policies": [policy_entry], + "ManagedPolicies": [managed_policy_entry], "Refreshed": datetime.datetime.utcnow().isoformat(), "Active": True, "Repoed": "Never", diff --git a/repokid/utils/roledata.py b/repokid/utils/roledata.py index 7f14f903f..fd5db1ab2 100644 --- a/repokid/utils/roledata.py +++ b/repokid/utils/roledata.py @@ -75,6 +75,30 @@ def add_new_policy_version(dynamo_table, role, current_policy, update_source): "Policies" ] +def add_new_managed_policy_version(dynamo_table, role, current_managed_policy, update_source): + """ + Create a new entry in the history of policy versions in Dynamo. The entry contains the source of the new policy: + (scan, repo, or restore) the current time, and the current policy contents. Updates the role's policies with the + full policies including the latest. + + Args: + role (Role) + current_managed_policy (dict) + update_source (string): ['Repo', 'Scan', 'Restore'] + + Returns: + None + """ + policy_entry = { + "Source": update_source, + "Discovered": datetime.datetime.utcnow().isoformat(), + "Policy": current_managed_policy, + } + + add_to_end_of_list(dynamo_table, role.role_id, "ManagedPolicies", policy_entry) + role.managed_policies = get_role_data(dynamo_table, role.role_id, fields=["ManagedPolicies"])[ + "ManagedPolicies" + ] def find_and_mark_inactive(dynamo_table, account_number, active_roles): """ @@ -111,6 +135,7 @@ def find_newly_added_permissions(old_policy, new_policy): Returns: set: Exapnded set of permissions that are in the new policy and not the old one """ + LOGGER.info(Role({"Policies": [{"Policy": old_policy}]})) old_permissions, _ = _get_role_permissions( Role({"Policies": [{"Policy": old_policy}]}) ) @@ -178,8 +203,8 @@ def update_opt_out(dynamo_table, role): def update_role_data( - dynamo_table, account_number, role, current_policy, source="Scan", add_no_repo=True -): + dynamo_table, account_number, role, current_policy, current_managed_policy, source="Scan", add_no_repo=True +, include_managed_policies=True): """ Compare the current version of a policy for a role and what has been previously stored in Dynamo. - If current and new policy versions are different store the new version in Dynamo. Add any newly added @@ -193,6 +218,7 @@ def update_role_data( account_number role (Role): current role being updated current_policy (dict): representation of the current policy version + current_managed_policy (dict): representation of the current managed policy versions source: Default 'Scan' but could be Repo, Rollback, etc Returns: @@ -201,7 +227,7 @@ def update_role_data( # policy_entry: source, discovered, policy stored_role = get_role_data( - dynamo_table, role.role_id, fields=["OptOut", "Policies", "Tags"] + dynamo_table, role.role_id, fields=["OptOut", "Policies", "ManagedPolicies", "Tags"] ) if not stored_role: role_dict = store_initial_role_data( @@ -212,6 +238,7 @@ def update_role_data( role.role_name, account_number, current_policy, + current_managed_policy, role.tags, ) role.set_attributes(role_dict) @@ -226,19 +253,39 @@ def update_role_data( role.arn ) ) - newly_added_permissions = find_newly_added_permissions( old_policy, current_policy ) + else: newly_added_permissions = set() + # TODO Make this part of set_role_data instead to allow updating existing dynamo tables + # TODO this code will not work with existing dynamo tables - because old roles won't have ManagedPolicies + old_managed_policy = stored_role["ManagedPolicies"][-1]["Policy"] + if current_managed_policy != old_managed_policy: + add_new_managed_policy_version(dynamo_table, role, current_managed_policy, source) + LOGGER.info( + "{} has different managed policies than last time, adding to role store".format( + role.arn + ) + ) + + newly_added_managed_permissions = find_newly_added_permissions( + old_managed_policy, current_managed_policy + ) + + else: + newly_added_managed_permissions = set() + # update tags if needed if role.tags != stored_role.get("Tags", []): set_role_data(dynamo_table, role.role_id, {"Tags": role.tags}) if add_no_repo: update_no_repo_permissions(dynamo_table, role, newly_added_permissions) + if include_managed_policies: + update_no_repo_permissions(dynamo_table, role, newly_added_managed_permissions) update_opt_out(dynamo_table, role) set_role_data( dynamo_table, @@ -286,17 +333,20 @@ def update_stats(dynamo_table, roles, source="Scan"): add_to_end_of_list(dynamo_table, role.role_id, "Stats", new_stats) -def _update_repoable_services(role, repoable_permissions, eligible_permissions): +def _update_repoable_services(role, repoable_permissions, eligible_permissions, managed_permissions=False): ( repoable_permissions_set, repoable_services_set, ) = _convert_repoable_perms_to_perms_and_services( eligible_permissions, repoable_permissions ) - - # we're going to store both repoable permissions and repoable services in the field "RepoableServices" - role.repoable_services = repoable_services_set + repoable_permissions_set - role.repoable_permissions = len(repoable_permissions) + if managed_permissions: + role.repoable_managed_services = repoable_services_set + repoable_permissions_set + role.repoable_managed_permissions = len(repoable_permissions) + else: + # we're going to store both repoable permissions and repoable services in the field "RepoableServices" + role.repoable_services = repoable_services_set + repoable_permissions_set + role.repoable_permissions = len(repoable_permissions) def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=100): @@ -319,30 +369,44 @@ def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=10 """ repo_able_roles = [] eligible_permissions_dict = {} + eligible_managed_permissions_dict = {} for role in roles: total_permissions, eligible_permissions = _get_role_permissions(role) - role.total_permissions = len(total_permissions) + total_managed_permissions, eligible_managed_permissions = _get_role_managed_permissions(role) + role.total_permissions = len(total_permissions) + role.total_managed_permissions = len(total_managed_permissions) + LOGGER.info("managed permissions are: HOLA:\n") + LOGGER.info(role.total_managed_permissions) # if we don't have any access advisor data for a service than nothing is repoable if not role.aa_data: LOGGER.info("No data found in access advisor for {}".format(role.role_id)) role.repoable_permissions = 0 role.repoable_services = [] + role.repoable_managed_permissions = 0 + role.repoable_managed_services = [] continue # permissions are only repoable if the role isn't being disqualified by filter(s) if len(role.disqualified_by) == 0: repo_able_roles.append(role) eligible_permissions_dict[role.arn] = eligible_permissions + eligible_managed_permissions_dict[role.arn] = eligible_managed_permissions else: role.repoable_permissions = 0 role.repoable_services = [] + role.repoable_managed_permissions = 0 + role.repoable_managed_services = [] repoable_permissions_dict = {} + repoable_managed_permissions_dict = {} if batch: repoable_permissions_dict = _get_repoable_permissions_batch( repo_able_roles, eligible_permissions_dict, minimum_age, hooks, batch_size ) + repoable_managed_permissions_dict = _get_repoable_permissions_batch( + repo_able_roles, eligible_managed_permissions_dict, minimum_age, hooks, batch_size + ) else: for role in repo_able_roles: repoable_permissions_dict[role.arn] = _get_repoable_permissions( @@ -354,12 +418,26 @@ def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=10 minimum_age, hooks, ) + repoable_managed_permissions_dict[role.arn] = _get_repoable_permissions( + role.account, + role.role_name, + eligible_managed_permissions_dict[role.arn], + role.aa_data, + role.no_repo_permissions, + minimum_age, + hooks, + ) for role in repo_able_roles: eligible_permissions = eligible_permissions_dict[role.arn] repoable_permissions = repoable_permissions_dict[role.arn] _update_repoable_services(role, repoable_permissions, eligible_permissions) + eligible_managed_permissions = eligible_managed_permissions_dict[role.arn] + repoable_managed_permissions = repoable_managed_permissions_dict[role.arn] + _update_repoable_services(role, eligible_managed_permissions, repoable_managed_permissions, + managed_permissions=True) + def _convert_repoable_perms_to_perms_and_services( total_permissions, repoable_permissions @@ -772,6 +850,76 @@ def _get_repoed_policy(policies, repoable_permissions): return role_policies, empty_policies +def _get_repoed_managed_policy(managed_policies, repoable_managed_permissions): + """ + This function contains the logic to rewrite the policy to remove any repoable permissions. To do so we: + - Iterate over role policies + - Iterate over policy statements + - Skip Deny statements + - Remove any actions that are in repoable_permissions + - Remove any statements that now have zero actions + - Remove any policies that now have zero statements + + Args: + managed_policies (dict): All of the inline policies as a dict with name and policy contents + repoable_managed_permissions (set): A set of all of the repoable permissions for policies + + Returns: + dict: The rewritten set of all inline policies + list: Any policies that are now empty as a result of the rewrites + """ + # work with our own copy; don't mess with the CACHE copy. + role_managed_policies = copy.deepcopy(managed_policies) + + empty_policies = [] + for policy_name, policy in list(role_managed_policies.items()): + # list of indexes in the policy that are empty + empty_statements = [] + + if type(policy["Statement"]) is dict: + policy["Statement"] = [policy["Statement"]] + + for idx, statement in enumerate(policy["Statement"]): + if statement["Effect"].lower() == "allow": + if "Sid" in statement and statement["Sid"].startswith( + STATEMENT_SKIP_SID + ): + continue + + statement_actions = get_actions_from_statement(statement) + + if not statement_actions.intersection(repoable_managed_permissions): + # No permissions are being taken away; let's not modify this statement at all. + continue + + statement_actions = statement_actions.difference(repoable_managed_permissions) + + # get_actions_from_statement has already inverted this so our new statement should be 'Action' + if "NotAction" in statement: + del statement["NotAction"] + + # by putting this into a set, we lose order, which may be confusing to someone. + statement["Action"] = sorted(list(statement_actions)) + + # mark empty statements to be removed + if len(statement["Action"]) == 0: + empty_statements.append(idx) + + # do the actual removal of empty statements + for idx in sorted(empty_statements, reverse=True): + del policy["Statement"][idx] + + # mark empty policies to be removed + if len(policy["Statement"]) == 0: + empty_policies.append(policy_name) + + # do the actual removal of empty policies. + for policy_name in empty_policies: + del role_managed_policies[policy_name] # detach any managed policies that can be deleted. + + return role_managed_policies, empty_policies + + def _get_permissions_in_policy(policy_dict, warn_unknown_perms=False): """ Given a set of policies for a role, return a set of all allowed permissions @@ -831,6 +979,25 @@ def _get_role_permissions(role, warn_unknown_perms=False): return _get_permissions_in_policy(role.policies[-1]["Policy"]) +def _get_role_managed_permissions(role, warn_unknown_perms=False): + """ + Expand the most recent version of policies from a role to produce a list of all the permissions that are allowed + (permission is included in one or more statements that is allowed). To perform expansion the policyuniverse + library is used. The result is a list of all of the individual permissions that are allowed in any of the + statements. If our resultant list contains any permissions that aren't listed in the master list of permissions + we'll raise an exception with the set of unknown permissions found. + + Args: + role (Role): The role object that we're getting a list of permissions for + + Returns: + tuple + set - all managed permissions allowed by the policies + set - all managed permisisons allowed by the policies not marked with STATEMENT_SKIP_SID + """ + return _get_permissions_in_policy(role.managed_policies[-1]["Policy"]) + + def _get_services_in_permissions(permissions_set): """ Given a set of permissions, return a sorted set of services @@ -917,4 +1084,4 @@ def partial_update_role_data( "RepoableServices": role.repoable_services, }, ) - update_stats(dynamo_table, [role], source=source) + update_stats(dynamo_table, [role], source=source) # TODO update