Merge 9e380e9 into 3cf6d83

.gitignore

@@ -1,4 +1,5 @@
 .idea/
+.run/
 venv/
 *.pyc
 *.json

repokid/commands/repo.py

@@ -12,7 +12,6 @@
 #     See the License for the specific language governing permissions and
 #     limitations under the License.
 import csv
-import datetime
 import json
 import logging
 import pprint
@@ -28,19 +27,12 @@
 import repokid.hooks
 from repokid.datasource.access_advisor import AccessAdvisorDatasource
 from repokid.datasource.iam import IAMDatasource
-from repokid.exceptions import IAMError
-from repokid.exceptions import MissingRepoableServices
 from repokid.role import Role
 from repokid.role import RoleList
 from repokid.types import RepokidConfig
 from repokid.types import RepokidHooks
 from repokid.utils.dynamo import find_role_in_cache
 from repokid.utils.dynamo import role_ids_for_all_accounts
-from repokid.utils.iam import delete_policy
-from repokid.utils.iam import inline_policies_size_exceeds_maximum
-from repokid.utils.iam import replace_policies
-from repokid.utils.iam import update_repoed_description
-from repokid.utils.logging import log_deleted_and_repoed_policies
 from repokid.utils.permissions import get_services_in_permissions
 
 LOGGER = logging.getLogger("repokid")
@@ -67,91 +59,11 @@ def _repo_role(
     Returns:
         None
     """
-    errors: List[str] = []
-
     role_id = find_role_in_cache(role_name, account_number)
     # only load partial data that we need to determine if we should keep going
-    role = Role(role_id=role_id)
+    role = Role(role_id=role_id, config=config)
     role.fetch()
-
-    continuing = True
-
-    eligible, reason = role.is_eligible_for_repo()
-    if not eligible:
-        errors.append(f"Role {role_name} not eligible for repo: {reason}")
-        return errors
-
-    role.calculate_repo_scores(
-        config["filter_config"]["AgeFilter"]["minimum_age"], hooks
-    )
-    try:
-        repoed_policies, deleted_policy_names = role.get_repoed_policy(
-            scheduled=scheduled
-        )
-    except MissingRepoableServices as e:
-        errors.append(f"Role {role_name} cannot be repoed: {e}")
-        return errors
-
-    if inline_policies_size_exceeds_maximum(repoed_policies):
-        error = (
-            "Policies would exceed the AWS size limit after repo for role: {} in account {}.  "
-            "Please manually minify.".format(role_name, account_number)
-        )
-        LOGGER.error(error)
-        errors.append(error)
-        continuing = False
-
-    # if we aren't repoing for some reason, unschedule the role
-    if not continuing:
-        role.repo_scheduled = 0
-        role.scheduled_perms = []
-        role.store(["repo_scheduled", "scheduled_perms"])
-        return errors
-
-    if not commit:
-        log_deleted_and_repoed_policies(
-            deleted_policy_names, repoed_policies, role_name, account_number
-        )
-        return errors
-
-    conn = config["connection_iam"]
-    conn["account_number"] = account_number
-
-    for name in deleted_policy_names:
-        try:
-            delete_policy(name, role, account_number, conn)
-        except IAMError as e:
-            LOGGER.error(e)
-            errors.append(str(e))
-
-    if repoed_policies:
-        try:
-            replace_policies(repoed_policies, role, account_number, conn)
-        except IAMError as e:
-            LOGGER.error(e)
-            errors.append(str(e))
-
-    current_policies = get_role_inline_policies(role.dict(by_alias=True), **conn) or {}
-    role.add_policy_version(current_policies, source="Repo")
-
-    # regardless of whether we're successful we want to unschedule the repo
-    role.repo_scheduled = 0
-    role.scheduled_perms = []
-
-    repokid.hooks.call_hooks(hooks, "AFTER_REPO", {"role": role, "errors": errors})
-
-    if not errors:
-        # repos will stay scheduled until they are successful
-        role.repoed = datetime.datetime.now(tz=datetime.timezone.utc).isoformat()
-        update_repoed_description(role.role_name, conn)
-        role.gather_role_data(current_policies, hooks, source="Repo", add_no_repo=False)
-        LOGGER.info(
-            "Successfully repoed role: {} in account {}".format(
-                role.role_name, account_number
-            )
-        )
-    role.store()
-    return errors
+    return role.repo(hooks, commit=commit, scheduled=scheduled)
 
 
 def _rollback_role(
@@ -286,7 +198,9 @@ def _rollback_role(
                 errors.append(message)
 
     role.store()
-    role.gather_role_data(current_policies, hooks, source="Restore", add_no_repo=False)
+    role.gather_role_data(
+        hooks, current_policies=current_policies, source="Restore", add_no_repo=False
+    )
 
     if not errors:
         LOGGER.info(
@@ -324,14 +238,10 @@ def _repo_all_roles(
     role_ids = iam_datasource.seed(account_number)
     errors = []
 
-    roles = RoleList.from_ids(role_ids)
-    roles.fetch_all(fetch_aa_data=True)
-
+    roles = RoleList.from_ids(role_ids, config=config)
     roles = roles.get_active()
-
     if scheduled:
         roles = roles.get_scheduled()
-
     if not roles:
         LOGGER.info(f"No roles to repo in account {account_number}")
         return
@@ -353,23 +263,15 @@ def _repo_all_roles(
     for role in roles:
         if limit >= 0 and count == limit:
             break
-        error = _repo_role(
-            account_number,
-            role.role_name,
-            config,
-            hooks,
-            commit=commit,
-            scheduled=scheduled,
-        )
-        if error:
-            errors.append(error)
+        role_errors = role.repo(hooks, commit=commit, scheduled=scheduled)
+        if role_errors:
+            errors.extend(role_errors)
         repoed.append(role)
         count += 1
 
     if errors:
-        LOGGER.error(f"Error(s) during repo: {errors} (account: {account_number})")
-    else:
-        LOGGER.info(f"Successfully repoed {count} roles in account {account_number}")
+        LOGGER.error(f"Error(s) during repo in account: {account_number}: {errors}")
+    LOGGER.info(f"Successfully repoed {count} roles in account {account_number}")
 
     repokid.hooks.call_hooks(
         hooks,

repokid/commands/role.py

@@ -16,6 +16,7 @@
 import logging
 from typing import Any
 from typing import List
+from typing import Optional
 
 import tabview as t
 from policyuniverse.arn import ARN
@@ -32,7 +33,6 @@
 from repokid.utils.dynamo import get_all_role_ids_for_account
 from repokid.utils.dynamo import role_ids_for_all_accounts
 from repokid.utils.iam import inline_policies_size_exceeds_maximum
-from repokid.utils.iam import remove_permissions_from_role
 from repokid.utils.permissions import get_permissions_in_policy
 from repokid.utils.permissions import get_services_in_permissions
 
@@ -268,7 +268,7 @@ def _display_role(
 def _remove_permissions_from_roles(
     permissions: List[str],
     role_filename: str,
-    config: RepokidConfig,
+    config: Optional[RepokidConfig],
     hooks: RepokidHooks,
     commit: bool = False,
 ) -> None:
@@ -295,11 +295,9 @@ def _remove_permissions_from_roles(
         role_name = arn.name.split("/")[-1]
 
         role_id = find_role_in_cache(role_name, account_number)
-        role = Role(role_id=role_id)
+        role = Role(role_id=role_id, config=config)
         role.fetch()
 
-        remove_permissions_from_role(
-            account_number, permissions, role, config, hooks, commit=commit
-        )
+        role.remove_permissions(permissions, hooks, commit=commit)
 
         repokid.hooks.call_hooks(hooks, "AFTER_REPO", {"role": role})

repokid/commands/role_cache.py

@@ -66,10 +66,7 @@ def _update_role_cache(
     LOGGER.info("Updating role data for account {}".format(account_number))
     for role in tqdm(roles):
         role.account = account_number
-        current_policies = iam_datasource[role.role_id].get("RolePolicyList", {})
-        role.gather_role_data(
-            current_policies, hooks, config, source="Scan", store=False
-        )
+        role.gather_role_data(hooks, config=config, source="Scan", store=False)
 
     LOGGER.info("Finding inactive roles in account {}".format(account_number))
     find_and_mark_inactive(account_number, roles)

repokid/datasource/access_advisor.py

@@ -15,7 +15,7 @@
 import logging
 from typing import Any
 from typing import Dict
-from typing import KeysView
+from typing import Iterable
 from typing import Optional
 
 import requests
@@ -31,7 +31,9 @@
 logger = logging.getLogger("repokid")
 
 
-class AccessAdvisorDatasource(DatasourcePlugin[str, AccessAdvisorEntry], Singleton):
+class AccessAdvisorDatasource(
+    DatasourcePlugin[str, AccessAdvisorEntry], metaclass=Singleton
+):
     def __init__(self, config: Optional[RepokidConfig] = None):
         super().__init__(config=config)
 
@@ -102,7 +104,14 @@ def get(self, arn: str) -> AccessAdvisorEntry:
             return result
         raise NotFoundError
 
-    def seed(self, account_number: str) -> KeysView[str]:
-        aa_data = self._fetch(account_number=account_number)
-        self._data.update(aa_data)
-        return aa_data.keys()
+    def _get_arns_for_account(self, account_number: str) -> Iterable[str]:
+        return filter(lambda x: x.split(":")[4] == account_number, self.keys())
+
+    def seed(self, account_number: str) -> Iterable[str]:
+        if account_number not in self._seeded:
+            aa_data = self._fetch(account_number=account_number)
+            self._data.update(aa_data)
+            self._seeded.append(account_number)
+            return aa_data.keys()
+        else:
+            return self._get_arns_for_account(account_number)

repokid/datasource/iam.py

@@ -14,11 +14,13 @@
 
 import copy
 import logging
+from typing import Any
 from typing import Dict
-from typing import KeysView
+from typing import Iterable
 from typing import Optional
 
 from cloudaux.aws.iam import get_account_authorization_details
+from cloudaux.aws.iam import get_role_inline_policies
 
 from repokid.datasource.plugin import DatasourcePlugin
 from repokid.exceptions import NotFoundError
@@ -29,16 +31,19 @@
 logger = logging.getLogger("repokid")
 
 
-class IAMDatasource(DatasourcePlugin[str, IAMEntry], Singleton):
+class IAMDatasource(DatasourcePlugin[str, IAMEntry], metaclass=Singleton):
+    _arn_to_id: Dict[str, str] = {}
+
     def __init__(self, config: Optional[RepokidConfig] = None):
         super().__init__(config=config)
 
-    def _fetch(self, account_number: str) -> Dict[str, IAMEntry]:
+    def _fetch_account(self, account_number: str) -> Dict[str, IAMEntry]:
         logger.info("getting role data for account %s", account_number)
-        conn = copy.deepcopy(self.config["connection_iam"])
+        conn = copy.deepcopy(self.config.get("connection_iam", {}))
         conn["account_number"] = account_number
         auth_details = get_account_authorization_details(filter="Role", **conn)
         auth_details_by_id = {item["RoleId"]: item for item in auth_details}
+        self._arn_to_id.update({item["Arn"]: item["RoleId"] for item in auth_details})
         # convert policies list to dictionary to maintain consistency with old call which returned a dict
         for _, data in auth_details_by_id.items():
             data["RolePolicyList"] = {
@@ -47,26 +52,46 @@ def _fetch(self, account_number: str) -> Dict[str, IAMEntry]:
             }
         return auth_details_by_id
 
-    def get(self, arn: str) -> IAMEntry:
-        result = self._data.get(arn)
+    def _fetch(self, arn: str) -> IAMEntry:
+        # TODO: sort out arn vs role_id here
+        # we probably only have role_id, which isn't sufficient for this implementation
+        logger.info("getting role data for role %s", arn)
+        conn = copy.deepcopy(self.config["connection_iam"])
+        conn["account_number"] = arn.split(":")[4]
+        role = {"RoleName": arn.split("/")[-1]}
+        role_policies: Dict[str, Any] = get_role_inline_policies(role, **conn)
+        if not role_policies:
+            raise NotFoundError
+        self._data[arn] = role_policies
+        return role_policies
+
+    def get(self, role_id: str) -> IAMEntry:
+        result = self._data.get(role_id)
         if not result:
+            # TODO: call _fetch()
             raise NotFoundError
         return result
 
-    def seed(self, account_number: str) -> KeysView[str]:
-        fetched_data = self._fetch(account_number)
+    def _get_arns_for_account(self, account_number: str) -> Iterable[str]:
+        return filter(lambda x: x.split(":")[4] == account_number, self.keys())
+
+    def seed(self, account_number: str) -> Iterable[str]:
+        if account_number in self._seeded:
+            return self._get_arns_for_account(account_number)
+        fetched_data = self._fetch_account(account_number)
         new_keys = fetched_data.keys()
         self._data.update(fetched_data)
+        self._seeded.append(account_number)
         return new_keys
 
 
 # TODO: Implement retrieval of IAM data from AWS Config
-class ConfigDatasource(DatasourcePlugin[str, IAMEntry], Singleton):
+class ConfigDatasource(DatasourcePlugin[str, IAMEntry], metaclass=Singleton):
     def __init__(self, config: Optional[RepokidConfig] = None):
         super().__init__(config=config)
 
     def get(self, identifier: str) -> IAMEntry:
         pass
 
-    def seed(self, identifier: str) -> KeysView[str]:
+    def seed(self, identifier: str) -> Iterable[str]:
         pass