Merge f6707c6 into 2583bfe

.gitignore

@@ -16,3 +16,4 @@ tox.ini
 test-reports/*
 config.json
 .eggs/
+dynamodb-data/

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_language_version:
     python: python3
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.3.0  # Use the ref you want to point at
+    rev: v3.4.0  # Use the ref you want to point at
     hooks:
     -   id: trailing-whitespace
     -   id: check-ast
@@ -22,11 +22,11 @@ repos:
     -   id: flake8
         args: ["--exclude", "venv/,.tox/,.eggs/"]
 
--   repo: https://github.com/pycqa/isort
-    rev: 5.6.4
+-   repo: https://github.com/pre-commit/mirrors-isort
+    rev: 'v5.7.0'
     hooks:
       - id: isort
-        args: ["--sl"]
+        args: ["--sl", "--profile", "black"]
 
 -   repo: https://github.com/ambv/black
     rev: 20.8b1

.travis.yml

@@ -11,7 +11,7 @@ install:
 - pip install -r requirements.txt -r requirements-test.txt
 - pip install .
 script:
-- pre-commit run
+- pre-commit run -a
 - coverage run --source repokid -m py.test
 - bandit -r . -ll -ii -x repokid/tests/
 after_success:

README.md

@@ -35,7 +35,12 @@ The table should have the following properties:
 For development, you can run dynamo [locally](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html).
 
 To run locally:
-  `java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb -inMemory -port 8010`
+
+```bash
+docker-compose up
+```
+
+The endpoint for DynamoDB will be `http://localhost:8000`. A DynamoDB admin panel can be found at `http://localhost:8001`.
 
 If you run the development version the table and index will be created for you automatically.
 

repokid/cli/repokid_cli.py

@@ -34,7 +34,6 @@
 from repokid.commands.schedule import _schedule_repo
 from repokid.commands.schedule import _show_scheduled_roles
 from repokid.types import RepokidConfig
-from repokid.utils.dynamo import dynamo_get_or_create_table
 
 logger = logging.getLogger("repokid")
 
@@ -84,7 +83,7 @@ def _generate_default_config(filename: str = "") -> RepokidConfig:
         "dynamo_db": {
             "assume_role": "RepokidRole",
             "account_number": "<DYNAMO_TABLE_ACCOUNT_NUMBER>",
-            "endpoint": "<DYNAMO_TABLE_ENDPOINT>",
+            "endpoint": "<DYNAMO_TABLE_ENDPOINT (http://localhost:8000 if using docker compose)>",
             "region": "<DYNAMO_TABLE_REGION>",
             "session_name": "repokid",
         },
@@ -182,7 +181,6 @@ def cli(ctx: click.Context) -> None:
 
     ctx.obj["config"] = config
     ctx.obj["hooks"] = get_hooks(config.get("hooks", ["repokid.hooks.loggers"]))
-    ctx.obj["dynamo_table"] = dynamo_get_or_create_table(**config["dynamo_db"])
 
 
 @cli.command()
@@ -203,7 +201,7 @@ def update_role_cache(ctx: click.Context, account_number: str) -> None:
 
 @cli.command()
 @click.argument("account_number")
-@click.option("--inactive", default=False, help="Include inactive roles")
+@click.option("--inactive", is_flag=True, default=False, help="Include inactive roles")
 @click.pass_context
 def display_role_cache(ctx: click.Context, account_number: str, inactive: bool) -> None:
     _display_roles(account_number, inactive=inactive)
@@ -222,7 +220,7 @@ def find_roles_with_permissions(
 @cli.command()
 @click.argument("permissions", nargs=-1)
 @click.option("--role-file", "-f", required=True, help="File to read roles from")
-@click.option("--commit", "-c", default=False, help="Commit changes")
+@click.option("--commit", "-c", is_flag=True, default=False, help="Commit changes")
 @click.pass_context
 def remove_permissions_from_roles(
     ctx: click.Context, permissions: List[str], role_file: str, commit: bool
@@ -244,7 +242,7 @@ def display_role(ctx: click.Context, account_number: str, role_name: str) -> Non
 @cli.command()
 @click.argument("account_number")
 @click.argument("role_name")
-@click.option("--commit", "-c", default=False, help="Commit changes")
+@click.option("--commit", "-c", is_flag=True, default=False, help="Commit changes")
 @click.pass_context
 def repo_role(
     ctx: click.Context, account_number: str, role_name: str, commit: bool
@@ -258,7 +256,7 @@ def repo_role(
 @click.argument("account_number")
 @click.argument("role_name")
 @click.option("--selection", "-s", required=True, type=int)
-@click.option("--commit", "-c", default=False, help="Commit changes")
+@click.option("--commit", "-c", is_flag=True, default=False, help="Commit changes")
 @click.pass_context
 def rollback_role(
     ctx: click.Context,
@@ -276,7 +274,7 @@ def rollback_role(
 
 @cli.command()
 @click.argument("account_number")
-@click.option("--commit", "-c", default=False, help="Commit changes")
+@click.option("--commit", "-c", is_flag=True, default=False, help="Commit changes")
 @click.pass_context
 def repo_all_roles(ctx: click.Context, account_number: str, commit: bool) -> None:
     config = ctx.obj["config"]
@@ -307,7 +305,7 @@ def show_scheduled_roles(ctx: click.Context, account_number: str) -> None:
 @cli.command()
 @click.argument("account_number")
 @click.option("--role", "-r", required=False, type=str)
-@click.option("--all", "-a", default=False, help="Commit changes")
+@click.option("--all", "-a", is_flag=True, default=False, help="cancel for all roles")
 @click.pass_context
 def cancel_scheduled_repo(
     ctx: click.Context, account_number: str, role: str, all: bool
@@ -317,7 +315,7 @@ def cancel_scheduled_repo(
 
 @cli.command()
 @click.argument("account_number")
-@click.option("--commit", "-c", default=False, help="Commit changes")
+@click.option("--commit", "-c", is_flag=True, default=False, help="Commit changes")
 @click.pass_context
 def repo_scheduled_roles(ctx: click.Context, account_number: str, commit: bool) -> None:
     config = ctx.obj["config"]

repokid/commands/repo.py

@@ -73,8 +73,10 @@ def _repo_role(
 
     continuing = True
 
-    if not role.is_eligible_for_repo():
-        continuing = False
+    eligible, reason = role.is_eligible_for_repo()
+    if not eligible:
+        errors.append(f"Role {role_name} not eligible for repo: {reason}")
+        return errors
 
     role.calculate_repo_scores(
         config["filter_config"]["AgeFilter"]["minimum_age"], hooks
@@ -120,7 +122,7 @@ def _repo_role(
             LOGGER.error(e)
             errors.append(str(e))
 
-    current_policies = get_role_inline_policies(role.dict(), **conn) or {}
+    current_policies = get_role_inline_policies(role.dict(by_alias=True), **conn) or {}
     role.add_policy_version(current_policies, source="Repo")
 
     # regardless of whether we're successful we want to unschedule the repo
@@ -148,11 +150,11 @@ def _rollback_role(
     role_name: str,
     config: RepokidConfig,
     hooks: RepokidHooks,
-    selection: int = 0,
+    selection: int = -1,
     commit: bool = False,
 ) -> List[str]:
     """
-    Display the historical policy versions for a roll as a numbered list.  Restore to a specific version if selected.
+    Display the historical policy versions for a role as a numbered list.  Restore to a specific version if selected.
     Indicate changes that will be made and then actually make them if commit is selected.
 
     Args:
@@ -179,7 +181,7 @@ def _rollback_role(
         role.fetch()
 
     # no option selected, display a table of options
-    if not selection:
+    if selection < 0:
         headers = ["Number", "Source", "Discovered", "Permissions", "Services"]
         rows = []
         for index, policies_version in enumerate(role.policies):
@@ -201,25 +203,24 @@ def _rollback_role(
     conn = config["connection_iam"]
     conn["account_number"] = account_number
 
-    current_policies = get_role_inline_policies(role.dict(), **conn)
+    current_policies = get_role_inline_policies(role.dict(by_alias=True), **conn)
 
-    if selection:
-        pp = pprint.PrettyPrinter()
+    pp = pprint.PrettyPrinter()
 
-        print("Will restore the following policies:")
-        pp.pprint(role.policies[int(selection)]["Policy"])
+    print("Will restore the following policies:")
+    pp.pprint(role.policies[int(selection)]["Policy"])
 
-        print("Current policies:")
-        pp.pprint(current_policies)
+    print("Current policies:")
+    pp.pprint(current_policies)
 
-        current_permissions, _ = role.get_permissions_for_policy_version()
-        selected_permissions, _ = role.get_permissions_for_policy_version(
-            selection=selection
-        )
-        restored_permissions = selected_permissions - current_permissions
+    current_permissions, _ = role.get_permissions_for_policy_version()
+    selected_permissions, _ = role.get_permissions_for_policy_version(
+        selection=selection
+    )
+    restored_permissions = selected_permissions - current_permissions
 
-        print("\nResore will return these permissions:")
-        print("\n".join([perm for perm in sorted(restored_permissions)]))
+    print("\nResore will return these permissions:")
+    print("\n".join([perm for perm in sorted(restored_permissions)]))
 
     if not commit:
         return errors
@@ -275,6 +276,7 @@ def _rollback_role(
                 LOGGER.error(message, exc_info=True)
                 errors.append(message)
 
+    role.store()
     role.gather_role_data(current_policies, hooks, source="Restore", add_no_repo=False)
 
     if not errors:

repokid/commands/role.py

@@ -23,6 +23,7 @@
 from tqdm import tqdm
 
 import repokid.hooks
+from repokid.exceptions import MissingRepoableServices
 from repokid.role import Role
 from repokid.role import RoleList
 from repokid.types import RepokidConfig
@@ -235,22 +236,26 @@ def _display_role(
     for permission in permissions:
         service = permission.split(":")[0]
         action = permission.split(":")[1]
-        repoable = permission in role.repoable_services
+        is_repoable_permission = permission in role.repoable_services
+        is_repoable_service = permission.split(":")[0] in role.repoable_services
+        # repoable is is True if the action (`permission`) is in the list of repoable
+        # services OR if the service (`permission.split(":")[0]`) is in the list
+        repoable = is_repoable_permission or is_repoable_service
         rows.append([service, action, repoable])
 
     rows = sorted(rows, key=lambda x: (x[2], x[0], x[1]))
     print(tabulate(rows, headers=headers) + "\n\n")
 
-    repoed_policies, _ = role.get_repoed_policy()
-
-    if repoed_policies:
+    try:
+        repoed_policies, _ = role.get_repoed_policy()
         print(
             "Repo'd Policies: \n{}".format(
                 json.dumps(repoed_policies, indent=2, sort_keys=True)
             )
         )
-    else:
+    except MissingRepoableServices:
         print("All Policies Removed")
+        repoed_policies = {}
 
     # need to check if all policies would be too large
     if inline_policies_size_exceeds_maximum(repoed_policies):

repokid/commands/role_cache.py

@@ -13,9 +13,10 @@
 #     limitations under the License.
 import logging
 
-from cloudaux.aws.iam import get_account_authorization_details
 from tqdm import tqdm
 
+from repokid.datasource.access_advisor import AccessAdvisorDatasource
+from repokid.datasource.iam import IAMDatasource
 from repokid.filters.utils import get_filter_plugins
 from repokid.role import Role
 from repokid.role import RoleList
@@ -54,31 +55,17 @@ def _update_role_cache(
     Returns:
         None
     """
-    conn = config["connection_iam"]
-    conn["account_number"] = account_number
+    access_advisor_datasource = AccessAdvisorDatasource()
+    access_advisor_datasource.seed(account_number)
+    iam_datasource = IAMDatasource()
+    iam_datasource.seed(account_number)
 
-    LOGGER.info(
-        "Getting current role data for account {} (this may take a while for large accounts)".format(
-            account_number
-        )
-    )
-
-    role_data = get_account_authorization_details(filter="Role", **conn)
-    role_data_by_id = {item["RoleId"]: item for item in role_data}
-
-    # convert policies list to dictionary to maintain consistency with old call which returned a dict
-    for _, data in role_data_by_id.items():
-        data["RolePolicyList"] = {
-            item["PolicyName"]: item["PolicyDocument"]
-            for item in data["RolePolicyList"]
-        }
-
-    roles = RoleList([Role(**rd) for rd in role_data])
+    roles = RoleList([Role(**rd) for rd in iam_datasource.values()])
 
     LOGGER.info("Updating role data for account {}".format(account_number))
     for role in tqdm(roles):
         role.account = account_number
-        current_policies = role_data_by_id[role.role_id]["RolePolicyList"]
+        current_policies = iam_datasource[role.role_id].get("RolePolicyList", {})
         role.gather_role_data(
             current_policies, hooks, config, source="Scan", store=False
         )

repokid/commands/schedule.py

@@ -147,7 +147,7 @@ def _cancel_scheduled_repo(
     role.fetch()
 
     if not role.repo_scheduled:
-        LOGGER.warn(
+        LOGGER.warning(
             "Repo was not scheduled for role {} in account {}".format(
                 role.role_name, account_number
             )

repokid/datasource/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2021 Netflix, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.