Change to get-act-auth-details

The iam call get-account-authorization-details returns all of the
information we need but is much much faster than individually
enumerating all policies on a role-by-role basis. Switch to using
it instead.

README.md

@@ -62,6 +62,7 @@ RepokidRole:
         "iam:DeleteInstanceProfile",
         "iam:DeleteRole",
         "iam:DeleteRolePolicy",
+        "iam:GetAccountAuthorizationDetails",
         "iam:GetInstanceProfile",
         "iam:GetRole",
         "iam:GetRolePolicy",

repokid/__init__.py

@@ -19,7 +19,7 @@
 
 import import_string
 
-__version__ = '0.7.18'
+__version__ = '0.8.0'
 
 
 def init_config():

repokid/cli/repokid_cli.py

@@ -46,7 +46,7 @@
 
 import botocore
 from cloudaux import CloudAux
-from cloudaux.aws.iam import list_roles, get_role_inline_policies
+from cloudaux.aws.iam import get_account_authorization_details, get_role_inline_policies
 from cloudaux.aws.sts import sts_conn
 from docopt import docopt
 import import_string
@@ -367,13 +367,21 @@ def update_role_cache(account_number, dynamo_table, config, hooks):
     conn = config['connection_iam']
     conn['account_number'] = account_number
 
-    roles = Roles([Role(role_data) for role_data in list_roles(**conn)])
+    LOGGER.info('Getting current role data for account {} (this may take a while for large accounts)'.format(account_number))
+    role_data = get_account_authorization_details(filter='Role', **conn)
+    role_data_by_id = {item['RoleId']: item for item in role_data}
+
+    # convert policies list to dictionary to maintain consistency with old call which returned a dict
+    for _, data in role_data_by_id.items():
+        data['RolePolicyList'] = {item['PolicyName']: item['PolicyDocument'] for item in data['RolePolicyList']}
+
+    roles = Roles([Role(role_data) for role_data in role_data])
 
     active_roles = []
     LOGGER.info('Updating role data for account {}'.format(account_number))
     for role in tqdm(roles):
         role.account = account_number
-        current_policies = get_role_inline_policies(role.as_dict(), **conn) or {}
+        current_policies = role_data_by_id[role.role_id]['RolePolicyList']
         active_roles.append(role.role_id)
         roledata.update_role_data(dynamo_table, account_number, role, current_policies)
 

repokid/tests/test_repokid_cli.py

@@ -144,24 +144,23 @@ class TestRepokidCLI(object):
     @patch('repokid.utils.roledata._calculate_repo_scores')
     @patch('repokid.cli.repokid_cli.set_role_data')
     @patch('repokid.cli.repokid_cli._get_aardvark_data')
-    @patch('repokid.cli.repokid_cli.get_role_inline_policies')
-    @patch('repokid.cli.repokid_cli.list_roles')
-    def test_repokid_update_role_cache(self, mock_list_roles, mock_get_role_inline_policies,
-                                       mock_get_aardvark_data, mock_set_role_data, mock_calculate_repo_scores,
-                                       mock_update_role_data, mock_find_and_mark_inactive, mock_update_stats):
+    @patch('repokid.cli.repokid_cli.get_account_authorization_details')
+    def test_repokid_update_role_cache(self, mock_get_account_authorization_details, mock_get_aardvark_data,
+                                       mock_set_role_data, mock_calculate_repo_scores, mock_update_role_data,
+                                       mock_find_and_mark_inactive, mock_update_stats):
 
         hooks = {}
-        # only active roles
-        mock_list_roles.return_value = ROLES[:3]
-
-        mock_get_role_inline_policies.side_effect = [ROLE_POLICIES['all_services_used'],
-                                                     ROLE_POLICIES['unused_ec2'],
-                                                     ROLE_POLICIES['all_services_used']]
+
+        role_data = ROLES[:3]
+        role_data[0]['RolePolicyList'] = [{'PolicyName': 'all_services_used', 'PolicyDocument': ROLE_POLICIES['all_services_used'] }]
+        role_data[1]['RolePolicyList'] = [{'PolicyName': 'unused_ec2', 'PolicyDocument': ROLE_POLICIES['unused_ec2'] }]
+        role_data[2]['RolePolicyList'] = [{'PolicyName': 'all_services_used', 'PolicyDocument': ROLE_POLICIES['all_services_used'] }]
 
+        mock_get_account_authorization_details.side_effect = [role_data]
         mock_get_aardvark_data.return_value = AARDVARK_DATA
 
         def update_role_data(dynamo_table, account_number, role, current_policies):
-            role.policies = role.policies = [{'Policy': current_policies}]
+            role.policies = [{'Policy': current_policies}]
 
         mock_update_role_data.side_effect = update_role_data
 
@@ -186,9 +185,9 @@ def update_role_data(dynamo_table, account_number, role, current_policies):
 
         # validate update data called for each role
         assert mock_update_role_data.mock_calls == [
-            call(dynamo_table, account_number, Role(ROLES[0]), ROLE_POLICIES['all_services_used']),
-            call(dynamo_table, account_number, Role(ROLES[1]), ROLE_POLICIES['unused_ec2']),
-            call(dynamo_table, account_number, Role(ROLES[2]), ROLE_POLICIES['all_services_used'])]
+            call(dynamo_table, account_number, Role(ROLES[0]), {'all_services_used': ROLE_POLICIES['all_services_used']}),
+            call(dynamo_table, account_number, Role(ROLES[1]), {'unused_ec2': ROLE_POLICIES['unused_ec2']}),
+            call(dynamo_table, account_number, Role(ROLES[2]), {'all_services_used': ROLE_POLICIES['all_services_used']})]
 
         # all roles active
         assert mock_find_and_mark_inactive.mock_calls == [call(dynamo_table, account_number,

requirements.txt

@@ -4,13 +4,13 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
-boto3==1.7.83
+boto3==1.8.4
 boto==2.49.0              # via cloudaux
-botocore==1.10.83         # via boto3, cloudaux, s3transfer
-certifi==2018.8.13        # via requests
+botocore==1.11.4          # via boto3, cloudaux, s3transfer
+certifi==2018.8.24        # via requests
 chardet==3.0.4            # via requests
 click==6.7                # via pip-tools
-cloudaux==1.5.0
+cloudaux==1.5.1
 defusedxml==0.5.0         # via cloudaux
 docopt==0.6.2
 docutils==0.14            # via botocore
@@ -21,7 +21,7 @@ idna==2.7                 # via requests
 import-string==0.1.0
 inflection==0.3.1         # via cloudaux
 jmespath==0.9.3           # via boto3, botocore
-joblib==0.12.2            # via cloudaux
+joblib==0.12.3            # via cloudaux
 marshmallow==2.15.4
 pip-tools==2.0.2
 pkginfo==1.4.2            # via twine
@@ -35,4 +35,4 @@ tabulate==0.8.2
 tabview==1.4.3
 tqdm==4.25.0
 twine==1.11.0
-urllib3==1.23             # via requests
+urllib3==1.23             # via botocore, requests