From 7f67f551d5192a7318a816f0665e8d0b388af6cc Mon Sep 17 00:00:00 2001
From: abainbridge <abainbridge@netflix.com>
Date: Thu, 25 Jul 2019 13:38:13 -0700
Subject: [PATCH] added batch processing code

---
 .gitignore                |   1 +
 .idea/misc.xml            |   4 +
 .idea/modules.xml         |   8 ++
 .idea/repokid.iml         |  12 ++
 .idea/vcs.xml             |   6 +
 .idea/workspace.xml       | 287 ++++++++++++++++++++++++++++++++++++++
 repokid/utils/roledata.py | 160 +++++++++++++++------
 7 files changed, 438 insertions(+), 40 deletions(-)
 create mode 100644 .idea/misc.xml
 create mode 100644 .idea/modules.xml
 create mode 100644 .idea/repokid.iml
 create mode 100644 .idea/vcs.xml
 create mode 100644 .idea/workspace.xml

diff --git a/.gitignore b/.gitignore
index e9fff7d7a..dd13f95c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+venv/
 *.pyc
 *.json
 *.log
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 000000000..cf5951cdc
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7 (repokid)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 000000000..ea6a06974
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/repokid.iml" filepath="$PROJECT_DIR$/.idea/repokid.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/repokid.iml b/.idea/repokid.iml
new file mode 100644
index 000000000..48500e151
--- /dev/null
+++ b/.idea/repokid.iml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 2.7 (repokid)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="projectConfiguration" value="pytest" />
+    <option name="PROJECT_TEST_RUNNER" value="pytest" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 000000000..94a25f7f4
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
new file mode 100644
index 000000000..e91e06e64
--- /dev/null
+++ b/.idea/workspace.xml
@@ -0,0 +1,287 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="ef3e6c6a-b7ba-47f8-a2d4-54112f06e4c3" name="Default Changelist" comment="">
+      <change afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/modules.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/repokid.iml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/repokid/utils/roledata.py" beforeDir="false" afterPath="$PROJECT_DIR$/repokid/utils/roledata.py" afterDir="false" />
+    </list>
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileEditorManager">
+    <leaf>
+      <file pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/repokid/utils/roledata.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="349">
+              <caret line="533" column="23" lean-forward="true" selection-start-line="533" selection-start-column="23" selection-end-line="533" selection-end-column="23" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/repokid/tests/test_roledata.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="1185">
+              <caret line="84" selection-start-line="84" selection-end-line="84" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/repokid/hooks/loggers/__init__.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="240">
+              <caret line="17" column="83" selection-start-line="17" selection-start-column="83" selection-end-line="17" selection-end-column="83" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/repokid/hooks/loggers.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="225">
+              <caret line="16" column="41" selection-start-line="16" selection-start-column="41" selection-end-line="16" selection-end-column="41" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+    </leaf>
+  </component>
+  <component name="FindInProjectRecents">
+    <findStrings>
+      <find>AFTER_SCHEDULE_REPO</find>
+      <find>DURING_REPOABLE_CALCULATION</find>
+      <find>config</find>
+      <find>role.arn</find>
+      <find>repoable_services_set</find>
+      <find>_update_repoable_services</find>
+      <find>_get_repoable_permissions</find>
+      <find>minimum_age</find>
+      <find>potentially_repoable_permissions</find>
+      <find>aa_data</find>
+      <find>_get_repoable_permissions_batch</find>
+      <find>deep cop</find>
+      <find>copy</find>
+    </findStrings>
+    <dirStrings>
+      <dir>$PROJECT_DIR$/repokid</dir>
+    </dirStrings>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/repokid/utils/roledata.py" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectConfigurationFiles">
+    <option name="files">
+      <list>
+        <option value="$PROJECT_DIR$/.idea/repokid.iml" />
+        <option value="$PROJECT_DIR$/.idea/vcs.xml" />
+        <option value="$PROJECT_DIR$/.idea/misc.xml" />
+        <option value="$PROJECT_DIR$/.idea/modules.xml" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectFrameBounds">
+    <option name="x" value="1459" />
+    <option name="y" value="23" />
+    <option name="width" value="1740" />
+    <option name="height" value="1150" />
+  </component>
+  <component name="ProjectView">
+    <navigator proportions="" version="1">
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="Scope" />
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="repokid" type="b2602c69:ProjectViewProjectNode" />
+              <item name="repokid" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="nodejs_npm_path_reset_for_default_project" value="true" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="RunManager">
+    <configuration name="pytest for test_roledata.TestRoledata" type="tests" factoryName="py.test" temporary="true" nameIsGenerated="true">
+      <module name="repokid" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/repokid/tests" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <option name="_new_keywords" value="&quot;&quot;" />
+      <option name="_new_additionalArguments" value="&quot;&quot;" />
+      <option name="_new_target" value="&quot;test_roledata.TestRoledata&quot;" />
+      <option name="_new_targetType" value="&quot;PYTHON&quot;" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python tests.pytest for test_roledata.TestRoledata" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="ef3e6c6a-b7ba-47f8-a2d4-54112f06e4c3" name="Default Changelist" comment="" />
+      <created>1564074836208</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1564074836208</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="TestHistory">
+    <history-entry file="pytest_for_test_roledata_TestRoledata - 2019.07.25 at 12h 53m 09s.xml">
+      <configuration name="pytest for test_roledata.TestRoledata" configurationId="tests" />
+    </history-entry>
+    <history-entry file="pytest_for_test_roledata_TestRoledata - 2019.07.25 at 13h 04m 20s.xml">
+      <configuration name="pytest for test_roledata.TestRoledata" configurationId="tests" />
+    </history-entry>
+    <history-entry file="pytest_for_test_roledata_TestRoledata - 2019.07.25 at 13h 35m 31s.xml">
+      <configuration name="pytest for test_roledata.TestRoledata" configurationId="tests" />
+    </history-entry>
+    <history-entry file="pytest_for_test_roledata_TestRoledata - 2019.07.25 at 13h 36m 14s.xml">
+      <configuration name="pytest for test_roledata.TestRoledata" configurationId="tests" />
+    </history-entry>
+    <history-entry file="pytest_for_test_roledata_TestRoledata - 2019.07.25 at 13h 36m 31s.xml">
+      <configuration name="pytest for test_roledata.TestRoledata" configurationId="tests" />
+    </history-entry>
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="1459" y="23" width="1740" height="1150" extended-state="0" />
+    <editor active="true" />
+    <layout>
+      <window_info id="Favorites" side_tool="true" />
+      <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.18266433" />
+      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
+      <window_info anchor="bottom" id="Version Control" />
+      <window_info anchor="bottom" id="Python Console" />
+      <window_info anchor="bottom" id="Terminal" />
+      <window_info anchor="bottom" id="Event Log" side_tool="true" />
+      <window_info anchor="bottom" id="Message" order="0" />
+      <window_info anchor="bottom" id="Find" order="1" />
+      <window_info anchor="bottom" id="Run" order="2" />
+      <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.31758034" />
+      <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
+      <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
+      <window_info anchor="bottom" id="TODO" order="6" />
+      <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
+      <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
+      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
+    </layout>
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/repokid/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="227">
+          <caret line="83" column="38" selection-start-line="83" selection-start-column="38" selection-end-line="83" selection-end-column="38" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/tests/test_repokid_cli.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="227">
+          <caret line="284" column="59" selection-start-line="284" selection-start-column="59" selection-end-line="284" selection-end-column="59" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/role.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="450">
+          <caret line="30" column="22" selection-start-line="30" selection-start-column="22" selection-end-line="30" selection-end-column="22" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/hooks/loggers.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="225">
+          <caret line="16" column="41" selection-start-line="16" selection-start-column="41" selection-end-line="16" selection-end-column="41" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/tests/test_dispatcher_cli.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state>
+          <folding>
+            <element signature="e#0#11#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/tests/__init__.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/hooks/loggers/__init__.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="240">
+          <caret line="17" column="83" selection-start-line="17" selection-start-column="83" selection-end-line="17" selection-end-column="83" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/pycharm/_jb_pytest_runner.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="45">
+          <caret line="3" selection-start-line="3" selection-end-line="3" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/tests/test_roledata.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1185">
+          <caret line="84" selection-start-line="84" selection-end-line="84" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/repokid/utils/roledata.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="349">
+          <caret line="533" column="23" lean-forward="true" selection-start-line="533" selection-start-column="23" selection-end-line="533" selection-end-column="23" />
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>
\ No newline at end of file
diff --git a/repokid/utils/roledata.py b/repokid/utils/roledata.py
index 40e3ed0bd..18502eb97 100644
--- a/repokid/utils/roledata.py
+++ b/repokid/utils/roledata.py
@@ -233,7 +233,16 @@ def update_stats(dynamo_table, roles, source='Scan'):
                 add_to_end_of_list(dynamo_table, role.role_id, 'Stats', new_stats)
 
 
-def _calculate_repo_scores(roles, minimum_age, hooks):
+def _update_repoable_services(role, repoable_permissions, eligible_permissions):
+    (repoable_permissions_set, repoable_services_set) = _convert_repoable_perms_to_perms_and_services(
+        eligible_permissions, repoable_permissions)
+
+    # we're going to store both repoable permissions and repoable services in the field "RepoableServices"
+    role.repoable_services = repoable_services_set + repoable_permissions_set
+    role.repoable_permissions = len(repoable_permissions)
+
+
+def _calculate_repo_scores(roles, minimum_age, hooks, batch=False, batch_size=100):
     """
     Get the total and repoable permissions count and set of repoable services for every role in the account.
     For each role:
@@ -251,9 +260,10 @@ def _calculate_repo_scores(roles, minimum_age, hooks):
     Returns:
         None
     """
+    repo_able_roles = []
+    eligible_permissions_dict = {}
     for role in roles:
         total_permissions, eligible_permissions = _get_role_permissions(role)
-
         role.total_permissions = len(total_permissions)
 
         # if we don't have any access advisor data for a service than nothing is repoable
@@ -265,19 +275,26 @@ def _calculate_repo_scores(roles, minimum_age, hooks):
 
         # permissions are only repoable if the role isn't being disqualified by filter(s)
         if len(role.disqualified_by) == 0:
-            repoable_permissions = _get_repoable_permissions(role.account, role.role_name, eligible_permissions,
-                                                             role.aa_data, role.no_repo_permissions, minimum_age, hooks)
-            (repoable_permissions_set, repoable_services_set) = _convert_repoable_perms_to_perms_and_services(
-                eligible_permissions, repoable_permissions)
-
-            role.repoable_permissions = len(repoable_permissions)
-
-            # we're going to store both repoable permissions and repoable services in the field "RepoableServices"
-            role.repoable_services = repoable_services_set + repoable_permissions_set
+            repo_able_roles.append(role)
+            eligible_permissions_dict[role.arn] = eligible_permissions
         else:
             role.repoable_permissions = 0
             role.repoable_services = []
 
+    repoable_permissions_dict = {}
+    if batch:
+        repoable_permissions_dict = _get_repoable_permissions_batch(repo_able_roles, eligible_permissions_dict, minimum_age, hooks, batch_size)
+    else:
+        for role in repo_able_roles:
+            repoable_permissions_dict[role.arn] = _get_repoable_permissions(role.account, role.role_name, eligible_permissions_dict[role.arn],
+                                                             role.aa_data, role.no_repo_permissions, minimum_age, hooks)
+
+    for role in repo_able_roles:
+        eligible_permissions = eligible_permissions_dict[role.arn]
+        repoable_permissions = repoable_permissions_dict[role.arn]
+        _update_repoable_services(role, repoable_permissions, eligible_permissions)
+
+
 
 def _convert_repoable_perms_to_perms_and_services(total_permissions, repoable_permissions):
     """
@@ -382,31 +399,7 @@ def _get_epoch_authenticated(service_authenticated):
         return (None, False)
 
 
-def _get_repoable_permissions(account_number, role_name, permissions, aa_data, no_repo_permissions, minimum_age,
-                              hooks):
-    """
-    Generate a list of repoable permissions for a role based on the list of all permissions the role's policies
-    currently allow and Access Advisor data for the services included in the role's policies.
-
-    The first step is to come up with a list of services that were used within the time threshold (the same defined)
-    in the age filter config. Permissions are repoable if they aren't in the used list, aren't in the constant list
-    of unsupported services/actions (IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES, IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS),
-    and aren't being temporarily ignored because they're on the no_repo_permissions list (newly added).
-
-    Args:
-        account_number
-        role_name
-        permissions (list): The full list of permissions that the role's permissions allow
-        aa_data (list): A list of Access Advisor data for a role. Each element is a dictionary with a couple required
-                        attributes: lastAuthenticated (epoch time in milliseconds when the service was last used and
-                        serviceNamespace (the service used)
-        no_repo_permissions (dict): Keys are the name of permissions and values are the time the entry expires
-        minimum_age: Minimum age of a role (in days) for it to be repoable
-        hooks: Dict containing hook names and functions to run
-
-    Returns:
-        set: Permissions that are 'repoable' (not used within the time threshold)
-    """
+def _get_potentially_repoable_permissions(role_name, account_number, aa_data, permissions, no_repo_permissions, minimum_age):
     ago = datetime.timedelta(minimum_age)
     now = datetime.datetime.now(tzlocal())
 
@@ -428,10 +421,10 @@ def _get_repoable_permissions(account_number, role_name, permissions, aa_data, n
         if not valid_authenticated:
             LOGGER.error("Got malformed Access Advisor data for {role_name} in {account_number} for service {service}"
                          ": {last_authenticated}".format(
-                             role_name=role_name,
-                             account_number=account_number,
-                             service=service.get('serviceNamespace'),
-                             last_authenticated=service['lastAuthenticated']))
+                role_name=role_name,
+                account_number=account_number,
+                service=service.get('serviceNamespace'),
+                last_authenticated=service['lastAuthenticated']))
             used_services.add(service['serviceNamespace'])
 
         accessed = datetime.datetime.fromtimestamp(accessed, tzlocal())
@@ -452,6 +445,36 @@ def _get_repoable_permissions(account_number, role_name, permissions, aa_data, n
             permission_decision.repoable = True
             permission_decision.decider = 'Access Advisor'
 
+    return potentially_repoable_permissions
+
+
+def _get_repoable_permissions(account_number, role_name, permissions, aa_data, no_repo_permissions, minimum_age,
+                              hooks):
+    """
+    Generate a list of repoable permissions for a role based on the list of all permissions the role's policies
+    currently allow and Access Advisor data for the services included in the role's policies.
+
+    The first step is to come up with a list of services that were used within the time threshold (the same defined)
+    in the age filter config. Permissions are repoable if they aren't in the used list, aren't in the constant list
+    of unsupported services/actions (IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES, IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS),
+    and aren't being temporarily ignored because they're on the no_repo_permissions list (newly added).
+
+    Args:
+        account_number
+        role_name
+        permissions (list): The full list of permissions that the role's permissions allow
+        aa_data (list): A list of Access Advisor data for a role. Each element is a dictionary with a couple required
+                        attributes: lastAuthenticated (epoch time in milliseconds when the service was last used and
+                        serviceNamespace (the service used)
+        no_repo_permissions (dict): Keys are the name of permissions and values are the time the entry expires
+        minimum_age: Minimum age of a role (in days) for it to be repoable
+        hooks: Dict containing hook names and functions to run
+
+    Returns:
+        set: Permissions that are 'repoable' (not used within the time threshold)
+    """
+    potentially_repoable_permissions = _get_potentially_repoable_permissions(role_name, account_number, aa_data, permissions, no_repo_permissions, minimum_age)
+
     hooks_output = repokid.hooks.call_hooks(hooks, 'DURING_REPOABLE_CALCULATION',
                                             {'account_number': account_number,
                                              'role_name': role_name,
@@ -468,6 +491,63 @@ def _get_repoable_permissions(account_number, role_name, permissions, aa_data, n
                 hooks_output['potentially_repoable_permissions'].items() if permission_value.repoable])
 
 
+
+def _get_repoable_permissions_batch(repo_able_roles, permissions_dict, minimum_age, hooks, batch_size):
+    """
+    Generate a dictionary mapping of role arns to their repoable permissions based on the list of all permissions the role's policies
+    currently allow and Access Advisor data for the services included in the role's policies.
+
+    The first step is to come up with a list of services that were used within the time threshold (the same defined)
+    in the age filter config. Permissions are repoable if they aren't in the used list, aren't in the constant list
+    of unsupported services/actions (IAM_ACCESS_ADVISOR_UNSUPPORTED_SERVICES, IAM_ACCESS_ADVISOR_UNSUPPORTED_ACTIONS),
+    and aren't being temporarily ignored because they're on the no_repo_permissions list (newly added).
+
+    Args:
+    repo_able_roles: (list): List of the roles that can be checked for repoing
+    permissions_dict (dict): Mapping role arns to their full list of permissions that the role's permissions allow
+    minimum_age: Minimum age of a role (in days) for it to be repoable
+    hooks: Dict containing hook names and functions to run
+
+    Returns:
+        dict: Mapping role arns to set of permissions that are 'repoable' (not used within the time threshold)
+    """
+
+    if len(repo_able_roles) == 0:
+        return {}
+
+    repo_able_roles_batches = copy.deepcopy(repo_able_roles)
+    potentially_repoable_permissions_dict = {}
+    repoable_set_dict = {}
+    repoable_log_dict = {}
+
+    for role in repo_able_roles:
+        potentially_repoable_permissions_dict[role.arn] = _get_potentially_repoable_permissions(role.role_name, role.account, role.aa_data, permissions_dict[role.arn], role.no_repo_permissions, minimum_age)
+
+    while len(repo_able_roles_batches) > 0:
+        role_batch = repo_able_roles_batches[:batch_size]
+        repo_able_roles_batches = repo_able_roles_batches[batch_size:]
+
+
+        hooks_output = repokid.hooks.call_hooks(hooks, 'DURING_REPOABLE_CALCULATION_BATCH',
+                                                {'role_batch': role_batch,
+                                                 'potentially_repoable_permissions': potentially_repoable_permissions_dict,
+                                                 'minimum_age': minimum_age})
+        for role_arn, output in hooks_output.items():
+            repoable_set = set([permission_name for permission_name, permission_value in
+                                output['potentially_repoable_permissions'].items() if permission_value.repoable])
+            repoable_set_dict[role_arn] = repoable_set
+            repoable_log_dict[role_arn] = ''.join('{}: {}\n'.format(perm, decision.decider)
+                             for perm, decision in output['potentially_repoable_permissions'].items())
+
+    for role in repo_able_roles:
+        LOGGER.debug('Repoable permissions for role {role_name} in {account_number}:\n{repoable}'.format(
+            role_name=role.role_name,
+            account_number=role.account,
+            repoable=repoable_log_dict[role.arn]))
+    return repoable_set_dict
+
+
+
 def _get_repoed_policy(policies, repoable_permissions):
     """
     This function contains the logic to rewrite the policy to remove any repoable permissions. To do so we: