From ed17a05d292e8394e25331fd15cbe3a1d08aa1f0 Mon Sep 17 00:00:00 2001
From: Saurabh Bangad <35229617+saurabhbangad@users.noreply.github.com>
Date: Fri, 26 Jan 2018 00:15:53 +0100
Subject: [PATCH 1/6] Retrieve all audit events across an OCI Tenancy

This script retrieves all audit logs across an OCI Tenancy for a defined timespan.

This script relies on OCI config as per the format defined at
https://docs.us-phoenix-1.oraclecloud.com/Content/API/Concepts/sdkconfig.htm

This script will work at a tenancy level only.

The config file should have the following contents
User OCID
RSA private key in PEM format
Fingerprint
Tenancy OCID
---
 examples/RetrieveAuditEvents.py | 139 ++++++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100644 examples/RetrieveAuditEvents.py

diff --git a/examples/RetrieveAuditEvents.py b/examples/RetrieveAuditEvents.py
new file mode 100644
index 0000000000..227b686211
--- /dev/null
+++ b/examples/RetrieveAuditEvents.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+'''
+This script retrieves all audit logs across an OCI Tenancy
+for a defined timespan.
+
+This script relies on OCI config as per the format defined at
+https://docs.us-phoenix-1.oraclecloud.com/Content/API/Concepts/sdkconfig.htm
+
+This script will work at a tenancy level only.
+
+The config file should have the following contents
+User OCID
+RSA private key in PEM format
+Fingerprint
+Tenancy OCID
+'''
+
+import oci
+
+__author__ = "saurabh.bangad@oracle.com"
+
+configfile = "~/.oci/config"
+
+config = oci.config.from_file(configfile)  # Path to relevant config file
+tenancy_id = config["tenancy"]
+# This array will be used to store the list of compartments in the tenancy
+compartments = []
+compartments.append(tenancy_id)  # Tenancy is the first compartment
+regions = []  # This array will be used to store the list of available regions
+auditevents = []
+
+
+#  Initiate the client with the locally available config
+#  These are global variables and used across all the functions
+audit = oci.audit.audit_client.AuditClient(config)
+identity = oci.identity.IdentityClient(config)
+
+
+# Timespan defined by variables start_time and end_time
+start_time = '2018-01-15T00:00:00Z'
+end_time = '2018-01-17T00:00:00Z'
+
+
+def get_regions():
+    '''To retrieve the list of all available regions
+    '''
+    listofregions = []
+    rawsoutput = []
+    rawoutput = identity.list_regions()
+    for i in range(len(rawoutput.data)):
+        listofregions.append(str(rawoutput.data[i].name))
+    return listofregions
+
+
+def paginate(operation, *args, **kwargs):
+    '''Function to paginate API calls.
+
+    Refer to
+    https://github.com/oracle/oci-python-sdk/blob/master/docs/quickstart.rst
+
+    For better exception handling refer to
+    https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/index.html#oci.exceptions.ServiceError
+    '''
+    while True:
+        try:
+            response = operation(*args, **kwargs)
+            for value in response.data:
+                yield value
+            kwargs["page"] = response.next_page
+            if not response.has_next_page:
+                break
+        except oci.exceptions.ServiceError as e:
+            if e.code == 'NotAuthorizedOrNotFound':
+                print 'Error: Please check if you have sufficient \
+                permissions to execute the operation' + str(operation.im_func)
+            elif e.status == '500':
+                print 'InternalServerError: Please retry after sometime.'
+            else:
+                print 'Something went wrong.'
+                print 'Here is the info about the exception:'
+                print str(e)
+            quit()
+
+
+def get_compartments():
+    ''' Retrieve the list of compartments under the tenancy
+    '''
+    listofcompartments = []
+    for list_compartments_json in paginate(
+            identity.list_compartments,
+            compartment_id=tenancy_id):
+            listofcompartments.append(str(list_compartments_json.id))
+    return listofcompartments
+
+
+def get_audit_events(start_time, end_time):
+    '''Get events iteratively for each compartment defined in 'compartments' array
+    for the region defined in "config['region']".
+    '''
+    listofauditevents = []
+    for i in range(len(compartments)):
+        for events in paginate(
+                audit.list_events,
+                compartment_id=compartments[i],
+                start_time=start_time,
+                end_time=end_time):
+            '''The following condition eliminates all the read API calls.
+              Uncomment this if write events need to be filtered.
+              '''
+            # if events.request_action != 'GET':
+            listofauditevents.append(events)
+    return listofauditevents
+
+
+'''Here's the main code that retrives audits across the tenancy'''
+
+#  Retrieve the list of all the OCI regions.
+#  This gracefully handles addition of new regions.
+regions = get_regions()
+
+#  Retrieve all the compartments.
+#  This gracefully handles compartment changes on a tenancy.
+compartments = get_compartments()
+
+#  For a region defined by config['region'] get the logs for each compartment
+for i in range(len(regions)):
+    config['region'] = regions[i]
+
+    #  Reintialize with a different region value
+    audit = oci.audit.audit_client.AuditClient(config)
+    identity = oci.identity.IdentityClient(config)
+
+    #  To separate results by region use print here
+    auditevents = get_audit_events(
+        start_time, end_time)
+    if auditevents:
+        print auditevents

From 1b1f861b7c23cff0919dcbd0f413d91bad220364 Mon Sep 17 00:00:00 2001
From: Saurabh Bangad <35229617+saurabhbangad@users.noreply.github.com>
Date: Tue, 13 Feb 2018 19:02:14 +0100
Subject: [PATCH 2/6] Changes based on review and file name.

Accommodated the changes suggested by @nathan-vu

The following is a summary of all the changes:
-file name from RetrieveAuditEvents.py to retrieve_audit_events.py
-instead of hard coding to start_time and end_time, they receive values on execution
-oci pagination is being used instead of custom definition
-removed the len(range(list)) for
-get_regions() and get_compartments() now accept identity as their parameter
-region is updated with set_region()
-changes to variables name; they use snake_casing
-edited comments
---
 examples/RetrieveAuditEvents.py   | 139 ------------------------------
 examples/retrieve_audit_events.py | 108 +++++++++++++++++++++++
 2 files changed, 108 insertions(+), 139 deletions(-)
 delete mode 100644 examples/RetrieveAuditEvents.py
 create mode 100644 examples/retrieve_audit_events.py

diff --git a/examples/RetrieveAuditEvents.py b/examples/RetrieveAuditEvents.py
deleted file mode 100644
index 227b686211..0000000000
--- a/examples/RetrieveAuditEvents.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
-'''
-This script retrieves all audit logs across an OCI Tenancy
-for a defined timespan.
-
-This script relies on OCI config as per the format defined at
-https://docs.us-phoenix-1.oraclecloud.com/Content/API/Concepts/sdkconfig.htm
-
-This script will work at a tenancy level only.
-
-The config file should have the following contents
-User OCID
-RSA private key in PEM format
-Fingerprint
-Tenancy OCID
-'''
-
-import oci
-
-__author__ = "saurabh.bangad@oracle.com"
-
-configfile = "~/.oci/config"
-
-config = oci.config.from_file(configfile)  # Path to relevant config file
-tenancy_id = config["tenancy"]
-# This array will be used to store the list of compartments in the tenancy
-compartments = []
-compartments.append(tenancy_id)  # Tenancy is the first compartment
-regions = []  # This array will be used to store the list of available regions
-auditevents = []
-
-
-#  Initiate the client with the locally available config
-#  These are global variables and used across all the functions
-audit = oci.audit.audit_client.AuditClient(config)
-identity = oci.identity.IdentityClient(config)
-
-
-# Timespan defined by variables start_time and end_time
-start_time = '2018-01-15T00:00:00Z'
-end_time = '2018-01-17T00:00:00Z'
-
-
-def get_regions():
-    '''To retrieve the list of all available regions
-    '''
-    listofregions = []
-    rawsoutput = []
-    rawoutput = identity.list_regions()
-    for i in range(len(rawoutput.data)):
-        listofregions.append(str(rawoutput.data[i].name))
-    return listofregions
-
-
-def paginate(operation, *args, **kwargs):
-    '''Function to paginate API calls.
-
-    Refer to
-    https://github.com/oracle/oci-python-sdk/blob/master/docs/quickstart.rst
-
-    For better exception handling refer to
-    https://oracle-cloud-infrastructure-python-sdk.readthedocs.io/en/latest/api/index.html#oci.exceptions.ServiceError
-    '''
-    while True:
-        try:
-            response = operation(*args, **kwargs)
-            for value in response.data:
-                yield value
-            kwargs["page"] = response.next_page
-            if not response.has_next_page:
-                break
-        except oci.exceptions.ServiceError as e:
-            if e.code == 'NotAuthorizedOrNotFound':
-                print 'Error: Please check if you have sufficient \
-                permissions to execute the operation' + str(operation.im_func)
-            elif e.status == '500':
-                print 'InternalServerError: Please retry after sometime.'
-            else:
-                print 'Something went wrong.'
-                print 'Here is the info about the exception:'
-                print str(e)
-            quit()
-
-
-def get_compartments():
-    ''' Retrieve the list of compartments under the tenancy
-    '''
-    listofcompartments = []
-    for list_compartments_json in paginate(
-            identity.list_compartments,
-            compartment_id=tenancy_id):
-            listofcompartments.append(str(list_compartments_json.id))
-    return listofcompartments
-
-
-def get_audit_events(start_time, end_time):
-    '''Get events iteratively for each compartment defined in 'compartments' array
-    for the region defined in "config['region']".
-    '''
-    listofauditevents = []
-    for i in range(len(compartments)):
-        for events in paginate(
-                audit.list_events,
-                compartment_id=compartments[i],
-                start_time=start_time,
-                end_time=end_time):
-            '''The following condition eliminates all the read API calls.
-              Uncomment this if write events need to be filtered.
-              '''
-            # if events.request_action != 'GET':
-            listofauditevents.append(events)
-    return listofauditevents
-
-
-'''Here's the main code that retrives audits across the tenancy'''
-
-#  Retrieve the list of all the OCI regions.
-#  This gracefully handles addition of new regions.
-regions = get_regions()
-
-#  Retrieve all the compartments.
-#  This gracefully handles compartment changes on a tenancy.
-compartments = get_compartments()
-
-#  For a region defined by config['region'] get the logs for each compartment
-for i in range(len(regions)):
-    config['region'] = regions[i]
-
-    #  Reintialize with a different region value
-    audit = oci.audit.audit_client.AuditClient(config)
-    identity = oci.identity.IdentityClient(config)
-
-    #  To separate results by region use print here
-    auditevents = get_audit_events(
-        start_time, end_time)
-    if auditevents:
-        print auditevents
diff --git a/examples/retrieve_audit_events.py b/examples/retrieve_audit_events.py
new file mode 100644
index 0000000000..ca92162dd2
--- /dev/null
+++ b/examples/retrieve_audit_events.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+#  coding: utf-8
+#  Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+
+#  This script retrieves all audit logs across an OCI Tenancy.
+#  for a timespan defined by start_time and end_time.
+#  This script will work at a tenancy level only.
+
+
+import oci
+from datetime import date
+from dateutil.relativedelta import relativedelta
+
+
+def get_regions(identity):
+    '''To retrieve the list of all available regions.
+    '''
+    list_of_regions = []
+    list_regions_response = identity.list_regions()
+    for r in list_regions_response.data:
+        list_of_regions.append(r.name)
+    return list_of_regions
+
+
+def get_compartments(identity):
+    ''' Retrieve the list of compartments under the tenancy.
+    '''
+    compartment_ocids = []
+    list_compartments_response = oci.pagination.list_call_get_all_results(
+        identity.list_compartments,
+        compartment_id=config['tenancy']).data
+    for c in list_compartments_response:
+        compartment_ocids.append(c.id)
+    return compartment_ocids
+
+
+def get_audit_events(audit, compartment_ocids, start_time, end_time, r):
+    '''Get events iteratively for each compartment defined in 'compartments' array
+    for the region defined in "config['region']".
+    '''
+    list_of_audit_events = []
+    for c in compartment_ocids:
+        #  To separate results by compartments use print here.
+        list_events_response = oci.pagination.list_call_get_all_results(
+            audit.list_events,
+            compartment_id=c,
+            start_time=start_time,
+            end_time=end_time).data
+
+        #  Results for a compartment 'c' for a region defined
+        #  in 'audit' object.
+        #  Events can be filtered here based on some constraints.
+        #  For example, to filter write events make use of
+        #  the following condition
+        #  list_events_response.data.request_action != 'GET'
+        list_of_audit_events.append(list_events_response)
+    return list_of_audit_events
+
+
+'''Here's the code that retrives audits across the tenancy'''
+#  Setting configuration
+#  Default path for configuration file is "~/.oci/config"
+config = oci.config.from_file()
+tenancy_id = config["tenancy"]
+
+#  Initiate the client with the locally available config.
+#  These are global variables and used across all the functions.
+audit = oci.audit.audit_client.AuditClient(config)
+identity = oci.identity.IdentityClient(config)
+
+
+#  Timespan defined by variables start_time and end_time(today).
+today = date.today() + relativedelta(months=0)
+today_minus_one_month = date.today() + relativedelta(months=-1)
+
+#  ListEvents expects timestamps into RFC3339 format.
+#  Converting dates into RFC3339 e.g. 2018-01-15T00:00:00Z
+end_time = str(today.year) + '-' + str(today.month).zfill(2) \
+    + '-' + str(today.day) + 'T00:00:00Z'
+start_time = str(today_minus_one_month.year) + '-' \
+    + str(today_minus_one_month.month).zfill(2) + '-' \
+    + str(today_minus_one_month.day) + 'T00:00:00Z'
+
+regions = []  # This array will be used to store the list of available regions.
+regions = get_regions(identity)
+
+# This array will be used to store the list of compartments in the tenancy.
+compartments = []
+compartments.append(tenancy_id)  # Tenancy is the first compartment.
+compartments = get_compartments(identity)
+
+audit = oci.audit.audit_client.AuditClient(config)
+audit_events = []
+
+#  For each region get the logs for each compartment.
+for r in regions:
+    #  Intialize with a region value.
+    audit.base_client.set_region(r)
+    #  To separate results by region use print here.
+    audit_events = get_audit_events(
+        audit,
+        compartments,
+        start_time,
+        end_time, r)
+
+    #  Results for a region 'r' for each compartment.
+    if audit_events:
+        print audit_events

From 0cab943f6b8f4c2bfcd72c7daa108ebcdceda516 Mon Sep 17 00:00:00 2001
From: Saurabh Bangad <35229617+saurabhbangad@users.noreply.github.com>
Date: Fri, 16 Feb 2018 09:15:07 +0100
Subject: [PATCH 3/6] Changes based on 2nd review on 20180215

The following were the changes made in the latest version:
-start_time and end_time do not do any str operations and rely on datetime.datetime
-logs are retrieved only for the timespan of last 5 days instead of a month; this would be a better experience considering the time retrieve results
-changes to get_compartments; it will accept tenancy_id as its parameter and it will also be the added to the list even before ListCompartments api call is made
-unnecessary initializations were removed
-change of comment style for the description of functions
-parameter 'r' was removed from get_regions() which was originally put to demonstrate the region being used

Changes not made:
-No changes were made to the retrieval method i.e. it is still eagerly loading all the results
---
 examples/retrieve_audit_events.py | 47 ++++++++++++++-----------------
 1 file changed, 21 insertions(+), 26 deletions(-)

diff --git a/examples/retrieve_audit_events.py b/examples/retrieve_audit_events.py
index ca92162dd2..6457223a59 100644
--- a/examples/retrieve_audit_events.py
+++ b/examples/retrieve_audit_events.py
@@ -4,16 +4,19 @@
 
 #  This script retrieves all audit logs across an OCI Tenancy.
 #  for a timespan defined by start_time and end_time.
+#  This sample script retrieves Audit events for last 5 days.
 #  This script will work at a tenancy level only.
 
 
 import oci
 from datetime import date
+import datetime
 from dateutil.relativedelta import relativedelta
 
 
 def get_regions(identity):
-    '''To retrieve the list of all available regions.
+    '''
+    To retrieve the list of all available regions.
     '''
     list_of_regions = []
     list_regions_response = identity.list_regions()
@@ -22,20 +25,24 @@ def get_regions(identity):
     return list_of_regions
 
 
-def get_compartments(identity):
-    ''' Retrieve the list of compartments under the tenancy.
+def get_compartments(identity, tenancy_id):
+    '''
+    Retrieve the list of compartments under the tenancy.
     '''
     compartment_ocids = []
+    #  Store tenancy id as the first compartment
+    compartment_ocids.append(tenancy_id)
     list_compartments_response = oci.pagination.list_call_get_all_results(
         identity.list_compartments,
-        compartment_id=config['tenancy']).data
+        compartment_id=tenancy_id).data
     for c in list_compartments_response:
         compartment_ocids.append(c.id)
     return compartment_ocids
 
 
-def get_audit_events(audit, compartment_ocids, start_time, end_time, r):
-    '''Get events iteratively for each compartment defined in 'compartments' array
+def get_audit_events(audit, compartment_ocids, start_time, end_time):
+    '''
+    Get events iteratively for each compartment defined in 'compartments' array
     for the region defined in "config['region']".
     '''
     list_of_audit_events = []
@@ -54,7 +61,7 @@ def get_audit_events(audit, compartment_ocids, start_time, end_time, r):
         #  the following condition
         #  list_events_response.data.request_action != 'GET'
         list_of_audit_events.append(list_events_response)
-    return list_of_audit_events
+        return list_of_audit_events
 
 
 '''Here's the code that retrives audits across the tenancy'''
@@ -64,33 +71,21 @@ def get_audit_events(audit, compartment_ocids, start_time, end_time, r):
 tenancy_id = config["tenancy"]
 
 #  Initiate the client with the locally available config.
-#  These are global variables and used across all the functions.
-audit = oci.audit.audit_client.AuditClient(config)
 identity = oci.identity.IdentityClient(config)
 
-
 #  Timespan defined by variables start_time and end_time(today).
-today = date.today() + relativedelta(months=0)
-today_minus_one_month = date.today() + relativedelta(months=-1)
-
 #  ListEvents expects timestamps into RFC3339 format.
-#  Converting dates into RFC3339 e.g. 2018-01-15T00:00:00Z
-end_time = str(today.year) + '-' + str(today.month).zfill(2) \
-    + '-' + str(today.day) + 'T00:00:00Z'
-start_time = str(today_minus_one_month.year) + '-' \
-    + str(today_minus_one_month.month).zfill(2) + '-' \
-    + str(today_minus_one_month.day) + 'T00:00:00Z'
-
-regions = []  # This array will be used to store the list of available regions.
+#  For the purposes of sample script, logs of last 5 days.
+end_time = datetime.datetime.utcnow()
+start_time = end_time + datetime.timedelta(days=-5)
+
+# This array will be used to store the list of available regions.
 regions = get_regions(identity)
 
 # This array will be used to store the list of compartments in the tenancy.
-compartments = []
-compartments.append(tenancy_id)  # Tenancy is the first compartment.
-compartments = get_compartments(identity)
+compartments = get_compartments(identity, tenancy_id)
 
 audit = oci.audit.audit_client.AuditClient(config)
-audit_events = []
 
 #  For each region get the logs for each compartment.
 for r in regions:
@@ -101,7 +96,7 @@ def get_audit_events(audit, compartment_ocids, start_time, end_time, r):
         audit,
         compartments,
         start_time,
-        end_time, r)
+        end_time)
 
     #  Results for a region 'r' for each compartment.
     if audit_events:

From 0e3e1e0f5fa05ac816b69f37e3bbbf157939d50e Mon Sep 17 00:00:00 2001
From: Saurabh Bangad <35229617+saurabhbangad@users.noreply.github.com>
Date: Fri, 16 Feb 2018 17:47:11 +0100
Subject: [PATCH 4/6] list_of_audit_events relies on extend

---
 examples/retrieve_audit_events.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/examples/retrieve_audit_events.py b/examples/retrieve_audit_events.py
index 6457223a59..6d2557b362 100644
--- a/examples/retrieve_audit_events.py
+++ b/examples/retrieve_audit_events.py
@@ -9,9 +9,7 @@
 
 
 import oci
-from datetime import date
 import datetime
-from dateutil.relativedelta import relativedelta
 
 
 def get_regions(identity):
@@ -56,11 +54,7 @@ def get_audit_events(audit, compartment_ocids, start_time, end_time):
 
         #  Results for a compartment 'c' for a region defined
         #  in 'audit' object.
-        #  Events can be filtered here based on some constraints.
-        #  For example, to filter write events make use of
-        #  the following condition
-        #  list_events_response.data.request_action != 'GET'
-        list_of_audit_events.append(list_events_response)
+        list_of_audit_events.extend(list_events_response)
         return list_of_audit_events
 
 

From 65215062fdd53aa903dde211b50ca297f98fcd8e Mon Sep 17 00:00:00 2001
From: Saurabh Bangad <35229617+saurabhbangad@users.noreply.github.com>
Date: Tue, 20 Feb 2018 09:43:44 +0100
Subject: [PATCH 5/6] Updates to comments

Updated the following sections' comments:
-get_audit_events
---
 examples/retrieve_audit_events.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/examples/retrieve_audit_events.py b/examples/retrieve_audit_events.py
index 6d2557b362..ce826e26a0 100644
--- a/examples/retrieve_audit_events.py
+++ b/examples/retrieve_audit_events.py
@@ -40,12 +40,15 @@ def get_compartments(identity, tenancy_id):
 
 def get_audit_events(audit, compartment_ocids, start_time, end_time):
     '''
-    Get events iteratively for each compartment defined in 'compartments' array
-    for the region defined in "config['region']".
+    Get events iteratively for each compartment defined in 'compartments_ocids'
+    for the region defined in 'audit'.
+    This method eagerly loads all audit records in the time range and it does
+    have performance implications of lot of audit records.
+    Ideally, the generator method in oci.pagination should be used to lazily
+    load results.
     '''
     list_of_audit_events = []
     for c in compartment_ocids:
-        #  To separate results by compartments use print here.
         list_events_response = oci.pagination.list_call_get_all_results(
             audit.list_events,
             compartment_id=c,
@@ -58,7 +61,6 @@ def get_audit_events(audit, compartment_ocids, start_time, end_time):
         return list_of_audit_events
 
 
-'''Here's the code that retrives audits across the tenancy'''
 #  Setting configuration
 #  Default path for configuration file is "~/.oci/config"
 config = oci.config.from_file()

From c5fc951e57304241848836e561a5cfc667654de4 Mon Sep 17 00:00:00 2001
From: Saurabh Bangad <35229617+saurabhbangad@users.noreply.github.com>
Date: Tue, 20 Feb 2018 09:50:37 +0100
Subject: [PATCH 6/6] Change of order for import modules

---
 examples/retrieve_audit_events.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/retrieve_audit_events.py b/examples/retrieve_audit_events.py
index ce826e26a0..eed865eef5 100644
--- a/examples/retrieve_audit_events.py
+++ b/examples/retrieve_audit_events.py
@@ -7,9 +7,8 @@
 #  This sample script retrieves Audit events for last 5 days.
 #  This script will work at a tenancy level only.
 
-
-import oci
 import datetime
+import oci
 
 
 def get_regions(identity):