Merge 000e161 into fb7ce26

historical_reports/__about__.py

@@ -9,7 +9,7 @@
 __summary__ = "Collection of reporting functions built on top of Historical data sets."
 __uri__ = "https://github.com/Netflix-Skunkworks/historical-reports"
 
-__version__ = "0.1.11"
+__version__ = "0.1.12"
 
 __author__ = "The Historical developers"
 __email__ = "security@netflix.com"

historical_reports/s3/cli.py

@@ -48,5 +48,5 @@ def get_dump_prefix(ctx, param, prefix):
 @click.option("-c", "--commit", default=False, is_flag=True, help="Will only dump to S3 if commit flag is present")
 def generate(bucket, exclude_fields, dump_prefix, commit):
     if not commit:
-        log.warning("COMMIT FLAG NOT SET -- NOT SAVING ANYTHING TO S3!")
+        log.warning("[@] COMMIT FLAG NOT SET -- NOT SAVING ANYTHING TO S3!")
     dump_report(commit=commit)

historical_reports/s3/entrypoints.py

@@ -30,12 +30,16 @@ def handler(event, context):
     set_config_from_input(event)
 
     if event.get("Records"):
+        log.debug('[@] Received update event with records.')
+
         # Deserialize the records:
         records = deserialize_records(event["Records"])
+        log.debug('[ ] Received the (deserialized) records: {}'.format(records))
 
         # Update event:
         update_records(records)
 
     else:
+        log.debug('[@] Received a scheduled event for a full report.')
         # Generate event:
         dump_report()

historical_reports/s3/generate.py

@@ -21,18 +21,18 @@
 
 def dump_report(commit=True):
     # Get all the data from DynamoDB:
-    log.debug("Starting... Beginning scan.")
+    log.debug("[@] Starting... Beginning scan.")
     all_buckets = CurrentS3Model.scan()
 
     generated_file = S3ReportSchema(strict=True).dump({"all_buckets": all_buckets}).data
 
     # Dump to S3:
     if commit:
-        log.debug("Saving to S3.")
+        log.debug("[-->] Saving to S3.")
 
         # Replace <empty> with "" <-- Due to Pynamo/Dynamo issues...
         dump_to_s3(json.dumps(generated_file, indent=4).replace("\"<empty>\"", "\"\"").encode("utf-8"))
     else:
-        log.debug("Commit flag not set, not saving.")
+        log.debug("[/] Commit flag not set, not saving.")
 
-    log.debug("Completed S3 report generation.")
+    log.debug("[@] Completed S3 report generation.")

historical_reports/s3/models.py

@@ -39,7 +39,7 @@ class BucketField(Field):
     def _serialize(self, value, attr=None, data=None):
         buckets = data.get("buckets", {})
         for b in data["all_buckets"]:
-            log.debug("Fetched details for bucket: {}".format(b.arn))
+            log.debug("[+] Fetched details for bucket: {}".format(b.arn))
             name = b.BucketName
 
             # Add the bucket:

historical_reports/s3/update.py

@@ -55,27 +55,27 @@ def process_dynamodb_record(record, s3_report):
         # If the current object is too big for SNS, and it's not in the current table, then delete it.
         # -- OR -- if this a soft-deletion? (Config set to {})
         if not modified_bucket or not modified_bucket.configuration.attribute_values:
-            log.debug('Processing deletion for: {}'.format(record['dynamodb']['NewImage']["BucketName"]["S"]))
+            log.debug('[ ] Processing deletion for: {}'.format(record['dynamodb']['NewImage']["BucketName"]["S"]))
             s3_report["buckets"].pop(record['dynamodb']['NewImage']["BucketName"]["S"], None)
         else:
-            log.debug('Processing: {}'.format(modified_bucket.BucketName))
+            log.debug('[ ] Processing: {}'.format(modified_bucket.BucketName))
             s3_report["all_buckets"].append(modified_bucket)
 
 
 def update_records(records, commit=True):
-    log.debug("Starting Record Update.")
+    log.debug("[@] Starting Record Update.")
 
     # First, grab the existing JSON from S3:
     existing_json = fetch_from_s3()
-    log.debug("Grabbed all the existing data from S3.")
+    log.debug("[+] Grabbed all the existing data from S3.")
 
     # If the existing JSON is not present for some reason, then...
     if not existing_json:
         if commit and CONFIG.export_if_missing:
             CONFIG.dump_to_buckets = CONFIG.import_bucket.split(",")
             CONFIG.dump_to_prefix = CONFIG.import_prefix
-            log.info("The report does not exist. Dumping the full report to {}/{}".format(CONFIG.import_bucket,
-                                                                                          CONFIG.import_prefix))
+            log.info("[!] The report does not exist. Dumping the full report to {}/{}".format(CONFIG.import_bucket,
+                                                                                              CONFIG.import_prefix))
             dump_report()
 
         else:
@@ -96,12 +96,12 @@ def update_records(records, commit=True):
 
     # Dump to S3:
     if commit:
-        log.debug("Saving to S3.")
+        log.debug("[-->] Saving to S3.")
 
         # Replace <empty> with "" <-- Due to Pynamo/Dynamo issues...
         dump_to_s3(json.dumps(generated_file, indent=4, default=decimal_default).replace("\"<empty>\"", "\"\"").encode(
             "utf-8"))
     else:
-        log.debug("Commit flag not set, not saving.")
+        log.debug("[/] Commit flag not set, not saving.")
 
-    log.debug("Completed S3 report update.")
+    log.debug("[@] Completed S3 report update.")

historical_reports/s3/util.py

@@ -44,7 +44,7 @@ def dump_to_s3(file):
 
     # Loop over each bucket and dump:
     for bucket in CONFIG.dump_to_buckets:
-        log.debug("[ ] Dumping to {}/{}".format(bucket, CONFIG.dump_to_prefix))
+        log.debug("[-->] Dumping to {}/{}".format(bucket, CONFIG.dump_to_prefix))
         _upload_to_s3(file, client, bucket, CONFIG.dump_to_prefix)
         log.debug("[+] Complete")
 

setup.py

@@ -23,7 +23,7 @@
 
 
 install_requires = [
-    'historical>=0.3.16',
+    'historical>=0.3.17',
     'retrying==1.3.3',
     'click==6.7'
 ]