Minor adjustments to access log csv

api/handlers/reporthandler.py

@@ -1,11 +1,11 @@
+import bson
 import copy
 import unicodecsv as csv
 import datetime
-import os
-
-import bson
 import dateutil
+import os
 import pymongo
+import pytz
 
 from .. import config
 from .. import tempdir as tempfile
@@ -19,29 +19,29 @@
 BYTES_IN_MEGABYTE = float(1<<20)
 ACCESS_LOG_FIELDS = [
     "_id",
+    "timestamp",
     "access_type",
-    "context.acquisition.id",
-    "context.acquisition.label",
-    "context.analysis.id",
-    "context.analysis.label",
-    "context.collection.id",
-    "context.collection.label",
+    "origin.id",
+    "origin.method",
+    "origin.name",
+    "origin.type",
     "context.group.id",
     "context.group.label",
     "context.project.id",
     "context.project.label",
-    "context.session.id",
-    "context.session.label",
     "context.subject.id",
     "context.subject.label",
+    "context.session.id",
+    "context.session.label",
+    "context.acquisition.id",
+    "context.acquisition.label",
+    "context.analysis.id",
+    "context.analysis.label",
+    "context.collection.id",
+    "context.collection.label",
     "context.ticket_id",
-    "origin.id",
-    "origin.method",
-    "origin.name",
-    "origin.type",
     "request_method",
-    "request_path",
-    "timestamp"
+    "request_path"
 ]
 
 class APIReportException(Exception):
@@ -50,7 +50,6 @@ class APIReportException(Exception):
 class APIReportParamsException(Exception):
     pass
 
-
 class ReportHandler(base.RequestHandler):
 
     def __init__(self, request=None, response=None):
@@ -73,24 +72,16 @@ def get(self, report_type):
             raise NotImplementedError('Report type {} is not supported'.format(report_type))
 
         if self.superuser_request or report.user_can_generate(self.uid):
-            # If csv is true create a temp file to respond with
-            if report_type == 'accesslog' and self.request.params.get('csv') == 'true':
 
+            if self.is_true('csv'):
                 tempdir = tempfile.TemporaryDirectory(prefix='.tmp', dir=config.get_item('persistent', 'data_path'))
-                csv_file = open(os.path.join(tempdir.name, 'accesslog.csv'), 'w+')
-                writer = csv.DictWriter(csv_file, ACCESS_LOG_FIELDS)
-                writer.writeheader()
-                try:
-                    for doc in report.build():
-                        writer.writerow(doc)
-
-                except APIReportException as e:
-                    self.abort(404, str(e))
-                # Need to close and reopen file to flush buffer into file
-                csv_file.close()
-                self.response.app_iter = open(os.path.join(tempdir.name, 'accesslog.csv'), 'r')
+                filepath = os.path.join(tempdir.name, report.csv_filename)
+
+                report.build_csv(filepath)
+
+                self.response.app_iter = open(filepath, 'r')
                 self.response.headers['Content-Type'] = 'text/csv'
-                self.response.headers['Content-Disposition'] = 'attachment; filename="accesslog.csv"'
+                self.response.headers['Content-Disposition'] = 'attachment; filename="{}"'.format(report.csv_filename)
             else:
                 return report.build()
         else:
@@ -99,6 +90,8 @@ def get(self, report_type):
 
 class Report(object):
 
+    csv_filename = 'report.csv'
+
     def __init__(self, params):
         """
         Initialize a Report
@@ -119,6 +112,13 @@ def build(self):
         """
         raise NotImplementedError()
 
+    def build_csv(self, filepath): # pylint: disable=unused-argument
+        """
+        Build and return a csv file of the report
+        """
+        raise APIReportParamsException('This report does not support csv file format.')
+
+
     @staticmethod
     def _get_result_list(output):
         """
@@ -481,6 +481,9 @@ class AccessLogReport(Report):
       - information about the session/project/group in which the action took place
     """
 
+    # What to name csvs generated from this report
+    csv_filename = 'accesslog.csv'
+
     def __init__(self, params):
         """
         Initialize an Access Log Report
@@ -503,7 +506,6 @@ def __init__(self, params):
         limit= params.get('limit', 100)
         subject = params.get('subject', None)
         access_types = params.getall('access_type')
-        csv_bool = (params.get('csv') == 'true')
 
         if start_date:
             start_date = dateutil.parser.parse(start_date)
@@ -531,7 +533,6 @@ def __init__(self, params):
         self.limit          = limit
         self.subject        = subject
         self.access_types   = access_types
-        self.csv_bool       = csv_bool
 
     def user_can_generate(self, uid):
         """
@@ -541,21 +542,6 @@ def user_can_generate(self, uid):
             return True
         return False
 
-    def flatten(self, json_obj, flat, prefix = ""):
-        """
-        flattens a document to not have nested objects
-        """
-
-        for field in json_obj.keys():
-            if isinstance(json_obj[field], dict):
-                flat = self.flatten(json_obj[field], flat, prefix = prefix + field + ".")
-            else:
-                flat[prefix + field] = json_obj[field]
-        return flat
-
-    def make_csv_ready(self, cursor):
-        return [self.flatten(json_obj, {}) for json_obj in cursor]
-
     def build(self):
         query = {}
 
@@ -572,11 +558,25 @@ def build(self):
         if self.access_types:
             query['access_type'] = {'$in': self.access_types}
 
-        cursor = config.log_db.access_log.find(query).limit(self.limit).sort('timestamp', pymongo.DESCENDING).batch_size(1000)
-        if self.csv_bool:
-            return self.make_csv_ready(cursor)
+        return config.log_db.access_log.find(query).limit(self.limit).sort('timestamp', pymongo.DESCENDING).batch_size(1000)
+
+    def build_csv(self, filepath):
+        csv_file = open(filepath, 'w+')
+        writer = csv.DictWriter(csv_file, ACCESS_LOG_FIELDS)
+        writer.writeheader()
+
+        for doc in self.build():
+
+            # Format timestamp as ISO UTC
+            doc['timestamp'] = pytz.timezone('UTC').localize(doc['timestamp']).isoformat()
+
+            # mongo_dict flattens dictionaries using a dot notation
+            writer.writerow(util.mongo_dict(doc))
+
+        # Need to close and reopen file to flush buffer into file
+        csv_file.close()
+
 
-        return cursor
 
 class UsageReport(Report):
     """

api/web/base.py

@@ -374,8 +374,6 @@ def log_user_access(self, access_type, cont_name=None, cont_id=None, multifile=F
 
                 for k,v in tree.iteritems():
                     context[k] = {'id': str(v['_id']), 'label': v.get('label')}
-                    if k == 'group':
-                        context[k]['label'] = v.get('name')
                     if k == 'subject':
                         context[k]['label'] = v.get('code')
             log_map['context'] = context