Merge pull request #2271 from WikiWatershed/arr/api-logging

Log Geoprocessing API Requests

src/mmw/apps/core/decorators.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import sys
+import rollbar
+
+from django.utils.timezone import now
+from django.conf import settings
+
+from apps.core.models import RequestLog
+
+
+rollbar_settings = getattr(settings, 'ROLLBAR', {})
+if rollbar_settings:
+    rollbar.init(rollbar_settings.get('access_token'),
+                 rollbar_settings.get('environment'))
+
+
+def log_request(view):
+    """
+    Log the request and its response as a RequestLog model
+    """
+
+    def decorator(request, *args, **kwargs):
+        requested_at = now()
+
+        view_result = view(request, *args, **kwargs)
+
+        user = request.user if request.user.is_authenticated() else None
+
+        response_time = now() - requested_at
+        response_ms = int(response_time.total_seconds() * 1000)
+
+        log = RequestLog.objects.create(
+            user=user,
+            job_uuid=view_result.data.get('job', None),
+            requested_at=requested_at,
+            response_ms=response_ms,
+            status_code=view_result.status_code,
+            path=request.path,
+            query_params=request.query_params.dict(),
+            method=request.method,
+            host=request.get_host(),
+            remote_addr=_get_remote_addr(request))
+        try:
+            log.save()
+        except Exception:
+            if rollbar_settings:
+                rollbar.report_exc_info(sys.exc_info(), request)
+
+        return view_result
+
+    decorator.__name__ = view.__name__
+    decorator.__dict__ = view.__dict__
+    decorator.__doc__ = view.__doc__
+
+    return decorator
+
+
+def _get_remote_addr(request):
+    # get IP
+    ipaddr = request.META.get("HTTP_X_FORWARDED_FOR", None)
+    if ipaddr:
+        # X_FORWARDED_FOR returns client1, proxy1, proxy2,...
+            return [x.strip() for x in ipaddr.split(",")][0]
+    else:
+        return request.META.get("REMOTE_ADDR", "")

src/mmw/apps/core/migrations/0002_requestlog.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+from django.conf import settings
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RequestLog',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('job_uuid', models.UUIDField(help_text='If async request, the uuid of the submitted job', null=True)),
+                ('requested_at', models.DateTimeField(help_text='When the request was made')),
+                ('response_ms', models.PositiveIntegerField(help_text='How long the response took in ms')),
+                ('path', models.CharField(help_text='The requested URI', max_length=400)),
+                ('query_params', models.TextField(help_text='Stringified requested query params dictionary', null=True)),
+                ('status_code', models.PositiveIntegerField(help_text='HTTP status code sent back')),
+                ('method', models.CharField(help_text='HTTP method', max_length=255)),
+                ('host', models.URLField(help_text='The host from which the request was sent')),
+                ('remote_addr', models.GenericIPAddressField(help_text='The IP address from which the request was sent')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)),
+            ],
+        ),
+    ]

src/mmw/apps/core/models.py

@@ -21,3 +21,34 @@ class Job(models.Model):
 
     def __unicode__(self):
         return unicode(self.uuid)
+
+
+class RequestLog(models.Model):
+    user = models.ForeignKey(AUTH_USER_MODEL,
+                             on_delete=models.SET_NULL,
+                             null=True)
+    job_uuid = models.UUIDField(
+        null=True,
+        help_text='If async request, the uuid of the submitted job')
+    requested_at = models.DateTimeField(
+        help_text='When the request was made')
+    response_ms = models.PositiveIntegerField(
+        help_text='How long the response took in ms')
+    path = models.CharField(
+        max_length=400,
+        help_text='The requested URI')
+    query_params = models.TextField(
+        null=True,
+        help_text='Stringified requested query params dictionary')
+    status_code = models.PositiveIntegerField(
+        help_text='HTTP status code sent back')
+    method = models.CharField(
+        max_length=255,
+        help_text='HTTP method')
+    host = models.URLField(
+        help_text='The host from which the request was sent')
+    remote_addr = models.GenericIPAddressField(
+        help_text='The IP address from which the request was sent')
+
+    def __unicode__(self):
+        return self.user + " " + self.path

src/mmw/apps/geoprocessing_api/views.py

@@ -16,6 +16,7 @@
 from apps.core.tasks import (save_job_error,
                              save_job_result)
 from apps.core.permissions import IsTokenAuthenticatedOrNotSwagger
+from apps.core.decorators import log_request
 from apps.modeling import geoprocessing
 from apps.modeling.views import load_area_of_interest
 from apps.geoprocessing_api import tasks
@@ -24,6 +25,7 @@
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_rwd(request, format=None):
     """
     Starts a job to run Rapid Watershed Delineation on a point-based location.
@@ -185,6 +187,7 @@ def start_rwd(request, format=None):
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_analyze_land(request, format=None):
     """
     Starts a job to produce a land-use histogram for a given area.
@@ -379,6 +382,7 @@ def start_analyze_land(request, format=None):
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_analyze_soil(request, format=None):
     """
     Starts a job to produce a soil-type histogram for a given area.
@@ -505,6 +509,7 @@ def start_analyze_soil(request, format=None):
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_analyze_animals(request, format=None):
     """
     Starts a job to produce counts for animals in a given area.
@@ -615,6 +620,7 @@ def start_analyze_animals(request, format=None):
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_analyze_pointsource(request, format=None):
     """
     Starts a job to analyze the discharge monitoring report annual
@@ -704,6 +710,7 @@ def start_analyze_pointsource(request, format=None):
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_analyze_catchment_water_quality(request, format=None):
     """
     Starts a job to calculate the calibrated GWLF-E (MapShed) model
@@ -822,6 +829,7 @@ def start_analyze_catchment_water_quality(request, format=None):
 @decorators.api_view(['POST'])
 @decorators.authentication_classes((TokenAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def start_analyze_climate(request, format=None):
     """
     Start a job to calculate the monthly climate (precipitation

src/mmw/apps/modeling/views.py

@@ -28,6 +28,7 @@
 from apps.core.models import Job
 from apps.core.tasks import save_job_error, save_job_result
 from apps.core.permissions import IsTokenAuthenticatedOrNotSwagger
+from apps.core.decorators import log_request
 from apps.modeling import tasks, geoprocessing
 from apps.modeling.mapshed.tasks import (geoprocessing_chains,
                                          combine,
@@ -488,6 +489,7 @@ def drb_point_sources(request):
 @decorators.authentication_classes((TokenAuthentication,
                                     SessionAuthentication, ))
 @decorators.permission_classes((IsTokenAuthenticatedOrNotSwagger, ))
+@log_request
 def get_job(request, job_uuid, format=None):
     """
     Get a job's status. If it's complete, get its result.