Add Django Easy Audit (#1081)

* WIP Debugging issue with django-easy-audit

* WIP narrowing down easyaudit bug

* WIP Continued work on easyaudit problem

* Finished adding easyaudit and related integration tests

* adjust expected queries for easyaudit

* remove crudevents and loginevents from the django admin site

* check number of groups in curdevents, not group ids

* Reduce num of queries in test based on performance-enchanced fork of easyaudit

access_audit/admin.py

@@ -0,0 +1,6 @@
+from easyaudit.models import CRUDEvent, LoginEvent
+from django.contrib import admin
+
+
+admin.site.unregister(CRUDEvent)
+admin.site.unregister(LoginEvent)

access_audit/helpers.py

@@ -0,0 +1,6 @@
+
+
+def dont_audit_fixture_loading(
+        instance, object_json_repr, created, raw, using, update_fields,
+        **kwargs):
+    return not raw

access_audit/middleware.py

@@ -0,0 +1,8 @@
+from intake.middleware import MiddlewareBase
+from easyaudit.middleware.easyaudit import clear_request
+
+
+class ClearRequestMiddleware(MiddlewareBase):
+
+    def process_response(self, response):
+        clear_request()

access_audit/tests/test_middleware.py

@@ -0,0 +1,25 @@
+from django.test import TestCase
+from user_accounts.tests.factories import UserProfileFactory
+from django.conf import settings
+from easyaudit.middleware.easyaudit import (
+    get_current_request, get_current_user)
+
+
+class TestMiddleware(TestCase):
+    """
+    easyaudit does not properly clear requests after they're finished
+    This means that users can be misattributed to subsequent actions.
+    Our middleware forces the request to be cleared.
+    """
+
+    def test_request_is_cleared(self):
+        # log in user
+        user = UserProfileFactory().user
+        self.client.login(
+            username=user.username, password=settings.TEST_USER_PASSWORD)
+
+        # do action
+        self.client.get('/')
+        # check if request is cleared
+        self.assertIsNone(get_current_request())
+        self.assertIsNone(get_current_user())

features/easy_audit.feature

@@ -0,0 +1,15 @@
+Feature: CRUD events are audited
+  Background:
+    Given a superuser
+      And an org user at "ebclc"
+
+  Scenario: Admin actions create CRUD events
+    Given I log in as a superuser
+     When I go to the admin edit page for "ebclc" user
+      And I check "is_staff"
+      And I select the "followup_staff" option in "groups_old"
+      And I click "a#id_groups_add_link"
+      And I click "input[name='_save']"
+     Then the latest "auth.User" "update" event should have "superuser" as the user
+      And the latest "auth.User" "update" event should have "True" for "is_staff"
+      And the latest "auth.User" "m2m_change" event should have "2" ids in "groups"

features/steps/audit_steps.py

@@ -0,0 +1,67 @@
+import json
+from behave import when, then
+from easyaudit.models import CRUDEvent
+from django.contrib.auth.models import User
+from django.contrib.contenttypes.models import ContentType
+from django.core.urlresolvers import reverse
+from pprint import pprint
+from urllib.parse import urljoin
+
+
+def get_latest_crudevent_for_appmodel_and_event_type(content_type, event_type):
+    # acceptable values are:
+    #   "create", "update", "delete", "m2m_change", "m2m_change_rev"
+    event_type_idx = getattr(CRUDEvent, event_type.upper())
+    content_type_app, content_type_model = content_type.lower().split(".")
+    event_model_content_type_id = ContentType.objects.filter(
+        app_label=content_type_app,
+        model=content_type_model).first().id
+    return CRUDEvent.objects.filter(
+        event_type=event_type_idx,
+        content_type_id=event_model_content_type_id).latest('datetime')
+
+
+@when('I go to the admin edit page for "{org_slug}" user')
+def visit_org_user_admin_edit_page(context, org_slug):
+    user = User.objects.filter(profile__organization__slug=org_slug).first()
+    url = reverse('admin:auth_user_change', args=[user.id])
+    context.browser.get(urljoin(context.test.live_server_url, url))
+
+
+@then('the latest CRUD event should not have a user')
+def test_latest_crud_event_no_user(context):
+    badevent = CRUDEvent.objects.latest('datetime')
+    print(vars(badevent))
+    context.test.assertIsNone(badevent.user)
+
+
+@then(str(
+    'the latest "{content_type}" "{event_type}" event should have "{username}"'
+    ' as the user'))
+def test_crudevent_username(context, content_type, event_type, username):
+    expected_event = get_latest_crudevent_for_appmodel_and_event_type(
+        content_type, event_type)
+    context.test.assertEqual(username, expected_event.user.username)
+
+
+@then(str(
+    'the latest "{content_type}" "{event_type}" event should have "{value}"'
+    ' for "{key}"'))
+def test_crudevent_has_matching_property(
+        context, content_type, event_type, value, key):
+    expected_event = get_latest_crudevent_for_appmodel_and_event_type(
+        content_type, event_type)
+    event_data = json.loads(expected_event.object_json_repr)[0]
+    context.test.assertEqual(value, str(event_data["fields"][key]))
+
+
+@then(str(
+    'the latest "{content_type}" "{event_type}" event should have '
+    '"{id_count}" ids in "{field_name}"'))
+def test_crudevent_property_contains(
+        context, content_type, event_type, id_count, field_name):
+    expected_event = get_latest_crudevent_for_appmodel_and_event_type(
+        content_type, event_type)
+    event_data = json.loads(expected_event.object_json_repr)[0]
+    context.test.assertEqual(
+        int(id_count), len(event_data["fields"][field_name]))

features/steps/form_input_steps.py

@@ -17,7 +17,7 @@ def click_submit_button(context, form_selector=''):
 
 @when('"{checkbox_value}" is clicked on the "{checkbox_name}" radio button')
 @when('the "{checkbox_name}" checkbox option "{checkbox_value}" is clicked')
-def click_checkbox(context, checkbox_name, checkbox_value):
+def click_checkbox_choice(context, checkbox_name, checkbox_value):
     selector = "input[name='%s'][value='%s']" % (
         checkbox_name,
         checkbox_value,
@@ -49,3 +49,33 @@ def type_in_email_input(context, input_name, value):
     selector = "input[name='{}'][type='email']".format(input_name)
     text = context.browser.find_element_by_css_selector(selector)
     text.send_keys(value)
+
+
+@when('I check "{checkbox_name}"')
+def click_checkbox(context, checkbox_name):
+    selector = "input[name='{}']".format(checkbox_name)
+    checkbox = context.browser.find_element_by_css_selector(selector)
+    checkbox.click()
+
+
+@when('I click "{css_selector}"')
+def click_element(context, css_selector):
+    element = context.browser.find_element_by_css_selector(css_selector)
+    element.click()
+
+
+@when('I select the "{option_text}" option in "{select_name}"')
+def double_click(context, option_text, select_name):
+    selector = "select[name={}] option".format(select_name)
+    option_elements = context.browser.find_elements_by_css_selector(selector)
+    selected_options = [
+        element for element in option_elements
+        if option_text in element.text]
+    if len(selected_options) != 1:
+        raise Exception(
+            str(
+                'Could not find one single option with {} in text.\n'
+                'found: {}').format(
+                    option_text, [element.text for element in option_elements])
+        )
+    selected_options[0].click()

features/steps/user_accounts_steps.py

@@ -45,3 +45,16 @@ def login_as_applicant_support_user(context):
 def login_as_org_user(context, org_slug='ebclc'):
     login_as(
         context, "bgolder+demo+{}_user@codeforamerica.org".format(org_slug))
+
+
+@given('a superuser')
+def load_superuser(context):
+    org = Organization.objects.get(slug='cfa')
+    factories.profile_for_org_and_group_names(
+        org, group_names=[groups.FOLLOWUP_STAFF, groups.APPLICATION_REVIEWERS],
+        is_staff=True, is_superuser=True, username='superuser')
+
+
+@given('I log in as a superuser')
+def login_as_superuser(context):
+    login_as(context, "bgolder+demo+superuser@codeforamerica.org")

intake/tests/factories/__init__.py

@@ -19,6 +19,8 @@
     make_app_ids_for_sf,
     apps_queryset,
 )
+from .tag_factory import TagFactory
+from .submission_tag_link_factory import SubmissionTagLinkFactory
 
 
 __all__ = [
@@ -37,5 +39,7 @@
     StatusNotificationFactory,
     FillablePDFFactory,
     FilledPDFFactory,
-    PrebuiltPDFBundleFactory
+    PrebuiltPDFBundleFactory,
+    TagFactory,
+    SubmissionTagLinkFactory
 ]

intake/tests/factories/submission_tag_link_factory.py

@@ -0,0 +1,14 @@
+import factory
+from intake import models
+from user_accounts.tests.factories import UserFactory
+from .tag_factory import TagFactory
+from .form_submission_factory import FormSubmissionFactory
+
+
+class SubmissionTagLinkFactory(factory.DjangoModelFactory):
+    content_object = factory.SubFactory(FormSubmissionFactory)
+    tag = factory.SubFactory(TagFactory)
+    user = factory.SubFactory(UserFactory)
+
+    class Meta:
+        model = models.SubmissionTagLink

intake/tests/factories/tag_factory.py

@@ -0,0 +1,9 @@
+import factory
+from taggit import models
+
+
+class TagFactory(factory.DjangoModelFactory):
+    name = factory.Sequence(lambda n: 'tag-{}'.format(n))
+
+    class Meta:
+        model = models.Tag

intake/tests/models/test_tag.py

@@ -1,65 +1,49 @@
 from django.test import TestCase
-from intake.tests import mock
-from user_accounts.tests.mock import create_user
+from intake.tests.factories import SubmissionTagLinkFactory
+from intake import models
 from taggit.models import Tag
 from django.contrib.auth.models import User
-from intake import models
 
 
 class TestSubmissionTagLink(TestCase):
 
-    def setUp(self):
-        super().setUp()
-        self.sub = mock.make_submission()
-        self.sub_id = self.sub.id
-        self.user = create_user()
-        self.user_id = self.user.id
-        self.tag = mock.make_tag()
-        self.tag_id = self.tag.id
-
-    def make_link(self):
-        link = models.SubmissionTagLink(
-            content_object_id=self.sub_id, user_id=self.user_id,
-            tag_id=self.tag_id)
-        link.save()
-        return link
-
     def test_creation(self):
-        link = self.make_link()
+        link = SubmissionTagLinkFactory()
         self.assertTrue(link.added)
 
     def test_related_queries(self):
-        link = self.make_link()
-        results = self.sub.tags.all()
-        self.assertIn(self.tag, results)
-        self.assertIn(link, self.tag.intake_submissiontaglink_items.all())
-        self.assertIn(link, self.sub.tag_links.all())
+        link = SubmissionTagLinkFactory()
+        results = link.content_object.tags.all()
+        self.assertIn(link.tag, results)
+        self.assertIn(link, link.tag.intake_submissiontaglink_items.all())
+        self.assertIn(link, link.content_object.tag_links.all())
         results = models.FormSubmission.objects.filter(
-            tags__name=self.tag.name)
-        self.assertIn(self.sub, results)
+            tags__name=link.tag.name)
+        self.assertIn(link.content_object, results)
 
     def test_deletion(self):
-        link = self.make_link()
+        link = SubmissionTagLinkFactory()
         # make sure related objects still exist after deletion
         link.delete()
         self.assertEqual(
-            models.FormSubmission.objects.filter(id=self.sub_id).count(), 1)
+            models.FormSubmission.objects.filter(
+                id=link.content_object_id).count(), 1)
         self.assertEqual(
-            Tag.objects.filter(id=self.tag_id).count(), 1)
+            Tag.objects.filter(id=link.tag_id).count(), 1)
         self.assertEqual(
-            User.objects.filter(id=self.user_id).count(), 1)
-        self.assertEqual(self.sub.tags.all().count(), 0)
+            User.objects.filter(id=link.user_id).count(), 1)
+        self.assertEqual(link.content_object.tags.all().count(), 0)
 
     def test_tag_deletion(self):
         # Deletes link if tag is deleted
-        link_id = self.make_link().id
-        Tag.objects.filter(id=self.tag_id).delete()
+        link = SubmissionTagLinkFactory()
+        Tag.objects.filter(id=link.tag_id).delete()
         self.assertEqual(
-            models.SubmissionTagLink.objects.filter(id=link_id).count(), 0)
+            models.SubmissionTagLink.objects.filter(id=link.id).count(), 0)
 
     def test_user_deletion(self):
         # Link remains if user is deleted, with link.user = None
-        link_id = self.make_link().id
-        User.objects.filter(id=self.user_id).delete()
-        link = models.SubmissionTagLink.objects.get(id=link_id)
+        link = SubmissionTagLinkFactory()
+        User.objects.filter(id=link.user_id).delete()
+        link = models.SubmissionTagLink.objects.get(id=link.id)
         self.assertEqual(link.user, None)

intake/tests/services/test_transfers_service.py

@@ -43,6 +43,6 @@ def test_expected_number_of_queries(self):
         to_org = Organization.objects.get(slug='ebclc')
         application = models.Application.objects.filter(
             organization__slug='a_pubdef').first()
-        with self.assertNumQueries(10):
+        with self.assertNumQueries(19):
             TransferService.transfer_application(
                 user, application, to_org, 'there was a temporal anomaly')

project/settings/base.py

@@ -44,6 +44,8 @@
     'behave_django',
     'favicons',
     'clips',
+    'easyaudit',
+    'access_audit',
 ]
 
 MIDDLEWARE = [
@@ -60,7 +62,9 @@
     'intake.middleware.PersistReferrerMiddleware',
     'intake.middleware.PersistSourceMiddleware',
     'intake.middleware.GetCleanIpAddressMiddleware',
-    'intake.middleware.CountUniqueVisitorsMiddleware'
+    'intake.middleware.CountUniqueVisitorsMiddleware',
+    'access_audit.middleware.ClearRequestMiddleware',
+    'easyaudit.middleware.easyaudit.EasyAuditMiddleware',
 ]
 
 ROOT_URLCONF = 'project.urls'
@@ -129,6 +133,10 @@
 WSGI_APPLICATION = 'project.wsgi.application'
 
 MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage'
+
+# django-easy-audit settings
+DJANGO_EASY_AUDIT_WATCH_LOGIN_EVENTS = True
+
 # django-allauth and django-invitations
 ACCOUNT_FORMS = {
     'login': 'user_accounts.forms.LoginForm'
@@ -245,3 +253,19 @@ def COMPRESS_JINJA2_GET_ENVIRONMENT():
         },
     },
 }
+
+
+DJANGO_EASY_AUDIT_CRUD_DIFFERENCE_CALLBACKS = [
+    'access_audit.helpers.dont_audit_fixture_loading']
+
+DJANGO_EASY_AUDIT_REGISTERED_CLASSES = [
+    'auth.User',
+    'auth.Group',
+    'user_accounts.UserProfile',
+    'invitations.Invitation',
+    'intake.FormSubmission',
+    'intake.Application',
+    'intake.StatusUpdate',
+    'intake.ApplicationTransfer',
+    'intake.ApplicationNote'
+]

requirements/app.txt

@@ -27,6 +27,7 @@ amqp==2.1.4
 PyYAML==3.12
 user-agents==1.1.0
 ua-parser==0.7.3
+https://github.com/rossettistone/django-easy-audit/archive/patched.tar.gz
 
 # factory-boy breaks on later version fo fake-factory
 # we need to pin fake-factory