Merge 28b7fab into a577ea5

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501

prs2/harvester/models.py

@@ -3,17 +3,11 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.gis.geos import GEOSGeometry, Point
-from django.core.files import File
+from django.core.files.base import ContentFile
 from django.db import models
 from django.utils.encoding import python_2_unicode_compatible
-import json
 import logging
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO  # Python3
 from shapely.geometry import Polygon
-import sys
 import xmltodict
 
 from referral.models import (
@@ -298,7 +292,7 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True,
             new_record = Record.objects.create(
                 name=self.subject, referral=new_ref, order_date=datetime.today())
             file_name = 'emailed_referral_{}.html'.format(reference)
-            new_file = File(StringIO(self.body))
+            new_file = ContentFile(self.body)
             new_record.uploaded_file.save(file_name, new_file)
             new_record.save()
             logger.info('New PRS record generated: {}'.format(new_record))
@@ -309,11 +303,7 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True,
                 new_record = Record.objects.create(
                     name=i.name, referral=new_ref, order_date=datetime.today())
                 # Duplicate the uploaded file.
-                if sys.version_info > (3, 0):
-                    data = StringIO(i.attachment.read().decode('utf-8'))
-                else:
-                    data = StringIO(i.attachment.read())
-                new_file = File(data)
+                new_file = ContentFile(i.attachment.read())
                 new_record.uploaded_file.save(i.name, new_file)
                 new_record.save()
                 logger.info('New PRS record generated: {}'.format(new_record))

prs2/harvester/utils.py

@@ -3,7 +3,7 @@
 from confy import env
 from datetime import date, datetime
 from django.conf import settings
-from django.core.files import File
+from django.core.files.base import ContentFile
 from django.core.mail import EmailMultiAlternatives
 import email
 from imaplib import IMAP4_SSL
@@ -38,14 +38,17 @@ def fetch_email(imap, uid):
     Email is returned as an email.Message class object.
     """
     message = None
-    status, response = imap.fetch(str(uid), '(BODY.PEEK[])')
+    status, response = imap.fetch(uid, '(BODY.PEEK[])')
 
     if status != 'OK':
         return status, response
 
     for i in response:
         if isinstance(i, tuple):
-            message = email.message_from_string(i[1])
+            s = i[1]
+            if isinstance(s, bytes):
+                s = s.decode('utf-8')
+            message = email.message_from_string(s)
 
     return status, message
 
@@ -65,20 +68,12 @@ def harvest_email(uid, message):
 
     message_body = None
     attachments = []
-    # Build the whitelist of receiving mailboxes (we only harvest messages
-    # sent to these addresses).
-    # Whitelist should consist of a .txt file, one email per line.
-    try:
-        f = open('mailbox_whitelist.txt', 'r')
-        whitelist = [i.strip() for i in f.readlines()]
-    except:
-        whitelist = False
 
     for p in parts:
         # 'text/html' content is the email body.
         if p.get_content_type() == 'text/html':
             message_body = p
-        # Other content types (not multipart/mixed) are attachments.
+        # Other content types (not multipart/mixed) are attachments (normally application/octet-stream).
         elif p.get_content_type() != 'multipart/mixed':
             attachments.append(p)
 
@@ -87,7 +82,7 @@ def harvest_email(uid, message):
         try:
             # Check the 'To' address against the whitelist of mailboxes.
             to_e = email.utils.parseaddr(message.get('To'))[1]
-            if whitelist and not to_e.lower() in whitelist:
+            if not to_e.lower() in settings.ASSESSOR_EMAILS:
                 LOGGER.info('Email UID {} to {} harvest was skipped'.format(uid, to_e))
                 return None  # Not in the whitelist; skip.
             from_e = email.utils.parseaddr(message.get('From'))[1]
@@ -105,11 +100,13 @@ def harvest_email(uid, message):
             for a in attachments:
                 att_new = EmailAttachment(
                     emailed_referral=em_new, name=a.get_filename())
-                data = StringIO(base64.decodestring(a.get_payload()))
-                new_file = File(data)
+                try:
+                    data = a.get_payload(decode=True)
+                except:
+                    data = StringIO(base64.decodestring(a.get_payload()))
+                new_file = ContentFile(data)
                 att_new.attachment.save(a.get_filename(), new_file)
                 att_new.save()
-                data.close()
                 LOGGER.info('Email attachment created: {}'.format(att_new.name))
         except Exception as e:
             LOGGER.error('Email UID {} generated exception during harvest'.format(uid))
@@ -160,6 +157,9 @@ def harvest_unread_emails(from_email):
 
     if uids:
         for uid in uids:
+            # Decode uid to a string if required.
+            if isinstance(uid, bytes):
+                uid = uid.decode('utf-8')
             # Fetch email message.
             if EmailedReferral.objects.filter(email_uid=str(uid)).exists():
                 # Already harvested? Mark it as read.

prs2/settings.py

@@ -131,7 +131,10 @@
 REFERRAL_EMAIL_USER = env('REFERRAL_EMAIL_USER', 'referrals')
 REFERRAL_EMAIL_PASSWORD = env('REFERRAL_EMAIL_PASSWORD', 'password')
 REFERRAL_ASSIGNEE_FALLBACK = env('REFERRAL_ASSIGNEE_FALLBACK', 'admin')
-PLANNING_EMAILS = env('PLANNING_EMAILS', ['referrals@planning.wa.gov.au'])
+# Whitelist of sender emails (only harvest referrals sent by these):
+PLANNING_EMAILS = env('PLANNING_EMAILS', 'referrals@dplh.wa.gov.au').split(',')
+# Whitelist of receiving mailboxes (only harvest referrals sent to these):
+ASSESSOR_EMAILS = env('ASSESSOR_EMAILS', '').split(',')
 
 # Database configuration
 DATABASES = {

requirements.txt

@@ -24,6 +24,6 @@ django-webtest==1.9.2
 python-magic==0.4.15
 lxml==4.1.1
 django-downloadview==1.9
-openpyxl==2.5.0
+openpyxl==2.5.2
 xmltodict==0.11.0
 shapely==1.6.4