Merge branch 'upgrade-django-1.10.4' into rest-api

teamtemp/responses/forms.py

@@ -119,7 +119,7 @@ def __init__(self, *args, **kwargs):
 
     def clean_team_name(self):
         team_name = self.cleaned_data['team_name']
-        matches = re.findall(r'[^A-Za-z0-9 \'-]', team_name)
+        matches = re.findall(r'[^\w-]', team_name)
         if matches:
             error = '"{team_name}" contains invalid characters '\
                     '{matches}'.format(team_name=escape(team_name), matches=list({str(x) for x in matches}))
@@ -189,7 +189,7 @@ def clean_word(self):
             error = 'Max {max_word_count} Words'.format(max_word_count=escape(self.max_word_count))
             raise forms.ValidationError(error)
 
-        return word
+        return word.lower()
 
 class ResultsPasswordForm(forms.Form):
     error_css_class='error box'

teamtemp/responses/migrations/0004_auto_20160617_1324.py

@@ -40,7 +40,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='wordcloudimage',
             name='word_list',
-            field=models.CharField(db_index=True, max_length=5000),
+            field=models.CharField(max_length=5000),
         ),
         migrations.AlterUniqueTogether(
             name='teams',

teamtemp/responses/migrations/0005_lowercase_words.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.4 on 2016-12-12 01:35
+from __future__ import unicode_literals
+
+from django.db import migrations
+from django.db.models import F, Func
+
+def lowercase_word_cloud_image_word_list(apps, schema_editor):
+    WordCloudImage = apps.get_model('responses', 'WordCloudImage')
+    WordCloudImage.objects.annotate(
+        word_list_lower=Func(F('word_list'), function='LOWER')
+    ).all().update(word_list=F('word_list_lower'))
+
+def lowercase_team_response_history_word_list(apps, schema_editor):
+    TeamResponseHistory = apps.get_model('responses', 'TeamResponseHistory')
+    TeamResponseHistory.objects.annotate(
+        word_list_lower=Func(F('word_list'), function='LOWER')
+    ).all().update(word_list=F('word_list_lower'))
+
+def lowercase_temperature_response_word(apps, schema_editor):
+    TemperatureResponse = apps.get_model('responses', 'TemperatureResponse')
+    TemperatureResponse.objects.annotate(
+        word_lower=Func(F('word'), function='LOWER')
+    ).all().update(word=F('word_lower'))
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('responses', '0004_auto_20160617_1324'),
+    ]
+
+    operations = [
+        migrations.RunPython(lowercase_word_cloud_image_word_list),
+        migrations.RunPython(lowercase_team_response_history_word_list),
+        migrations.RunPython(lowercase_temperature_response_word),
+    ]

teamtemp/responses/models.py

@@ -3,12 +3,15 @@
 
 class WordCloudImage(models.Model):
     creation_date = models.DateField()
-    word_list = models.CharField(max_length=5000, db_index=True)
+    word_list = models.CharField(max_length=5000)
     image_url = models.CharField(max_length=255)
 
     def __unicode__(self):
         return u"{} {} {}".format(self.creation_date, self.word_list, self.image_url)
 
+    def clean(self):
+        self.word_list = self.word_list.lower()
+
 
 class User(models.Model):
     id = models.CharField(max_length=8, primary_key=True)
@@ -92,6 +95,10 @@ def __unicode__(self):
                                          self.archived, self.response_date,
                                          self.archive_date)
 
+    def clean(self):
+        self.word = self.word.lower()
+
+
 class TeamResponseHistory(models.Model):
     class Meta:
         verbose_name_plural = "Team response histories"
@@ -109,6 +116,10 @@ def __unicode__(self):
                                          self.word_list, self.responder_count,
                                          self.team_name, self.archive_date)
 
+    def clean(self):
+        self.word_list = self.word_list.lower()
+
+
 class Teams(models.Model):
     class Meta:
         verbose_name_plural = "Teams"

teamtemp/views.py

@@ -440,18 +440,34 @@ def require_dir(path):
         if exc.errno != errno.EEXIST:
             raise
 
+def media_dir(directory):
+    media_directory = os.path.join(settings.MEDIA_ROOT, directory) 
+    require_dir(media_directory)
+    return media_directory
+
+def media_basename(src):
+    name = urlparse(src).path.split('/')[-1]
+    return name
+
+def media_url(src, directory):
+    image_name = media_basename(src)
+    url = os.path.join(settings.MEDIA_URL, os.path.join(directory, image_name))
+    return url
+
+def media_file(src, directory):
+    image_name = media_basename(src)
+    media_directory = media_dir(directory)
+    filename = os.path.join(media_directory, image_name)
+    return filename
+
 def save_url(url, directory):
+    return_url = media_url(url, directory)
+    filename = media_file(url, directory)
 
-    image_name = urlparse(url).path.split('/')[-1]
-    return_url = os.path.join(settings.MEDIA_URL,os.path.join(directory,image_name))
-    if settings.MEDIA_ROOT:
-        directory = os.path.join(settings.MEDIA_ROOT, directory) 
-    else:
-        directory = os.path.join(settings.BASE_DIR, directory)
-    require_dir(directory)
-    filename = os.path.join(directory, image_name)
+    print >>sys.stderr, str(timezone.now()) + " Saving Word Cloud: " + url + " as " + filename + " (" + return_url +")"
 
     if not os.path.exists(filename):
+        print >>sys.stderr, str(timezone.now()) + " Saving Word Cloud: " + filename + " doesn't exist"
         urllib.urlretrieve(url, filename)
        #TODO if error return None
 
@@ -501,10 +517,25 @@ def cron(request, pin):
 
     if pin == cron_pin:
         auto_archive_surveys(request)
+        prune_word_cloud_cache()
         return HttpResponse()
     else:
         raise Http404
 
+def prune_word_cloud_cache(request):
+    timezone.activate(pytz.timezone('UTC'))
+    print >>sys.stderr,"prune_word_cloud_cache: Start at " + str(timezone.localtime(timezone.now())) + " UTC"
+
+    yesterday = datetime.now() + timedelta(days=-1)
+
+    WordCloudImage.objects.filter(creation_date__lte = yesterday).delete()
+
+    for word_cloud in WordCloudImage.objects.all():
+        if not os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), word_cloud.image_url)):
+            word_cloud.delete()
+
+    print >>sys.stderr,"prune_word_cloud_cache: Stop at " + str(timezone.localtime(timezone.now())) + " UTC"
+
 def auto_archive_surveys(request):
     timezone.activate(pytz.timezone('UTC'))
     print >>sys.stderr,"auto_archive_surveys: Start at " + str(timezone.localtime(timezone.now())) + " UTC"
@@ -882,23 +913,31 @@ def cached_word_cloud(word_list):
         for i in range(0,word['id__count']):
             words = words + word['word'] + " "
             word_count += 1
+
+    words = words.lower()
 
+    if words == "":
+        return None
+
     #TODO Write a better lookup and model to replace this hack
-    word_cloud_index = WordCloudImage.objects.filter(word_list = words)
+    word_cloud_index = WordCloudImage.objects.filter(word_list = words).order_by('-id')
 
     if word_cloud_index:
-        if os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), word_cloud_index[0].image_url)):
-            word_cloudurl =  word_cloud_index[0].image_url
+        filename = media_file(word_cloud_index[0].image_url, 'wordcloud_images')
+        if os.path.isfile(filename):
+            print >>sys.stderr, str(timezone.now()) + " Cached Word Cloud: " + filename + " found"
+            return word_cloud_index[0].image_url
         else:
+            print >>sys.stderr, str(timezone.now()) + " Cached Word Cloud: " + filename + " doesn't exist"
             #Files have been deleted remove from db and then regenerate
-            WordCloudImage.objects.filter(word_list = words).delete()
+            word_cloud_index.delete()
 
-    if word_cloudurl == "" and words != "":
-        word_cloudurl = generate_wordcloud(words)
-        if word_cloudurl:
-            word_cloud = WordCloudImage(creation_date = timezone.now(),
-                                        word_list = words, image_url = word_cloudurl)
-            word_cloud.save()
+    word_cloudurl = generate_wordcloud(words)
+    if word_cloudurl:
+        word_cloud = WordCloudImage(creation_date = timezone.now(),
+                                    word_list = words, image_url = word_cloudurl)
+        word_cloud.save()
+
     return word_cloudurl
 
 def generate_bvc_stats(survey_id_list, team_name, archive_date, num_iterations):