Implement disk space alerts. Closes #756.

client/utils.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import os
+
+def get_total_disk_space(p):
+    """
+    Calculate the total disk space of the device on which a given file path resides.
+    """
+    s = os.statvfs(p)
+    return s.f_frsize * s.f_blocks   
+
+def get_free_disk_space(p):
+    """
+    Returns the number of free bytes on the drive that ``p`` is on
+    """
+    s = os.statvfs(p)
+    return s.f_frsize * s.f_bavail
+

client/views.py

@@ -14,6 +14,7 @@
 from livesettings import config_value
 from tastypie.serializers import Serializer
 
+from client import utils
 from panda.api.category import CategoryResource
 from panda.models import ActivityLog, Category, Dataset, SearchLog, UserProxy
 
@@ -49,20 +50,6 @@ def index(request):
         })
     })
 
-def _get_total_disk_space(p):
-    """
-    Calculate the total disk space of the device on which a given file path resides.
-    """
-    s = os.statvfs(p)
-    return s.f_frsize * s.f_blocks   
-
-def _get_free_disk_space(p):
-    """
-    Returns the number of free bytes on the drive that ``p`` is on
-    """
-    s = os.statvfs(p)
-    return s.f_frsize * s.f_bavail
-
 def dashboard(request):
     """
     Render HTML for dashboard/metrics view.
@@ -143,22 +130,22 @@ def dashboard(request):
     upload_disk = os.stat(settings.MEDIA_ROOT).st_dev
     indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev
 
-    root_disk_total = _get_total_disk_space('/')
-    root_disk_free = _get_free_disk_space('/')
+    root_disk_total = utils.get_total_disk_space('/')
+    root_disk_free = utils.get_free_disk_space('/')
     root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100)
 
     if upload_disk != root_disk:    
-        upload_disk_total = _get_total_disk_space(settings.MEDIA_ROOT)
-        upload_disk_free = _get_free_disk_space(settings.MEDIA_ROOT)
+        upload_disk_total = utils.get_total_disk_space(settings.MEDIA_ROOT)
+        upload_disk_free = utils.get_free_disk_space(settings.MEDIA_ROOT)
         upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100)
     else:
         upload_disk_total = None
         upload_disk_free = None
         upload_disk_percent_used = None
 
     if indices_disk != root_disk:
-        indices_disk_total = _get_total_disk_space(settings.SOLR_DIRECTORY)
-        indices_disk_free = _get_free_disk_space(settings.SOLR_DIRECTORY)
+        indices_disk_total = utils.get_total_disk_space(settings.SOLR_DIRECTORY)
+        indices_disk_free = utils.get_free_disk_space(settings.SOLR_DIRECTORY)
         indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100)
     else:
         indices_disk_total = None

config/settings.py

@@ -143,6 +143,10 @@
     'run_subscriptions': {
         'task': 'panda.tasks.cron.run_subscriptions',
         'schedule': crontab(minute=30, hour=2)
+    },
+    'run_admin_alerts': {
+        'task': 'panda.tasks.cron.run_admin_alerts',
+        'schedule': crontab(minute=0, hour=4)
     }
 }
 

panda/tasks/__init__.py

@@ -8,6 +8,7 @@
 from panda.tasks.purge_data import PurgeDataTask
 from panda.tasks.purge_orphaned_uploads import PurgeOrphanedUploadsTask
 from panda.tasks.reindex import ReindexTask
+from panda.tasks.run_admin_alerts import RunAdminAlertsTask
 from panda.tasks.run_subscriptions import RunSubscriptionsTask
 
 TASKS_BY_TYPE = {

panda/tasks/run_admin_alerts.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+import logging
+import os
+
+from celery.task import Task
+from django.conf import settings
+from django.template import Context
+from livesettings import config_value
+
+from client.utils import get_total_disk_space, get_free_disk_space
+from panda.utils.mail import send_mail
+from panda.utils.notifications import get_email_subject_template, get_email_body_template
+
+class RunAdminAlertsTask(Task):
+    """
+    Notify administrators of anything which requires their attention (disk space, etc).
+    """
+    name = 'panda.tasks.cron.run_admin_alerts'
+
+    def run(self, *args, **kwargs):
+        from panda.models import UserProxy
+
+        log = logging.getLogger(self.name)
+        log.info('Running admin alerts')
+
+        # Disk space
+        root_disk = os.stat('/').st_dev
+        upload_disk = os.stat(settings.MEDIA_ROOT).st_dev
+        indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev
+
+        root_disk_total = get_total_disk_space('/')
+        root_disk_free = get_free_disk_space('/')
+        root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100)
+
+        if upload_disk != root_disk:    
+            upload_disk_total = get_total_disk_space(settings.MEDIA_ROOT)
+            upload_disk_free = get_free_disk_space(settings.MEDIA_ROOT)
+            upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100)
+        else:
+            upload_disk_total = None
+            upload_disk_free = None
+            upload_disk_percent_used = None
+
+        if indices_disk != root_disk:
+            indices_disk_total = get_total_disk_space(settings.SOLR_DIRECTORY)
+            indices_disk_free = get_free_disk_space(settings.SOLR_DIRECTORY)
+            indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100)
+        else:
+            indices_disk_total = None
+            indices_disk_free = None
+            indices_disk_percent_used = None
+
+        notify = False
+
+        for free in (root_disk_free, upload_disk_free, indices_disk_free):
+            if free is None:
+                continue
+
+            if free < settings.PANDA_AVAILABLE_SPACE_WARN:
+                notify = True
+
+        if notify:
+            context = Context({
+                'root_disk': root_disk,
+                'upload_disk': upload_disk,
+                'indices_disk': indices_disk,
+                'root_disk_total': root_disk_total,
+                'root_disk_free': root_disk_free,
+                'root_disk_percent_used': root_disk_percent_used,
+                'upload_disk_total': upload_disk_total,
+                'upload_disk_free': upload_disk_free,
+                'upload_disk_percent_used': upload_disk_percent_used,
+                'indices_disk_total': indices_disk_total,
+                'indices_disk_free': indices_disk_free,
+                'indices_disk_percent_used': indices_disk_percent_used,
+                'settings': settings,
+                'site_domain': config_value('DOMAIN', 'SITE_DOMAIN')
+            })
+
+            # Don't HTML escape plain-text emails
+            context.autoescape = False
+
+            email_subject = get_email_subject_template('disk_space_alert').render(context)
+            email_message = get_email_body_template('disk_space_alert').render(context)
+
+            recipients = UserProxy.objects.filter(is_superuser=True, is_active=True)
+
+            send_mail(email_subject.strip(), email_message, [r.email for r in recipients])
+
+        log.info('Finished running admin alerts')
+

panda/templates/notifications/disk_space_alert/email_body.txt

@@ -0,0 +1,6 @@
+{% if root_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA application is {% if root_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ root_disk_free|filesizeformat }} remains ({{ root_disk_percent_used|floatformat }}% full){% endif %}
+{% if upload_disk_total != None and upload_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA's uploads is {% if upload_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ upload_disk_free|filesizeformat }} remains ({{ upload_disk_percent_used|floatformat }}% full){% endif %}
+{% if indices_disk_total and indices_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA's search indices ({{ indices_disk }}) is {% if indices_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ indices_disk_free|filesizeformat }} remains ({{ indices_disk_percent_used|floatformat }}% full){% endif %}
+
+Your can always check your current disk usage on your PANDA Dashboard:
+http://{{ site_domain }}/#dashboard

panda/templates/notifications/disk_space_alert/email_subject.txt

@@ -0,0 +1 @@
+Low disk space!