Skip to content
Browse files

Implement disk space alerts. Closes #756.

  • Loading branch information...
1 parent 1445895 commit 647159ad408dffa2f29ba610740405c04eea7571 @onyxfish onyxfish committed
View
18 client/utils.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import os
+
+def get_total_disk_space(p):
+ """
+ Calculate the total disk space of the device on which a given file path resides.
+ """
+ s = os.statvfs(p)
+ return s.f_frsize * s.f_blocks
+
+def get_free_disk_space(p):
+ """
+ Returns the number of free bytes on the drive that ``p`` is on
+ """
+ s = os.statvfs(p)
+ return s.f_frsize * s.f_bavail
+
View
27 client/views.py
@@ -14,6 +14,7 @@
from livesettings import config_value
from tastypie.serializers import Serializer
+from client import utils
from panda.api.category import CategoryResource
from panda.models import ActivityLog, Category, Dataset, SearchLog, UserProxy
@@ -49,20 +50,6 @@ def index(request):
})
})
-def _get_total_disk_space(p):
- """
- Calculate the total disk space of the device on which a given file path resides.
- """
- s = os.statvfs(p)
- return s.f_frsize * s.f_blocks
-
-def _get_free_disk_space(p):
- """
- Returns the number of free bytes on the drive that ``p`` is on
- """
- s = os.statvfs(p)
- return s.f_frsize * s.f_bavail
-
def dashboard(request):
"""
Render HTML for dashboard/metrics view.
@@ -143,13 +130,13 @@ def dashboard(request):
upload_disk = os.stat(settings.MEDIA_ROOT).st_dev
indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev
- root_disk_total = _get_total_disk_space('/')
- root_disk_free = _get_free_disk_space('/')
+ root_disk_total = utils.get_total_disk_space('/')
+ root_disk_free = utils.get_free_disk_space('/')
root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100)
if upload_disk != root_disk:
- upload_disk_total = _get_total_disk_space(settings.MEDIA_ROOT)
- upload_disk_free = _get_free_disk_space(settings.MEDIA_ROOT)
+ upload_disk_total = utils.get_total_disk_space(settings.MEDIA_ROOT)
+ upload_disk_free = utils.get_free_disk_space(settings.MEDIA_ROOT)
upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100)
else:
upload_disk_total = None
@@ -157,8 +144,8 @@ def dashboard(request):
upload_disk_percent_used = None
if indices_disk != root_disk:
- indices_disk_total = _get_total_disk_space(settings.SOLR_DIRECTORY)
- indices_disk_free = _get_free_disk_space(settings.SOLR_DIRECTORY)
+ indices_disk_total = utils.get_total_disk_space(settings.SOLR_DIRECTORY)
+ indices_disk_free = utils.get_free_disk_space(settings.SOLR_DIRECTORY)
indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100)
else:
indices_disk_total = None
View
4 config/settings.py
@@ -143,6 +143,10 @@
'run_subscriptions': {
'task': 'panda.tasks.cron.run_subscriptions',
'schedule': crontab(minute=30, hour=2)
+ },
+ 'run_admin_alerts': {
+ 'task': 'panda.tasks.cron.run_admin_alerts',
+ 'schedule': crontab(minute=0, hour=4)
}
}
View
1 panda/tasks/__init__.py
@@ -8,6 +8,7 @@
from panda.tasks.purge_data import PurgeDataTask
from panda.tasks.purge_orphaned_uploads import PurgeOrphanedUploadsTask
from panda.tasks.reindex import ReindexTask
+from panda.tasks.run_admin_alerts import RunAdminAlertsTask
from panda.tasks.run_subscriptions import RunSubscriptionsTask
TASKS_BY_TYPE = {
View
92 panda/tasks/run_admin_alerts.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+import logging
+import os
+
+from celery.task import Task
+from django.conf import settings
+from django.template import Context
+from livesettings import config_value
+
+from client.utils import get_total_disk_space, get_free_disk_space
+from panda.utils.mail import send_mail
+from panda.utils.notifications import get_email_subject_template, get_email_body_template
+
+class RunAdminAlertsTask(Task):
+ """
+ Notify administrators of anything which requires their attention (disk space, etc).
+ """
+ name = 'panda.tasks.cron.run_admin_alerts'
+
+ def run(self, *args, **kwargs):
+ from panda.models import UserProxy
+
+ log = logging.getLogger(self.name)
+ log.info('Running admin alerts')
+
+ # Disk space
+ root_disk = os.stat('/').st_dev
+ upload_disk = os.stat(settings.MEDIA_ROOT).st_dev
+ indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev
+
+ root_disk_total = get_total_disk_space('/')
+ root_disk_free = get_free_disk_space('/')
+ root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100)
+
+ if upload_disk != root_disk:
+ upload_disk_total = get_total_disk_space(settings.MEDIA_ROOT)
+ upload_disk_free = get_free_disk_space(settings.MEDIA_ROOT)
+ upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100)
+ else:
+ upload_disk_total = None
+ upload_disk_free = None
+ upload_disk_percent_used = None
+
+ if indices_disk != root_disk:
+ indices_disk_total = get_total_disk_space(settings.SOLR_DIRECTORY)
+ indices_disk_free = get_free_disk_space(settings.SOLR_DIRECTORY)
+ indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100)
+ else:
+ indices_disk_total = None
+ indices_disk_free = None
+ indices_disk_percent_used = None
+
+ notify = False
+
+ for free in (root_disk_free, upload_disk_free, indices_disk_free):
+ if free is None:
+ continue
+
+ if free < settings.PANDA_AVAILABLE_SPACE_WARN:
+ notify = True
+
+ if notify:
+ context = Context({
+ 'root_disk': root_disk,
+ 'upload_disk': upload_disk,
+ 'indices_disk': indices_disk,
+ 'root_disk_total': root_disk_total,
+ 'root_disk_free': root_disk_free,
+ 'root_disk_percent_used': root_disk_percent_used,
+ 'upload_disk_total': upload_disk_total,
+ 'upload_disk_free': upload_disk_free,
+ 'upload_disk_percent_used': upload_disk_percent_used,
+ 'indices_disk_total': indices_disk_total,
+ 'indices_disk_free': indices_disk_free,
+ 'indices_disk_percent_used': indices_disk_percent_used,
+ 'settings': settings,
+ 'site_domain': config_value('DOMAIN', 'SITE_DOMAIN')
+ })
+
+ # Don't HTML escape plain-text emails
+ context.autoescape = False
+
+ email_subject = get_email_subject_template('disk_space_alert').render(context)
+ email_message = get_email_body_template('disk_space_alert').render(context)
+
+ recipients = UserProxy.objects.filter(is_superuser=True, is_active=True)
+
+ send_mail(email_subject.strip(), email_message, [r.email for r in recipients])
+
+ log.info('Finished running admin alerts')
+
View
6 panda/templates/notifications/disk_space_alert/email_body.txt
@@ -0,0 +1,6 @@
+{% if root_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA application is {% if root_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ root_disk_free|filesizeformat }} remains ({{ root_disk_percent_used|floatformat }}% full){% endif %}
+{% if upload_disk_total != None and upload_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA's uploads is {% if upload_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ upload_disk_free|filesizeformat }} remains ({{ upload_disk_percent_used|floatformat }}% full){% endif %}
+{% if indices_disk_total and indices_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA's search indices ({{ indices_disk }}) is {% if indices_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ indices_disk_free|filesizeformat }} remains ({{ indices_disk_percent_used|floatformat }}% full){% endif %}
+
+Your can always check your current disk usage on your PANDA Dashboard:
+http://{{ site_domain }}/#dashboard
View
1 panda/templates/notifications/disk_space_alert/email_subject.txt
@@ -0,0 +1 @@
+Low disk space!

0 comments on commit 647159a

Please sign in to comment.
Something went wrong with that request. Please try again.