diff --git a/client/utils.py b/client/utils.py new file mode 100644 index 0000000..f3cb83e --- /dev/null +++ b/client/utils.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import os + +def get_total_disk_space(p): + """ + Calculate the total disk space of the device on which a given file path resides. + """ + s = os.statvfs(p) + return s.f_frsize * s.f_blocks + +def get_free_disk_space(p): + """ + Returns the number of free bytes on the drive that ``p`` is on + """ + s = os.statvfs(p) + return s.f_frsize * s.f_bavail + diff --git a/client/views.py b/client/views.py index 845dabe..7987023 100644 --- a/client/views.py +++ b/client/views.py @@ -14,6 +14,7 @@ from livesettings import config_value from tastypie.serializers import Serializer +from client import utils from panda.api.category import CategoryResource from panda.models import ActivityLog, Category, Dataset, SearchLog, UserProxy @@ -49,20 +50,6 @@ def index(request): }) }) -def _get_total_disk_space(p): - """ - Calculate the total disk space of the device on which a given file path resides. - """ - s = os.statvfs(p) - return s.f_frsize * s.f_blocks - -def _get_free_disk_space(p): - """ - Returns the number of free bytes on the drive that ``p`` is on - """ - s = os.statvfs(p) - return s.f_frsize * s.f_bavail - def dashboard(request): """ Render HTML for dashboard/metrics view. @@ -143,13 +130,13 @@ def dashboard(request): upload_disk = os.stat(settings.MEDIA_ROOT).st_dev indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev - root_disk_total = _get_total_disk_space('/') - root_disk_free = _get_free_disk_space('/') + root_disk_total = utils.get_total_disk_space('/') + root_disk_free = utils.get_free_disk_space('/') root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) if upload_disk != root_disk: - upload_disk_total = _get_total_disk_space(settings.MEDIA_ROOT) - upload_disk_free = _get_free_disk_space(settings.MEDIA_ROOT) + upload_disk_total = utils.get_total_disk_space(settings.MEDIA_ROOT) + upload_disk_free = utils.get_free_disk_space(settings.MEDIA_ROOT) upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) else: upload_disk_total = None @@ -157,8 +144,8 @@ def dashboard(request): upload_disk_percent_used = None if indices_disk != root_disk: - indices_disk_total = _get_total_disk_space(settings.SOLR_DIRECTORY) - indices_disk_free = _get_free_disk_space(settings.SOLR_DIRECTORY) + indices_disk_total = utils.get_total_disk_space(settings.SOLR_DIRECTORY) + indices_disk_free = utils.get_free_disk_space(settings.SOLR_DIRECTORY) indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) else: indices_disk_total = None diff --git a/config/settings.py b/config/settings.py index 8450d0f..04f5fe5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -143,6 +143,10 @@ 'run_subscriptions': { 'task': 'panda.tasks.cron.run_subscriptions', 'schedule': crontab(minute=30, hour=2) + }, + 'run_admin_alerts': { + 'task': 'panda.tasks.cron.run_admin_alerts', + 'schedule': crontab(minute=0, hour=4) } } diff --git a/panda/tasks/__init__.py b/panda/tasks/__init__.py index b7d3c38..6cd66a9 100644 --- a/panda/tasks/__init__.py +++ b/panda/tasks/__init__.py @@ -8,6 +8,7 @@ from panda.tasks.purge_data import PurgeDataTask from panda.tasks.purge_orphaned_uploads import PurgeOrphanedUploadsTask from panda.tasks.reindex import ReindexTask +from panda.tasks.run_admin_alerts import RunAdminAlertsTask from panda.tasks.run_subscriptions import RunSubscriptionsTask TASKS_BY_TYPE = { diff --git a/panda/tasks/run_admin_alerts.py b/panda/tasks/run_admin_alerts.py new file mode 100644 index 0000000..11d0791 --- /dev/null +++ b/panda/tasks/run_admin_alerts.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +import logging +import os + +from celery.task import Task +from django.conf import settings +from django.template import Context +from livesettings import config_value + +from client.utils import get_total_disk_space, get_free_disk_space +from panda.utils.mail import send_mail +from panda.utils.notifications import get_email_subject_template, get_email_body_template + +class RunAdminAlertsTask(Task): + """ + Notify administrators of anything which requires their attention (disk space, etc). + """ + name = 'panda.tasks.cron.run_admin_alerts' + + def run(self, *args, **kwargs): + from panda.models import UserProxy + + log = logging.getLogger(self.name) + log.info('Running admin alerts') + + # Disk space + root_disk = os.stat('/').st_dev + upload_disk = os.stat(settings.MEDIA_ROOT).st_dev + indices_disk = os.stat(settings.SOLR_DIRECTORY).st_dev + + root_disk_total = get_total_disk_space('/') + root_disk_free = get_free_disk_space('/') + root_disk_percent_used = 100 - (float(root_disk_free) / root_disk_total * 100) + + if upload_disk != root_disk: + upload_disk_total = get_total_disk_space(settings.MEDIA_ROOT) + upload_disk_free = get_free_disk_space(settings.MEDIA_ROOT) + upload_disk_percent_used = 100 - (float(upload_disk_free) / upload_disk_total * 100) + else: + upload_disk_total = None + upload_disk_free = None + upload_disk_percent_used = None + + if indices_disk != root_disk: + indices_disk_total = get_total_disk_space(settings.SOLR_DIRECTORY) + indices_disk_free = get_free_disk_space(settings.SOLR_DIRECTORY) + indices_disk_percent_used = 100 - (float(indices_disk_free) / indices_disk_total * 100) + else: + indices_disk_total = None + indices_disk_free = None + indices_disk_percent_used = None + + notify = False + + for free in (root_disk_free, upload_disk_free, indices_disk_free): + if free is None: + continue + + if free < settings.PANDA_AVAILABLE_SPACE_WARN: + notify = True + + if notify: + context = Context({ + 'root_disk': root_disk, + 'upload_disk': upload_disk, + 'indices_disk': indices_disk, + 'root_disk_total': root_disk_total, + 'root_disk_free': root_disk_free, + 'root_disk_percent_used': root_disk_percent_used, + 'upload_disk_total': upload_disk_total, + 'upload_disk_free': upload_disk_free, + 'upload_disk_percent_used': upload_disk_percent_used, + 'indices_disk_total': indices_disk_total, + 'indices_disk_free': indices_disk_free, + 'indices_disk_percent_used': indices_disk_percent_used, + 'settings': settings, + 'site_domain': config_value('DOMAIN', 'SITE_DOMAIN') + }) + + # Don't HTML escape plain-text emails + context.autoescape = False + + email_subject = get_email_subject_template('disk_space_alert').render(context) + email_message = get_email_body_template('disk_space_alert').render(context) + + recipients = UserProxy.objects.filter(is_superuser=True, is_active=True) + + send_mail(email_subject.strip(), email_message, [r.email for r in recipients]) + + log.info('Finished running admin alerts') + diff --git a/panda/templates/notifications/disk_space_alert/email_body.txt b/panda/templates/notifications/disk_space_alert/email_body.txt new file mode 100644 index 0000000..9ea07d4 --- /dev/null +++ b/panda/templates/notifications/disk_space_alert/email_body.txt @@ -0,0 +1,6 @@ +{% if root_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA application is {% if root_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ root_disk_free|filesizeformat }} remains ({{ root_disk_percent_used|floatformat }}% full){% endif %} +{% if upload_disk_total != None and upload_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA's uploads is {% if upload_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ upload_disk_free|filesizeformat }} remains ({{ upload_disk_percent_used|floatformat }}% full){% endif %} +{% if indices_disk_total and indices_disk_free < settings.PANDA_AVAILABLE_SPACE_WARN %}* The drive containing your PANDA's search indices ({{ indices_disk }}) is {% if indices_disk_free < settings.PANDA_AVAILABLE_SPACE_CRITICAL %}CRITICALLY {% endif %}low on disk space: {{ indices_disk_free|filesizeformat }} remains ({{ indices_disk_percent_used|floatformat }}% full){% endif %} + +Your can always check your current disk usage on your PANDA Dashboard: +http://{{ site_domain }}/#dashboard diff --git a/panda/templates/notifications/disk_space_alert/email_subject.txt b/panda/templates/notifications/disk_space_alert/email_subject.txt new file mode 100644 index 0000000..ddc8bcf --- /dev/null +++ b/panda/templates/notifications/disk_space_alert/email_subject.txt @@ -0,0 +1 @@ +Low disk space!