[#5104] Implement sending report via email backend

akvo/rest/views/report.py

@@ -94,7 +94,7 @@ def program_reports(request, program_pk):
     serializer = ReportSerializer(queryset.distinct(), many=True)
     result = []
     for r in serializer.data:
-        r['url'] = r['url'].replace('{organisation}', str(organisation.id))
+        r['url'] = r['url'].replace('{organisation}', str(organisation.id)).replace('&download=true', '')
         result.append(r)
 
     return Response({'results': result})

akvo/rsr/management/commands/send_report_via_email.py

@@ -0,0 +1,8 @@
+from django.core.management.base import BaseCommand
+from akvo.rsr.views.py_reports.email_report import run_job
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        run_job()

akvo/rsr/migrations/0220_emailreportjob.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.10 on 2022-09-01 07:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rsr', '0219_iatiactivityvalidationjob_iatiorganisationvalidationjob'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EmailReportJob',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, null=True)),
+                ('started_at', models.DateTimeField(null=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('attempts', models.PositiveSmallIntegerField(default=0)),
+                ('report', models.CharField(max_length=100)),
+                ('payload', models.JSONField(default=dict)),
+                ('recipient', models.EmailField(max_length=254)),
+            ],
+        ),
+    ]

akvo/rsr/models/__init__.py

@@ -25,6 +25,7 @@
 from .crs_add import CrsAdd, CrsAddOtherFlag
 from .category import Category
 from .employment import Employment
+from .email_report_job import EmailReportJob
 from .focus_area import FocusArea
 from .fss import Fss, FssForecast
 from .goal import Goal
@@ -103,6 +104,7 @@
     'CrsAddOtherFlag',
     'DefaultPeriod',
     'Employment',
+    'EmailReportJob',
     'FocusArea',
     'Fss',
     'FssForecast',

akvo/rsr/models/email_report_job.py

@@ -0,0 +1,21 @@
+from django.db import models
+from django.utils.timezone import now
+
+
+class EmailReportJob(models.Model):
+    created_at = models.DateTimeField(null=True, auto_now_add=True, db_index=True, editable=False)
+    started_at = models.DateTimeField(null=True)
+    finished_at = models.DateTimeField(null=True)
+    attempts = models.PositiveSmallIntegerField(default=0)
+    report = models.CharField(max_length=100)
+    payload = models.JSONField(default=dict)
+    recipient = models.EmailField()
+
+    def mark_started(self):
+        self.started_at = now()
+        self.attempts = self.attempts + 1
+        self.save(update_fields=['started_at', 'attempts'])
+
+    def mark_finished(self):
+        self.finished_at = now()
+        self.save(update_fields=['finished_at'])

akvo/rsr/tests/views/py_reports/test_send_report_via_email.py

@@ -0,0 +1,50 @@
+from django.core import management
+from django.core import mail
+from django.urls import reverse
+from parameterized import parameterized
+from akvo.rsr.tests.base import BaseTestCase
+from akvo.rsr.models import EmailReportJob, Country
+from akvo.rsr.views.py_reports import (
+    program_overview_pdf_report,
+    program_overview_excel_report,
+    program_period_labels_overview_pdf_report,
+    results_indicators_with_map_pdf_reports,
+    nuffic_country_level_map_report,
+)
+
+
+class SendReportViaEmailTestCase(BaseTestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.program = self.create_program('Test program')
+        self.user = self.create_user('test@akvo.org', 'password', is_admin=True)
+        Country.objects.get_or_create(iso_code='nl')
+        self.c.login(username=self.user.email, password='password')
+        mail.outbox = []
+
+    def test_add_job(self):
+        response = self.c.get(reverse('py-reports-program-overview', args=(self.program.id,)))
+        self.assertEqual(202, response.status_code)
+        self.assertEqual(1, EmailReportJob.objects.count())
+        job = EmailReportJob.objects.first()
+        self.assertEqual(program_overview_pdf_report.REPORT_NAME, job.report)
+
+    @parameterized.expand([
+        ('py-reports-program-overview', '', program_overview_pdf_report.REPORT_NAME),
+        ('py-reports-program-overview-table', '', program_overview_excel_report.REPORT_NAME),
+        ('py-reports-program-period-labels-overview', '', program_period_labels_overview_pdf_report.REPORT_NAME),
+        ('py-reports-organisation-projects-results-indicators-map-overview', 'country=nl', results_indicators_with_map_pdf_reports.ORG_PROJECTS_REPORT_NAME),
+        ('py-reports-nuffic-country-level-report', 'country=nl', nuffic_country_level_map_report.REPORT_NAME),
+    ])
+    def test_send_report_via_email(self, url_name, query_params, report_name):
+        self.c.get(f"{reverse(url_name, args=(self.program.id,))}?{query_params}")
+        job = EmailReportJob.objects.first()
+        self.assertEqual(report_name, job.report)
+        self.assertEqual(1, EmailReportJob.objects.filter(finished_at__isnull=True).count())
+
+        management.call_command('send_report_via_email')
+
+        self.assertEqual(0, EmailReportJob.objects.filter(finished_at__isnull=True).count())
+        msg = mail.outbox[0]
+        self.assertEqual([self.user.email], msg.to)

akvo/rsr/views/py_reports/__init__.py

@@ -19,7 +19,7 @@
 from .results_indicators_with_map_pdf_reports import (
     render_project_results_indicators_overview,
     render_project_results_indicators_map_overview,
-    render_organisation_projects_results_indicators_map_overview
+    add_org_projects_email_report_job,
 )
 from .kickstart_word_report import render_report as render_kickstart_report
 from .eutf_narrative_word_report import render_report as render_eutf_narrative_word_report
@@ -33,11 +33,11 @@
     render_report as render_results_indicators_excel_report
 from .organisation_projects_overview_report import \
     render_report as render_org_projects_overview_report
-from .program_overview_excel_report import render_report as render_program_overview_excel_report
-from .program_overview_pdf_report import render_report as render_program_overview_pdf_report
-from .program_period_labels_overview_pdf_report import render_program_period_lables_overview
+from .program_overview_excel_report import add_email_report_job as add_program_overview_excel_report_email_job
+from .program_overview_pdf_report import add_email_report_job as add_program_overview_pdf_report_email_job
+from .program_period_labels_overview_pdf_report import add_email_report_job as add_program_period_labels_overview
 from .nuffic_country_level_map_report import \
-    render_country_level_report as render_nuffic_country_level_report
+    add_email_report_job as add_nuffic_country_level_report_job
 from .project_overview_pdf_report import render_report as render_project_overview_pdf_report
 
 
@@ -57,7 +57,6 @@ def check(request):
 __all__ = [
     'check',
     'render_project_results_indicators_map_overview',
-    'render_organisation_projects_results_indicators_map_overview',
     'render_project_results_indicators_excel_report',
     'render_project_updates_excel_report',
     'render_kickstart_report',
@@ -67,9 +66,10 @@ def check(request):
     'render_eutf_project_results_table_excel_report',
     'render_results_indicators_excel_report',
     'render_org_projects_overview_report',
-    'render_program_overview_excel_report',
-    'render_program_overview_pdf_report',
-    'render_program_period_lables_overview',
-    'render_nuffic_country_level_report',
     'render_project_overview_pdf_report',
+    'add_org_projects_email_report_job',
+    'add_program_overview_pdf_report_email_job',
+    'add_program_overview_excel_report_email_job',
+    'add_program_period_labels_overview',
+    'add_nuffic_country_level_report_job',
 ]

akvo/rsr/views/py_reports/email_report.py

@@ -0,0 +1,53 @@
+import logging
+
+from datetime import timedelta
+from django.db.models import Q
+from django.utils.timezone import now
+from akvo.rsr.models import EmailReportJob
+
+from . import (
+    program_overview_pdf_report,
+    program_overview_excel_report,
+    program_period_labels_overview_pdf_report,
+    results_indicators_with_map_pdf_reports,
+    nuffic_country_level_map_report,
+)
+
+TIMEOUT = timedelta(minutes=30)
+MAX_ATTEMPTS = 3
+HANDLER = {
+    program_overview_pdf_report.REPORT_NAME: program_overview_excel_report.handle_email_report,
+    program_overview_excel_report.REPORT_NAME: program_overview_excel_report.handle_email_report,
+    program_period_labels_overview_pdf_report.REPORT_NAME: program_period_labels_overview_pdf_report.handle_email_report,
+    results_indicators_with_map_pdf_reports.ORG_PROJECTS_REPORT_NAME: results_indicators_with_map_pdf_reports.handle_org_projects_email_report,
+    nuffic_country_level_map_report.REPORT_NAME: nuffic_country_level_map_report.handle_email_report,
+}
+
+logger = logging.getLogger(__name__)
+
+
+def run_job():
+    pending_jobs = _get_pending_jobs()
+    if not pending_jobs.exists():
+        return
+    job = pending_jobs.first()
+    job.mark_started()
+    try:
+        handler = _get_report_handler(job.report)
+        if handler:
+            handler(job.payload, job.recipient)
+        job.mark_finished()
+    except Exception:
+        logger.exception(f'Failed to genereate report {job.report} for {job.recipient}')
+
+
+def _get_pending_jobs():
+    started_timeout = now() - TIMEOUT
+    return EmailReportJob.objects\
+        .order_by('created_at')\
+        .filter(finished_at__isnull=True)\
+        .exclude(Q(attempts__gte=MAX_ATTEMPTS) | Q(started_at__gte=started_timeout))
+
+
+def _get_report_handler(report):
+    return HANDLER[report] if report in HANDLER else None

akvo/rsr/views/py_reports/nuffic_country_level_map_report.py

@@ -19,6 +19,68 @@
 
 from . import utils
 
+REPORT_NAME = 'nuffic_country_level_map_report'
+
+
+@login_required
+def add_email_report_job(request, program_id):
+    program = get_object_or_404(Project, pk=program_id)
+    country = request.GET.get('country', '').strip()
+    if not country:
+        return HttpResponseBadRequest('Please provide the country code!')
+    show_comment = request.GET.get('comment', '').strip()
+    start_date = request.GET.get('period_start', '').strip()
+    end_date = request.GET.get('period_end', '').strip()
+    return utils.add_email_report_job(
+        report=REPORT_NAME,
+        payload={
+            'program_id': program.id,
+            'country': country,
+            'show_comment': show_comment,
+            'start_date': start_date,
+            'end_date': end_date,
+        },
+        recipient=request.user.email
+    )
+
+
+def handle_email_report(params, recipient):
+    country = params.get('country')
+    show_comment = True if params.get('comment', '') == 'true' else False
+    start_date = utils.parse_date(params.get('period_start', ''), datetime(1900, 1, 1))
+    end_date = utils.parse_date(params.get('period_end', ''), datetime(2999, 12, 31))
+
+    country = Country.objects.get(iso_code=country)
+    program = Project.objects.get(pk=params['program_id'])
+
+    project_ids = program.descendants()\
+        .exclude(pk=program.id)\
+        .exclude(Q(title__icontains='test') | Q(subtitle__icontains='test'))\
+        .values_list('id', flat=True)
+    projects_in_country = Project.objects\
+        .filter(id__in=project_ids, locations__country=country)\
+        .distinct()
+    coordinates = [
+        Coordinate(location.latitude, location.longitude)
+        for project in projects_in_country
+        for location in project.locations.all()
+        if location.country == country
+    ]
+
+    now = datetime.today()
+
+    html = render_to_string('reports/nuffic-country-level-report.html', context={
+        'title': 'Country level report for projects in {}'.format(country.name),
+        'staticmap': get_staticmap_url(coordinates, Size(900, 600), zoom=11),
+        'projects': build_view_objects(projects_in_country, start_date, end_date),
+        'show_comment': show_comment,
+        'today': now.strftime('%d-%b-%Y'),
+    })
+
+    filename = '{}-{}-country-report.pdf'.format(now.strftime('%Y%b%d'), country.iso_code)
+
+    return utils.send_pdf_report(html, recipient, filename)
+
 
 @login_required
 @with_download_indicator

akvo/rsr/views/py_reports/program_overview_excel_report.py

@@ -195,6 +195,32 @@ def get_dynamic_column_start(aggregate_targets):
     return AGGREGATED_TARGET_VALUE_COLUMN + 1 if aggregate_targets else AGGREGATED_TARGET_VALUE_COLUMN
 
 
+REPORT_NAME = 'program_overview_excel_report'
+
+
+@login_required
+def add_email_report_job(request, program_id):
+    program = get_object_or_404(Project, pk=program_id)
+    return utils.add_email_report_job(
+        report=REPORT_NAME,
+        payload={
+            'program_id': program.id,
+            'period_start': request.GET.get('period_start', '').strip(),
+            'period_end': request.GET.get('period_end', '').strip(),
+        },
+        recipient=request.user.email
+    )
+
+
+def handle_email_report(params, recipient):
+    program = Project.objects.prefetch_related('results').get(pk=params['program_id'])
+    start_date = utils.parse_date(params.get('period_start', ''))
+    end_date = utils.parse_date(params.get('period_end', ''))
+    wb = generate_workbok(program, start_date, end_date)
+    filename = '{}-{}-program-overview-report.xlsx'.format(datetime.today().strftime('%Y%b%d'), program.id)
+    utils.send_excel_report(wb, recipient, filename)
+
+
 @login_required
 @with_download_indicator
 def render_report(request, program_id):

akvo/rsr/views/py_reports/program_overview_pdf_report.py

@@ -21,6 +21,43 @@
 
 from . import utils
 
+REPORT_NAME = 'program_overview_pdf_report'
+
+
+@login_required
+def add_email_report_job(request, program_id):
+    program = get_object_or_404(Project, pk=program_id)
+    return utils.add_email_report_job(
+        report=REPORT_NAME,
+        payload={
+            'program_id': program.id,
+            'period_start': request.GET.get('period_start', '').strip(),
+            'period_end': request.GET.get('period_end', '').strip(),
+        },
+        recipient=request.user.email
+    )
+
+
+def handle_email_report(params, recipient):
+    now = datetime.today()
+    program = Project.objects.prefetch_related('results').get(pk=params['program_id'])
+    start_date = utils.parse_date(params.get('period_start', ''))
+    end_date = utils.parse_date(params.get('period_end', ''))
+    program_view = build_view_object(program, start_date or datetime(1900, 1, 1), end_date or (datetime.today() + relativedelta(years=10)))
+    coordinates = [
+        Coordinate(loc.latitude, loc.longitude)
+        for loc in program_view.locations
+        if loc and loc.latitude and loc.longitude
+    ]
+    html = render_to_string('reports/program-overview.html', context={
+        'program': program_view,
+        'staticmap': get_staticmap_url(coordinates, Size(900, 600)),
+        'start_date': start_date,
+        'end_date': end_date,
+    })
+    filename = '{}-program-{}-overview.pdf'.format(now.strftime('%Y%b%d'), program.id)
+    utils.send_pdf_report(html, recipient, filename)
+
 
 class ProgramProxy(utils.ProjectProxy):
     def __init__(self, project, results={}):