Merge a2fdaf3 into 0bfb7eb

akvo/rest/views/report.py

@@ -94,7 +94,7 @@ def program_reports(request, program_pk):
     serializer = ReportSerializer(queryset.distinct(), many=True)
     result = []
     for r in serializer.data:
-        r['url'] = r['url'].replace('{organisation}', str(organisation.id))
+        r['url'] = r['url'].replace('{organisation}', str(organisation.id)).replace('&download=true', '')
         result.append(r)
 
     return Response({'results': result})

akvo/rsr/management/commands/send_report_via_email.py

@@ -0,0 +1,8 @@
+from django.core.management.base import BaseCommand
+from akvo.rsr.views.py_reports.email_report import run_job
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        run_job()

akvo/rsr/migrations/0220_emailreportjob.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.10 on 2022-09-01 07:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rsr', '0219_iatiactivityvalidationjob_iatiorganisationvalidationjob'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EmailReportJob',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, null=True)),
+                ('started_at', models.DateTimeField(null=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('attempts', models.PositiveSmallIntegerField(default=0)),
+                ('report', models.CharField(max_length=100)),
+                ('payload', models.JSONField(default=dict)),
+                ('recipient', models.EmailField(max_length=254)),
+            ],
+        ),
+    ]

akvo/rsr/models/__init__.py

@@ -25,6 +25,7 @@
 from .crs_add import CrsAdd, CrsAddOtherFlag
 from .category import Category
 from .employment import Employment
+from .email_report_job import EmailReportJob
 from .focus_area import FocusArea
 from .fss import Fss, FssForecast
 from .goal import Goal
@@ -103,6 +104,7 @@
     'CrsAddOtherFlag',
     'DefaultPeriod',
     'Employment',
+    'EmailReportJob',
     'FocusArea',
     'Fss',
     'FssForecast',

akvo/rsr/models/email_report_job.py

@@ -0,0 +1,21 @@
+from django.db import models
+from django.utils.timezone import now
+
+
+class EmailReportJob(models.Model):
+    created_at = models.DateTimeField(null=True, auto_now_add=True, db_index=True, editable=False)
+    started_at = models.DateTimeField(null=True)
+    finished_at = models.DateTimeField(null=True)
+    attempts = models.PositiveSmallIntegerField(default=0)
+    report = models.CharField(max_length=100)
+    payload = models.JSONField(default=dict)
+    recipient = models.EmailField()
+
+    def mark_started(self):
+        self.started_at = now()
+        self.attempts = self.attempts + 1
+        self.save(update_fields=['started_at', 'attempts'])
+
+    def mark_finished(self):
+        self.finished_at = now()
+        self.save(update_fields=['finished_at'])

akvo/rsr/spa/app/modules/reports/reports.jsx

@@ -1,11 +1,12 @@
 /* global window, document */
 import React, { useState, useEffect, useReducer } from 'react'
 import { connect } from 'react-redux'
-import { Button, Spin, Icon, Card, Select, DatePicker, Checkbox } from 'antd'
+import { Button, Spin, Icon, Card, Select, DatePicker, Checkbox, Modal } from 'antd'
 import { useTranslation } from 'react-i18next'
 import Cookie from 'js-cookie'
 import moment from 'moment'
 import classNames from 'classnames'
+import axios from 'axios'
 import { useFetch } from '../../utils/hooks'
 import api from '../../utils/api'
 import SUOrgSelect from '../users/su-org-select'
@@ -132,21 +133,43 @@ const Report = ({ report, currentOrg, projectId, programId, setDownloading }) =>
     if (programId) {
       downloadUrl = downloadUrl.replace('{program}', programId)
     }
+    if (downloadUrl.includes('download=true')) {
+      return (e) => {
+        e.stopPropagation()
+        const token = uid()
+        let timerId = setTimeout(function tick() {
+          if (Cookie.get(token)) {
+            clearTimeout(timerId)
+            setDownloading(false)
+          } else {
+            timerId = setTimeout(tick, 1000)
+          }
+        }, 1000)
+        setDownloading(true)
+        setTimeout(() => {
+          window.location.assign(`${downloadUrl}&did=${token}`)
+        }, 500)
+      }
+    }
     return (e) => {
       e.stopPropagation()
-      const token = uid()
-      let timerId = setTimeout(function tick() {
-        if (Cookie.get(token)) {
-          clearTimeout(timerId)
+      const handler = async function () {
+        try {
+          const { data } = await axios.get(downloadUrl)
+          Modal.success({
+            content: data,
+          });
+        } catch {
+          Modal.error({
+            title: 'Failed connecting to server',
+            content: 'Please try again in a few minutes...',
+          });
+        } finally {
           setDownloading(false)
-        } else {
-          timerId = setTimeout(tick, 1000)
         }
-      }, 1000)
+      }
       setDownloading(true)
-      setTimeout(() => {
-        window.location.assign(`${downloadUrl}&did=${token}`)
-      }, 500)
+      handler()
     }
   }
   return (

akvo/rsr/tests/views/py_reports/test_send_report_via_email.py

@@ -0,0 +1,50 @@
+from django.core import management
+from django.core import mail
+from django.urls import reverse
+from parameterized import parameterized
+from akvo.rsr.tests.base import BaseTestCase
+from akvo.rsr.models import EmailReportJob, Country
+from akvo.rsr.views.py_reports import (
+    program_overview_pdf_report,
+    program_overview_excel_report,
+    program_period_labels_overview_pdf_report,
+    results_indicators_with_map_pdf_reports,
+    nuffic_country_level_map_report,
+)
+
+
+class SendReportViaEmailTestCase(BaseTestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.program = self.create_program('Test program')
+        self.user = self.create_user('test@akvo.org', 'password', is_admin=True)
+        Country.objects.get_or_create(iso_code='nl')
+        self.c.login(username=self.user.email, password='password')
+        mail.outbox = []
+
+    def test_add_job(self):
+        response = self.c.get(reverse('py-reports-program-overview', args=(self.program.id,)))
+        self.assertEqual(202, response.status_code)
+        self.assertEqual(1, EmailReportJob.objects.count())
+        job = EmailReportJob.objects.first()
+        self.assertEqual(program_overview_pdf_report.REPORT_NAME, job.report)
+
+    @parameterized.expand([
+        ('py-reports-program-overview', '', program_overview_pdf_report.REPORT_NAME),
+        ('py-reports-program-overview-table', '', program_overview_excel_report.REPORT_NAME),
+        ('py-reports-program-period-labels-overview', '', program_period_labels_overview_pdf_report.REPORT_NAME),
+        ('py-reports-organisation-projects-results-indicators-map-overview', 'country=nl', results_indicators_with_map_pdf_reports.ORG_PROJECTS_REPORT_NAME),
+        ('py-reports-nuffic-country-level-report', 'country=nl', nuffic_country_level_map_report.REPORT_NAME),
+    ])
+    def test_send_report_via_email(self, url_name, query_params, report_name):
+        self.c.get(f"{reverse(url_name, args=(self.program.id,))}?{query_params}")
+        job = EmailReportJob.objects.first()
+        self.assertEqual(report_name, job.report)
+        self.assertEqual(1, EmailReportJob.objects.filter(finished_at__isnull=True).count())
+
+        management.call_command('send_report_via_email')
+
+        self.assertEqual(0, EmailReportJob.objects.filter(finished_at__isnull=True).count())
+        msg = mail.outbox[0]
+        self.assertEqual([self.user.email], msg.to)

akvo/rsr/views/py_reports/__init__.py

@@ -19,7 +19,7 @@
 from .results_indicators_with_map_pdf_reports import (
     render_project_results_indicators_overview,
     render_project_results_indicators_map_overview,
-    render_organisation_projects_results_indicators_map_overview
+    add_org_projects_email_report_job,
 )
 from .kickstart_word_report import render_report as render_kickstart_report
 from .eutf_narrative_word_report import render_report as render_eutf_narrative_word_report
@@ -33,11 +33,11 @@
     render_report as render_results_indicators_excel_report
 from .organisation_projects_overview_report import \
     render_report as render_org_projects_overview_report
-from .program_overview_excel_report import render_report as render_program_overview_excel_report
-from .program_overview_pdf_report import render_report as render_program_overview_pdf_report
-from .program_period_labels_overview_pdf_report import render_program_period_lables_overview
+from .program_overview_excel_report import add_email_report_job as add_program_overview_excel_report_email_job
+from .program_overview_pdf_report import add_email_report_job as add_program_overview_pdf_report_email_job
+from .program_period_labels_overview_pdf_report import add_email_report_job as add_program_period_labels_overview
 from .nuffic_country_level_map_report import \
-    render_country_level_report as render_nuffic_country_level_report
+    add_email_report_job as add_nuffic_country_level_report_job
 from .project_overview_pdf_report import render_report as render_project_overview_pdf_report
 
 
@@ -57,7 +57,6 @@ def check(request):
 __all__ = [
     'check',
     'render_project_results_indicators_map_overview',
-    'render_organisation_projects_results_indicators_map_overview',
     'render_project_results_indicators_excel_report',
     'render_project_updates_excel_report',
     'render_kickstart_report',
@@ -67,9 +66,10 @@ def check(request):
     'render_eutf_project_results_table_excel_report',
     'render_results_indicators_excel_report',
     'render_org_projects_overview_report',
-    'render_program_overview_excel_report',
-    'render_program_overview_pdf_report',
-    'render_program_period_lables_overview',
-    'render_nuffic_country_level_report',
     'render_project_overview_pdf_report',
+    'add_org_projects_email_report_job',
+    'add_program_overview_pdf_report_email_job',
+    'add_program_overview_excel_report_email_job',
+    'add_program_period_labels_overview',
+    'add_nuffic_country_level_report_job',
 ]

akvo/rsr/views/py_reports/email_report.py

@@ -0,0 +1,49 @@
+import logging
+
+from datetime import timedelta
+from django.db.models import Q
+from django.utils.timezone import now
+from akvo.rsr.models import EmailReportJob
+
+from . import (
+    program_overview_pdf_report,
+    program_overview_excel_report,
+    program_period_labels_overview_pdf_report,
+    results_indicators_with_map_pdf_reports,
+    nuffic_country_level_map_report,
+)
+
+TIMEOUT = timedelta(minutes=30)
+MAX_ATTEMPTS = 3
+HANDLER = {
+    program_overview_pdf_report.REPORT_NAME: program_overview_excel_report.handle_email_report,
+    program_overview_excel_report.REPORT_NAME: program_overview_excel_report.handle_email_report,
+    program_period_labels_overview_pdf_report.REPORT_NAME: program_period_labels_overview_pdf_report.handle_email_report,
+    results_indicators_with_map_pdf_reports.ORG_PROJECTS_REPORT_NAME: results_indicators_with_map_pdf_reports.handle_org_projects_email_report,
+    nuffic_country_level_map_report.REPORT_NAME: nuffic_country_level_map_report.handle_email_report,
+}
+
+logger = logging.getLogger(__name__)
+
+
+def run_job():
+    pending_jobs = _get_pending_jobs()
+    if not pending_jobs.exists():
+        return
+    job = pending_jobs.first()
+    job.mark_started()
+    try:
+        handler = HANDLER.get(job.report, None)
+        if handler:
+            handler(job.payload, job.recipient)
+        job.mark_finished()
+    except Exception:
+        logger.exception(f'Failed to genereate report {job.report} for {job.recipient}')
+
+
+def _get_pending_jobs():
+    started_timeout = now() - TIMEOUT
+    return EmailReportJob.objects\
+        .order_by('created_at')\
+        .filter(finished_at__isnull=True)\
+        .exclude(Q(attempts__gte=MAX_ATTEMPTS) | Q(started_at__gte=started_timeout))

akvo/rsr/views/py_reports/nuffic_country_level_map_report.py

@@ -7,34 +7,54 @@
 see < http://www.gnu.org/licenses/agpl.html >.
 """
 
-from akvo.rsr.decorators import with_download_indicator
 from akvo.rsr.models import Project, Country, IndicatorPeriod
 from akvo.rsr.staticmap import get_staticmap_url, Coordinate, Size
+from akvo.utils import to_bool
 from datetime import datetime
 from django.contrib.auth.decorators import login_required
 from django.db.models import Q
-from django.http import HttpResponse, HttpResponseBadRequest
+from django.http import HttpResponseBadRequest
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
 
 from . import utils
 
+REPORT_NAME = 'nuffic_country_level_map_report'
+
 
 @login_required
-@with_download_indicator
-def render_country_level_report(request, program_id):
+def add_email_report_job(request, program_id):
+    program = get_object_or_404(Project, pk=program_id)
     country = request.GET.get('country', '').strip()
     if not country:
         return HttpResponseBadRequest('Please provide the country code!')
-
-    show_comment = True if request.GET.get('comment', '').strip() == 'true' else False
-    start_date = utils.parse_date(request.GET.get('period_start', '').strip(), datetime(1900, 1, 1))
-    end_date = utils.parse_date(request.GET.get('period_end', '').strip(), datetime(2999, 12, 31))
-
-    country = get_object_or_404(Country, iso_code=country)
-    project = get_object_or_404(Project, id=program_id)
-    project_ids = project.descendants()\
-        .exclude(pk=program_id)\
+    show_comment = request.GET.get('comment', '').strip()
+    start_date = request.GET.get('period_start', '').strip()
+    end_date = request.GET.get('period_end', '').strip()
+    return utils.add_email_report_job(
+        report=REPORT_NAME,
+        payload={
+            'program_id': program.id,
+            'country': country,
+            'show_comment': show_comment,
+            'start_date': start_date,
+            'end_date': end_date,
+        },
+        recipient=request.user.email
+    )
+
+
+def handle_email_report(params, recipient):
+    country = params.get('country')
+    show_comment = to_bool(params.get('comment', ''))
+    start_date = utils.parse_date(params.get('period_start', ''), datetime(1900, 1, 1))
+    end_date = utils.parse_date(params.get('period_end', ''), datetime(2999, 12, 31))
+
+    country = Country.objects.get(iso_code=country)
+    program = Project.objects.get(pk=params['program_id'])
+
+    project_ids = program.descendants()\
+        .exclude(pk=program.id)\
         .exclude(Q(title__icontains='test') | Q(subtitle__icontains='test'))\
         .values_list('id', flat=True)
     projects_in_country = Project.objects\
@@ -57,12 +77,9 @@ def render_country_level_report(request, program_id):
         'today': now.strftime('%d-%b-%Y'),
     })
 
-    if request.GET.get('show-html', ''):
-        return HttpResponse(html)
-
     filename = '{}-{}-country-report.pdf'.format(now.strftime('%Y%b%d'), country.iso_code)
 
-    return utils.make_pdf_response(html, filename)
+    return utils.send_pdf_report(html, recipient, filename)
 
 
 def build_view_objects(projects, start_date=None, end_date=None):