OpenConceptLab/ocl_issues#1633 | Monthly usage report refactoring and…

… using CSV format

.coveragerc

@@ -6,6 +6,7 @@ omit =
     core/celery.py
     core/common/healthcheck/*
     core/reports/*
+    */reports.py
     core/wsgi.py
     */management/commands/*
     */migrations/*

core/collections/models.py

@@ -460,6 +460,14 @@ class Meta:
     mappings = models.ManyToManyField('mappings.Mapping', related_name='references', through=ReferencedMapping)
     collection = models.ForeignKey('collections.Collection', related_name='references', on_delete=models.CASCADE)
 
+    @staticmethod
+    def get_static_references_criteria():
+        return models.Q(
+            models.Q(resource_version__isnull=False) |
+            models.Q(resource_version__isnull=True, transform__isnull=False) |
+            models.Q(resource_version__isnull=True, transform__isnull=True, code__isnull=False, version__isnull=False)
+        )
+
     @property
     def resource_type(self):
         return COLLECTION_REFERENCE_TYPE

core/collections/reports.py

@@ -0,0 +1,108 @@
+from core.collections.models import Collection, Expansion, CollectionReference
+from core.common.constants import HEAD
+from core.reports.models import AbstractReport
+
+
+class CollectionReport(AbstractReport):
+    queryset = Collection.objects.filter(version=HEAD)
+    name = 'Collections'
+    select_related = ['created_by', 'organization', 'user']
+    verbose_fields = [
+        'mnemonic',
+        'name',
+        'created_by.username',
+        'created_at',
+        'parent_resource_type',
+        'parent_resource',
+        'canonical_url',
+        'custom_validation_schema'
+    ]
+    VERBOSE_HEADERS = [
+        "ID",
+        "Name",
+        "Created By",
+        "Created At",
+        "Owner Type",
+        "Owner",
+        "Canonical URL",
+        "Validation Schema"
+    ]
+
+
+class CollectionVersionReport(AbstractReport):
+    queryset = Collection.objects.exclude(version=HEAD)
+    name = 'Collection Versions'
+    select_related = ['created_by', 'organization', 'user']
+    verbose_fields = [
+        'version',
+        'mnemonic',
+        'name',
+        'created_by.username',
+        'created_at',
+        'parent_resource_type',
+        'parent_resource',
+        'custom_validation_schema'
+    ]
+    VERBOSE_HEADERS = [
+        "Version",
+        "ID",
+        "Name",
+        "Created By",
+        "Created At",
+        "Owner Type",
+        "Owner",
+        "Validation Schema"
+    ]
+
+
+class ExpansionReport(AbstractReport):
+    queryset = Expansion.objects.filter()
+    name = 'Expansions'
+    verbose = False
+
+
+class ReferenceReport(AbstractReport):
+    queryset = CollectionReference.objects.filter()
+    name = 'References'
+    grouped_label = "New References"
+    verbose = False
+    grouped = True
+    GROUPED_HEADERS = ["Resource Type", "Static", "Dynamic", "Total"]
+
+    @property
+    def grouped_queryset(self):
+        if not self.queryset.exists():
+            return []
+        concepts_queryset = self.queryset.filter(reference_type='concepts')
+        mappings_queryset = self.queryset.filter(reference_type='mappings')
+        total_concepts = concepts_queryset.count()
+        total_mappings = mappings_queryset.count()
+        static_criteria = CollectionReference.get_static_references_criteria()
+        total_static_concepts = concepts_queryset.filter(static_criteria).count()
+        total_static_mappings = mappings_queryset.filter(static_criteria).count()
+        return [
+            [
+                'Concepts',
+                total_static_concepts,
+                total_concepts - total_static_concepts,
+                total_concepts
+            ],
+            [
+                'Mappings',
+                total_static_mappings,
+                total_mappings - total_static_mappings,
+                total_mappings
+            ]
+        ]
+
+    @staticmethod
+    def to_grouped_stat_csv_row(obj):
+        return [*obj]
+
+    @property
+    def retired(self):
+        return 0
+
+    @property
+    def active(self):
+        return self.count

core/common/tasks.py

@@ -19,7 +19,9 @@
 from core.celery import app
 from core.common import ERRBIT_LOGGER
 from core.common.constants import CONFIRM_EMAIL_ADDRESS_MAIL_SUBJECT, PASSWORD_RESET_MAIL_SUBJECT
-from core.common.utils import write_export_file, web_url, get_resource_class_from_resource_name, get_export_service
+from core.common.utils import write_export_file, web_url, get_resource_class_from_resource_name, get_export_service, \
+    get_date_range_label
+from core.reports.models import ResourceUsageReport
 from core.toggles.models import Toggle
 
 logger = get_task_logger(__name__)
@@ -622,28 +624,21 @@ def beat_healthcheck():  # pragma: no cover
 
 
 @app.task(ignore_result=True)
-def monthly_usage_report():  # pragma: no cover
+def resources_report():  # pragma: no cover
     # runs on first of every month
-    # reports usage of prev month and trend over last 3 months
-    from core.reports.models import MonthlyUsageReport
+    # reports usage of prev month
     now = timezone.now().replace(day=1)
-    three_months_from_now = now - relativedelta(months=3)
-    last_month = now - relativedelta(months=1)
-    report = MonthlyUsageReport(
-        verbose=True, start=three_months_from_now, end=now, current_month_end=now, current_month_start=last_month)
-    report.prepare()
-    html_body = render_to_string('monthly_usage_report_for_mail.html', report.get_result_for_email())
-    FORMAT = '%Y-%m-%d'
-    start = report.start
-    end = report.end
+    report = ResourceUsageReport(start_date=now - relativedelta(months=1), end_date=now)
+    buff, file_name = report.generate()
+    date_range_label = get_date_range_label(report.start_date, report.end_date)
+    env = settings.ENV.upper()
     mail = EmailMessage(
-        subject=f"{settings.ENV.upper()} Monthly usage report: {start.strftime(FORMAT)} to {end.strftime(FORMAT)}",
-        body=html_body,
+        subject=f"{env} Monthly Resources Report: {date_range_label}",
+        body=f"Please find attached resources report of {env} for the period of {date_range_label}",
         to=[settings.REPORTS_EMAIL]
     )
-    mail.content_subtype = "html"
-    res = mail.send()
-    return res
+    mail.attach(file_name, buff.getvalue(), 'text/csv')
+    return mail.send()
 
 
 @app.task(ignore_result=True)

core/common/tests.py

@@ -21,7 +21,7 @@
 
 from core.collections.models import CollectionReference
 from core.common.constants import HEAD
-from core.common.tasks import delete_s3_objects, bulk_import_parallel_inline, monthly_usage_report
+from core.common.tasks import delete_s3_objects, bulk_import_parallel_inline, resources_report
 from core.common.utils import (
     compact_dict_by_values, to_snake_case, flower_get, task_exists, parse_bulk_import_task_id,
     to_camel_case,
@@ -1038,19 +1038,18 @@ def test_bulk_import_parallel_inline_valid_json(self, import_run_mock):
     def test_monthly_usage_report(self, email_message_mock):
         email_message_instance_mock = Mock(send=Mock(return_value=1))
         email_message_mock.return_value = email_message_instance_mock
-        res = monthly_usage_report()
+        res = resources_report()
 
         email_message_mock.assert_called_once()
         email_message_instance_mock.send.assert_called_once()
+        email_message_instance_mock.attach.assert_called_once_with(ANY, ANY, 'text/csv')
 
         self.assertEqual(res, 1)
         call_args = email_message_mock.call_args[1]
-        self.assertTrue("Monthly usage report" in call_args['subject'])
+        self.assertTrue("Monthly Resources Report" in call_args['subject'])
         self.assertEqual(call_args['to'], ['reports@openconceptlab.org'])
-        self.assertTrue('</html>' in call_args['body'])
-        self.assertTrue('concepts' in call_args['body'])
-        self.assertTrue('sources' in call_args['body'])
-        self.assertTrue('collections' in call_args['body'])
+        self.assertTrue('Please find attached resources report of' in call_args['body'])
+        self.assertTrue('for the period of' in call_args['body'])
         self.assertEqual(email_message_instance_mock.content_subtype, 'html')
 
 

core/common/utils.py

@@ -890,3 +890,18 @@ def get_falsy_values():
 
 def get_truthy_values():
     return ['true', True, 'True', 1, '1']
+
+
+def get_date_range_label(start_date, end_date):
+    start = from_string_to_date(start_date)
+    end = from_string_to_date(end_date)
+
+    start_month = start.strftime('%B')
+    end_month = end.strftime('%B')
+
+    if start.year == end.year:
+        if start_month == end_month:
+            return f"{start.day:02d} - {end.day:02d} {start_month} {start.year}"
+        return f"{start.day:02d} {start_month} - {end.day:02d} {end_month} {start.year}"
+
+    return f"{start.day:02d} {start_month} {start.year} - {end.day:02d} {end_month} {end.year}"

core/concepts/reports.py

@@ -0,0 +1,16 @@
+from django.db.models import F
+
+from core.reports.models import AbstractReport
+from core.concepts.models import Concept
+
+
+class ConceptReport(AbstractReport):
+    queryset = Concept.objects.filter(id=F('versioned_object_id'))
+    name = 'Concepts'
+    verbose = False
+
+
+class ConceptVersionReport(AbstractReport):
+    queryset = Concept.objects.exclude(id=F('versioned_object_id')).exclude(is_latest_version=True)
+    name = 'Concept Versions'
+    verbose = False

core/mappings/reports.py

@@ -0,0 +1,38 @@
+from django.db.models import F, Count
+
+from core.reports.models import AbstractReport
+from core.mappings.models import Mapping
+
+
+class MappingReport(AbstractReport):
+    queryset = Mapping.objects.filter(id=F('versioned_object_id'))
+    name = 'Mappings'
+    grouped_label = "New Mappings Grouped by Target Source"
+    verbose = False
+    grouped = True
+    GROUPED_HEADERS = ["Target Source ID", "Count"]
+
+    @property
+    def grouped_queryset(self):
+        from core.sources.models import Source
+        return Source.objects.values(
+            'id', 'mnemonic'
+        ).filter(
+            mappings_to__id=F('mappings_to__versioned_object_id'),
+            mappings_to__created_at__gte=self.start_date,
+            mappings_to__created_at__lte=self.end_date
+        ).annotate(
+            count=Count('mappings_to__id')
+        ).order_by('-count').values_list(
+            'mnemonic', 'count'
+        )
+
+    @staticmethod
+    def to_grouped_stat_csv_row(obj):
+        return [*obj]
+
+
+class MappingVersionReport(AbstractReport):
+    queryset = Mapping.objects.exclude(id=F('versioned_object_id')).exclude(is_latest_version=True)
+    name = 'Mapping Versions'
+    verbose = False

core/orgs/reports.py

@@ -0,0 +1,10 @@
+from core.reports.models import AbstractReport
+from core.orgs.models import Organization
+
+
+class OrganizationReport(AbstractReport):
+    queryset = Organization.objects.filter()
+    name = 'Organizations'
+    select_related = ['created_by']
+    verbose_fields = ['mnemonic', 'name', 'created_by.username', 'created_at']
+    VERBOSE_HEADERS = ["ID", "Name", "Created By", "Created At"]