CPA-314 Overall Summary add columns

- [x] $ per covid case
- [x] Covid statistics per country Active cases, current
- [x] Covid statistics per country Total cases
- [x] Total death cases, cumulative
- [x] When mapping contract titles in country_tender, filter out empty titles to avoided getting multiple  ” , , ”
- [x] Goodservices mapping to existing one.Didnt create new
- [x] Updated requirements.txt from pip-tools
- [x] Added filter with red_flag in contracts API eg: red_flag=1
- [x] Added filter with equity null and id eg: equity=not_null and equity_id=1
- [x] Created new libraries and helper directory and moved schedular and other functions
- [x] Added custom html in admin for validation and import details
- [x] Added validation and import details error catching functionality

Additional changes
- [x] Changed char to text field in temp table
- [x] updated **filter_args
- [x] Changed html for validation and import_details field in DataImportAdmin
- [x] Refactoring

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
           ./manage.py makemigrations --check --dry-run
           ./manage.py check --fail-level WARNING
           ./manage.py loaddata country redflag equitycategory equitykeywords
-          coverage run --source content,country,covidadmin,visualization,country/management/commands manage.py test --keepdb
+          coverage run --source content,country,covidadmin,visualization,libraries,helpers,country/management/commands manage.py test --keepdb
         env:
           DB_NAME: postgres
           DB_USER: postgres

.gitignore

@@ -50,3 +50,5 @@ deploy.sh
 
 #Pycharm
 *.idea/
+requirements_dev_m1.in
+requirements_dev_m1.txt

content/migrations/0051_dataimport_validation.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.8 on 2021-07-26 08:58
+
+import content.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('content', '0050_auto_20210503_1023'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataimport',
+            name='validation',
+            field=models.JSONField(default=content.models.default_validation_json),
+        ),
+        migrations.AddField(
+            model_name='dataimport',
+            name='import_details',
+            field=models.JSONField(default=content.models.default_validation_json),
+        ),
+    ]

content/models.py

@@ -239,19 +239,21 @@ class Meta:
         proxy = True
 
 
+def default_validation_json():  # This is a callable
+    return {"errors": [], "status": "not_started", "row_count": 0, "started_on": "", "completed_on": ""}
+
+
 class DataImport(Page):
     parent_page_types = ["content.Contents"]
 
-    subpage_types = []
-
     description = models.TextField(verbose_name=_("Description"), null=False, max_length=500000)
-
     import_file = models.FileField(null=True, blank=False, upload_to="documents", validators=[validate_file_extension])
-
     country = models.ForeignKey(Country, on_delete=models.PROTECT, blank=False, null=False)
     validated = models.BooleanField(null=False, blank=True, default=False)
     no_of_rows = models.CharField(verbose_name=_("No of rows"), null=True, max_length=10, default=0)
     imported = models.BooleanField(null=False, blank=True, default=False)
+    validation = models.JSONField(default=default_validation_json)
+    import_details = models.JSONField(default=default_validation_json)
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
 
@@ -266,13 +268,13 @@ class DataImport(Page):
         APIField("import_file"),
         APIField("country"),
     ]
+    subpage_types = []
     settings_panels = []
     promote_panels = []
     preview_modes = []
 
     def clean(self):
         super().clean()
-        self.slug = ""
 
     class Meta:
         verbose_name = "Data Import"

country/admin.py

@@ -1,8 +1,6 @@
 from django import forms
 from django.contrib import admin
 from django.db import models
-from django.db.models.signals import post_save
-from django.dispatch import receiver
 from django.forms import TextInput
 from django.utils.html import format_html
 
@@ -22,7 +20,7 @@
     Tender,
     Topic,
 )
-from .tasks import fix_contract_name_value, local_currency_to_usd
+from .tasks import process_currency_conversion
 
 
 class EquityInline(admin.TabularInline):
@@ -58,6 +56,22 @@ def has_add_permission(self, request, obj=None):
         return False
 
 
+def detail_view(items):
+    field_html = ""
+    for key, value in items:
+        if key == "errors":
+            error_list_html = ""
+            if len(value) > 0:
+                for error in value:
+                    error_list_html += f"<li>{error}</li>"
+            else:
+                error_list_html = "No Errors to show."
+            field_html += f"<li><strong>{key.capitalize()}</strong> : <ul>{error_list_html}</ul></li>"
+            continue
+        field_html += f"<li><strong>{key.capitalize()}</strong> : {value}</li>"
+    return format_html(field_html)
+
+
 @admin.register(DataImport)
 class DataImportAdmin(admin.ModelAdmin):
     exclude = (
@@ -73,8 +87,44 @@ class DataImportAdmin(admin.ModelAdmin):
         "show_in_menus",
         "seo_title",
         "content_type",
+        "validation",
+        "import_details",
     )
 
+    @admin.display()
+    def validation(self):
+        field_html = ""
+        for key, value in self.validation.items():
+            if key == "errors":
+                error_list_html = ""
+                if len(value) > 0:
+                    for error in value:
+                        error_list_html += f"<li>{error}</li>"
+                else:
+                    error_list_html = "No Errors to show."
+                field_html += f"<li><strong>{key.capitalize()}</strong> : <ul>{error_list_html}</ul></li>"
+                continue
+            field_html += f"<li><strong>{key.capitalize()}</strong> : {value}</li>"
+        return format_html(field_html)
+
+    @admin.display()
+    def import_details(self):
+        field_html = ""
+        for key, value in self.import_details.items():
+            if key == "errors":
+                error_list_html = ""
+                if len(value) > 0:
+                    for error in value:
+                        error_list_html += f"<li>{error}</li>"
+                else:
+                    error_list_html = "No Errors to show."
+                field_html += f"<li><strong>{key.capitalize()}</strong> : <ul>{error_list_html}</ul></li>"
+                continue
+            field_html += f"<li><strong>{key.capitalize()}</strong> : {value}</li>"
+        return format_html(field_html)
+
+    readonly_fields = (validation, import_details)
+
     def has_add_permission(self, request, obj=None):
         return False
 
@@ -93,14 +143,13 @@ def custom_title(self):
         else:
             return format_html(
                 f"""<img src="/static/admin/img/icon-no.svg" alt="False">
-                <a class="button" href="/data_validate?data_import_id={data_import_id}">
+                <a class="button" href="/data-validate?data_import_id={data_import_id}">
                 Validate</a>&nbsp;"""
             )
 
     custom_title.short_description = "Validation"
 
     def import_status(self):
-        # import_file = str(self.import_file).split("/")[-1]
         validated = bool(self.validated)
         if validated and self.imported:
             return format_html(
@@ -111,8 +160,9 @@ def import_status(self):
             return format_html("""<a class="button" disabled="True">Import</a>&nbsp;""")
         else:
             return format_html(
-                f"""<a class="button"
-                href="/data_import?country={str(self.country)}&data_import_id={str(self.page_ptr_id)}"""
+                f"""<a class="button" """
+                f"""href="/data-import?country_code={str(self.country.country_code_alpha_2)}"""
+                f"""&data_import_id={str(self.page_ptr_id)}"""
                 f"""&validated={validated}">Import</a>&nbsp;"""
             )
 
@@ -135,22 +185,22 @@ def import_status(self):
 
     def import_actions(self):
         data_import_id = str(self.page_ptr_id)
-        importbatch = ImportBatch.objects.get(data_import_id=data_import_id)
+        import_batch = ImportBatch.objects.filter(data_import_id=data_import_id).first()
         file_source = f"/media/{self.import_file}"
         if self.imported and self.validated:
             return format_html(
-                f"""<a class="button" disabled="True" >Edit</a>&nbsp;
+                f"""<a class="button" disabled="True">Edit</a>&nbsp;
                      <a class="button" href={file_source} download>Download Source File</a>&nbsp;
                      <a class="button" onclick="return confirm('Are you sure you want to delete?')"
-            href="/delete_dataset?data_import_id={data_import_id}"
+            href="/delete-dataset?data_import_id={data_import_id}"
             id="delete">Delete</a>&nbsp;"""
             )
         else:
             return format_html(
-                f"""<a class="button" href="/data_edit?data_import_id={importbatch.id}">Edit</a>&nbsp;
+                f"""<a class="button" href="/data-edit?data_import_id={import_batch.id if import_batch else ''}">Edit</a>&nbsp;
                 <a class="button" href={file_source} download>Download Source File</a>&nbsp;
             <a class="button" onclick="return confirm('Are you sure you want to delete?')"
-            href="/delete_dataset?data_import_id={data_import_id}"
+            href="/delete-dataset?data_import_id={data_import_id}"
             id="delete">Delete</a>&nbsp;"""
             )
 
@@ -321,19 +371,13 @@ def save_related(self, request, form, formsets, change):
         for form_set in formsets:
             if form_set.has_changed():
                 for value in form_set:
-                    contract_value_local = float(value["contract_value_local"].value())
-                    tender_value_local = float(value["tender_value_local"].value())
-                    award_value_local = float(value["award_value_local"].value())
+                    # todo: do not fetch country object in loop
                     country_id = value["country"].value()
                     source_currency = Country.objects.get(id=country_id).currency
-                    source_values = {
-                        "tender_value_local": tender_value_local or None,
-                        "award_value_local": award_value_local or None,
-                        "contract_value_local": contract_value_local or None,
-                    }
+
                     goods_services_row_id = value["id"].value()
-                    local_currency_to_usd.apply_async(
-                        args=(goods_services_row_id, conversion_date, source_currency, source_values), queue="covid19"
+                    process_currency_conversion.apply_async(
+                        args=(goods_services_row_id, conversion_date, source_currency), queue="covid19"
                     )
 
         super().save_related(request, form, formsets, change)
@@ -342,12 +386,11 @@ def save_related(self, request, form, formsets, change):
 admin.site.register(Tender, TenderAdmin)
 
 
-@receiver(post_save, sender=Tender)
-def save_model(sender, instance, **kwargs):
-    if getattr(instance, "from_admin_site", False):
-        tender_id = instance.id
-        tender_country = instance.country
-        fix_contract_name_value(tender_id, tender_country)
+# @receiver(post_save, sender=Tender)
+# def save_model(sender, instance, **kwargs):
+#     if getattr(instance, "from_admin_site", False):
+#         tender_id = instance.id
+#         update_contract_attributes(tender_id)
 
 
 @admin.register(RedFlag)

country/management/commands/export_summary_report.py

@@ -9,4 +9,5 @@ class Command(BaseCommand):
     def handle(self, *args, **kwargs):
         self.stdout.write("Exporting")
         export_summary_report.apply()
+
         return "Done"

country/management/commands/fetch_covid_active_cases.py

@@ -13,8 +13,8 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         today = datetime.date.today()
-
         dates_in_period = set()
+
         for year in range(2020, today.year + 1):
             for month in range(1, 13):
                 day = calendar.monthrange(year, month)[1]
@@ -38,6 +38,25 @@ def handle(self, *args, **kwargs):
             dates_to_query = dates_in_period - country_dates
             minimum_date = min(country_dates)
 
+            url = f"https://covid-api.com/api/reports/total?iso={country_code}"
+            print(country_code)
+            response = requests.get(url)
+            data = response.json()["data"]
+
+            if not data:
+                self.stdout.write(f"Fetching {url}... NO DATA")
+                active_cases = 0
+                total_cases = 0
+                total_death = 0
+            else:
+                active_cases = data["active"]
+                total_cases = data["confirmed"]
+                total_death = data["deaths"]
+
+            Country.objects.filter(country_code=country_code).update(
+                covid_active_cases=active_cases, covid_deaths_total=total_death, covid_cases_total=total_cases
+            )
+
             for date in sorted(dates_to_query):
                 # Don't retry dates from the beginning of the pandemic.
                 if minimum_date and date < minimum_date:

country/management/commands/process_currency_conversion.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.db.models import Q
 
-from country.models import GoodsServices
+from country.models import Country, GoodsServices
 from country.tasks import process_currency_conversion
 
 
@@ -10,20 +10,20 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         self.stdout.write("Processing currency conversion")
-        unconverted_tender = GoodsServices.objects.filter(
-            Q(tender_value_usd__isnull=True) | Q(award_value_usd__isnull=True) | Q(contract_value_usd__isnull=True)
-        )
+        countries = Country.objects.exclude(country_code_alpha_2="gl").all()
 
-        for tender in unconverted_tender:
-            id = tender.id
-            tender_date = tender.contract.contract_date
-            tender_value_local = tender.tender_value_local
-            contract_value_local = tender.contract_value_local
-            award_value_local = tender.award_value_local
-            currency = tender.country.currency
-            process_currency_conversion.apply_async(
-                args=(tender_value_local, award_value_local, contract_value_local, tender_date, currency, id),
-                queue="covid19",
+        for country in countries:
+            goods_services_items = GoodsServices.objects.filter(
+                Q(tender_value_usd__isnull=True) | Q(award_value_usd__isnull=True) | Q(contract_value_usd__isnull=True)
             )
-            self.stdout.write("Created task for id :" + str(id))
+            self.stdout.write(country.name + " Found: " + str(goods_services_items.count()))
+
+            for goods_services_obj in goods_services_items:
+                tender_date = goods_services_obj.contract.contract_date
+                process_currency_conversion.apply_async(
+                    args=(goods_services_obj.id, tender_date, country.currency),
+                    queue="covid19",
+                )
+                self.stdout.write("Created task for id :" + str(goods_services_obj.id))
+
         self.stdout.write("Done")

country/management/commands/summarize_country_buyer.py

@@ -5,13 +5,14 @@
 
 
 class Command(BaseCommand):
-    help = "Summarize country buyer's contract information"
+    help = "Summarize buyer's contract information"
 
     def add_arguments(self, parser):
         parser.add_argument("country_code", type=str, help="Country code")
 
     def handle(self, *args, **kwargs):
         country_code = kwargs["country_code"].upper()
+
         try:
             country = Country.objects.get(country_code_alpha_2=country_code)
         except Exception: