Refactoring tasks

- [x] Refactor delete_dataset task
- [x] Define constants directory in covidadmin
- [x] Upgrade sentry-sdk to 1.3.0 in requirements file manually
- [x] Refactor visualization APIView
- [x] Add tasks to delete redundant buyers and suppliers
- [x] Rename GET parameters
- [x] Transform country_code to uppercase before filtering
- [x] Use raw_delete instead of delete in delete buyers
- [x] Migration changes for goods_services and tender
- [x] Scheduling delete buyers and suppliers task
- [x] Fix GoodServices error in test

country/migrations/0035_tender_buyer_protect.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.4 on 2021-07-23 04:00
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country', '0034_auto_20210505_1156'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tender',
+            name='buyer',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='tenders', to='country.buyer'),
+        ),
+        migrations.AlterField(
+            model_name='tender',
+            name='supplier',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='tenders', to='country.supplier'),
+        ),
+    ]

country/migrations/0036_goods_buyer_remove.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.4 on 2021-07-23 04:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country', '0035_tender_buyer_protect'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='goodsservices',
+            name='buyer',
+        ),
+        migrations.RemoveField(
+            model_name='goodsservices',
+            name='supplier',
+        ),
+    ]

country/models/goods_services.py

@@ -1,10 +1,8 @@
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from country.models.buyer import Buyer
 from country.models.country import Country
 from country.models.goods_services_category import GoodsServicesCategory
-from country.models.supplier import Supplier
 from country.models.tender import Tender
 
 
@@ -23,11 +21,6 @@ class GoodsServices(models.Model):
     contract_title = models.TextField(verbose_name=_("Contract title"), null=True, blank=True)
     contract_desc = models.TextField(verbose_name=_("Contract description"), null=True, blank=True)
 
-    supplier = models.ForeignKey(
-        Supplier, on_delete=models.CASCADE, related_name="goods_services", null=True, blank=True
-    )
-    buyer = models.ForeignKey(Buyer, on_delete=models.CASCADE, related_name="goods_services", null=True, blank=True)
-
     quantity_units = models.CharField(verbose_name=_("Quantity,units"), max_length=1500, null=True)
     ppu_including_vat = models.CharField(verbose_name=_("Price per units including VAT"), max_length=1500, null=True)
 

country/models/tender.py

@@ -24,8 +24,8 @@ class Tender(models.Model):
     ]
 
     country = models.ForeignKey(Country, on_delete=models.CASCADE, related_name="tenders")
-    supplier = models.ForeignKey(Supplier, on_delete=models.CASCADE, related_name="tenders", null=True, blank=True)
-    buyer = models.ForeignKey(Buyer, on_delete=models.CASCADE, related_name="tenders", null=True, blank=True)
+    supplier = models.ForeignKey(Supplier, on_delete=models.PROTECT, related_name="tenders", null=True, blank=True)
+    buyer = models.ForeignKey(Buyer, on_delete=models.PROTECT, related_name="tenders", null=True, blank=True)
 
     contract_id = models.CharField(verbose_name=_("Contract ID"), max_length=150, null=True)
     contract_date = models.DateField(verbose_name=_("Contract date"), null=True, db_index=True)

country/tasks/__init__.py

@@ -2,6 +2,8 @@
 from .convert_local_to_usd import convert_local_to_usd
 from .country_contract_excel import country_contract_excel
 from .delete_dataset import delete_dataset
+from .delete_redundant_buyers import delete_redundant_buyers
+from .delete_redundant_suppliers import delete_redundant_suppliers
 from .evaluate_contract_equity import evaluate_contract_equity
 from .evaluate_contract_red_flag import evaluate_contract_red_flag
 from .evaluate_country_buyer import evaluate_country_buyer

country/tasks/delete_dataset.py

@@ -1,7 +1,7 @@
 from celery import Celery
 from django.core import management
 
-from country.models import Country, ImportBatch, TempDataImportTable, Tender
+from country.models import Country, ImportBatch
 from visualization.helpers.scheduler import ScheduleRunner
 
 app = Celery()
@@ -11,13 +11,6 @@
 def delete_dataset(data_import_id):
     import_batch = ImportBatch.objects.get(data_import_id=data_import_id)
     country_obj = Country.objects.filter(country_code_alpha_2=import_batch.country.country_code_alpha_2).first()
-    all_temp_data_id = [i.id for i in import_batch.import_batch.all()]
-    for temp_data_id in all_temp_data_id:
-        obj = TempDataImportTable.objects.get(id=temp_data_id)
-        tender_obj = Tender.objects.filter(temp_table_id=obj.id)
-        obj.delete()
-        tender_obj.delete()
-        print("Done for temp data " + str(temp_data_id))
     import_batch.delete()
     instance = ScheduleRunner()
     instance.task_scheduler(
@@ -32,5 +25,17 @@ def delete_dataset(data_import_id):
         interval=8,
         country_alpha_code=country_obj.country_code_alpha_2,
     )
+    instance.task_scheduler(
+        task_name="delete_redundant_buyers",
+        interval_name="every_hour",
+        interval=24,
+        country_alpha_code=country_obj.country_code_alpha_2,
+    )
+    instance.task_scheduler(
+        task_name="delete_redundant_suppliers",
+        interval_name="every_hour",
+        interval=24,
+        country_alpha_code=country_obj.country_code_alpha_2,
+    )
     management.call_command("export_summary_report", country_obj.country_code_alpha_2)
     return "Done"

country/tasks/delete_redundant_buyers.py

@@ -0,0 +1,16 @@
+from celery import Celery
+
+from country.models import Buyer, Tender
+
+app = Celery()
+
+
+@app.task(name="delete_redundant_buyers")
+def delete_redundant_buyers(country_code):
+    buyer_ids = [
+        buyer_id["buyer_id"]
+        for buyer_id in Tender.objects.filter(country__country_code_alpha_2=country_code).values("buyer_id").distinct()
+    ]
+    buyers_to_delete = Buyer.objects.filter(country__country_code_alpha_2=country_code).exclude(id__in=buyer_ids)
+    buyers_to_delete._raw_delete(buyers_to_delete.db)
+    return "Done"

country/tasks/delete_redundant_suppliers.py

@@ -0,0 +1,20 @@
+from celery import Celery
+
+from country.models import Supplier, Tender
+
+app = Celery()
+
+
+@app.task(name="delete_redundant_suppliers")
+def delete_redundant_suppliers(country_code):
+    supplier_ids = [
+        supplier_id["supplier_id"]
+        for supplier_id in Tender.objects.filter(country__country_code_alpha_2=country_code)
+        .values("supplier_id")
+        .distinct()
+    ]
+    suppliers_to_delete = Supplier.objects.filter(country__country_code_alpha_2=country_code).exclude(
+        id__in=supplier_ids
+    )
+    suppliers_to_delete._raw_delete(suppliers_to_delete.db)
+    return "Done"

country/tasks/import_tender_from_batch_id.py

@@ -181,8 +181,6 @@ def import_tender_from_batch_id(batch_id, country, currency):
                     country=country_obj,
                     goods_services_category=goods_services_category_obj,
                     contract=tender_obj,
-                    supplier=supplier_obj,
-                    buyer=buyer_obj,
                     classification_code=classification_code,
                     no_of_bidders=no_of_bidders or None,
                     contract_title=contract_title,

country/tests/commands/test_process_currency_conversion.py

@@ -55,8 +55,6 @@ def setUpModule():
         no_of_bidders=112,
         contract_title="title",
         contract_desc="desc",
-        supplier=supplier,
-        buyer=buyer,
         quantity_units=1233,
         ppu_including_vat=1234.0,
         tender_value_local=1234.0,

country/views/delete_data_set_view.py

@@ -4,6 +4,7 @@
 
 from content.models import DataImport
 from country.tasks.delete_dataset import delete_dataset
+from covidadmin.constants import queues
 
 
 class DeleteDataSetView(APIView):
@@ -13,7 +14,7 @@ def get(self, request):
             if data_import_id is not None:
                 data_import = DataImport.objects.get(page_ptr_id=data_import_id)
                 data_import.delete()
-                delete_dataset.apply_async(args=(data_import_id,), queue="covid19")
+                delete_dataset.apply_async(args=(data_import_id,), queue=queues.default)
                 messages.info(request, "Your dataset has been successfully deleted from the system !!")
                 return HttpResponseRedirect("/admin/content/dataimport")
             else:

covidadmin/constants/queues.py

@@ -0,0 +1 @@
+default = "covid19"

requirements.txt

@@ -116,7 +116,7 @@ requests==2.25.1
     # via
     #   -r requirements.in
     #   wagtail
-sentry-sdk==1.0.0
+sentry-sdk==1.3.0
     # via -r requirements.in
 six==1.15.0
     # via

requirements_dev.txt

@@ -261,7 +261,7 @@ requests==2.25.1
     #   coveralls
     #   transifex-client
     #   wagtail
-sentry-sdk==1.0.0
+sentry-sdk==1.3.0
     # via -r requirements.txt
 six==1.15.0
     # via

visualization/helpers/scheduler.py

@@ -1,6 +1,8 @@
 from celery import Celery
 
 from country.tasks.country_contract_excel import country_contract_excel
+from country.tasks.delete_redundant_buyers import delete_redundant_buyers
+from country.tasks.delete_redundant_suppliers import delete_redundant_suppliers
 from country.tasks.evaluate_contract_red_flag import evaluate_contract_red_flag
 from country.tasks.evaluate_country_buyer import evaluate_country_buyer
 from country.tasks.evaluate_country_supplier import evaluate_country_supplier
@@ -24,6 +26,8 @@ def __init__(self):
             "evaluate_contract_red_flag": evaluate_contract_red_flag,
             "export_summary_report": export_summary_report,
             "summarize_country_contracts": summarize_country_contracts,
+            "delete_redundant_suppliers": delete_redundant_suppliers,
+            "delete_redundant_buyers": delete_redundant_buyers,
         }
         self.datetime_now = datetime.datetime.now()
 

visualization/views/average_bids_view.py

@@ -16,15 +16,16 @@ def get(self, request):
         """
         Returns average bids for contracts
         """
-        country = self.request.GET.get("country", None)
-        buyer = self.request.GET.get("buyer")
+        country_code = self.request.GET.get("country", None)
+        buyer_id = self.request.GET.get("buyer")
 
         filter_args = {}
         filter_args["no_of_bidders__gte"] = 1
-        if country:
-            filter_args["country__country_code_alpha_2"] = country
-        if buyer:
-            filter_args = add_filter_args("buyer", buyer, filter_args)
+        if country_code:
+            country_code = str(country_code).upper()
+            filter_args["country__country_code_alpha_2"] = country_code
+        if buyer_id:
+            filter_args = add_filter_args("buyer", buyer_id, filter_args)
 
         # Month wise average of number of bids for contracts
         monthwise_data = (

visualization/views/buyer_summary_view.py

@@ -16,14 +16,15 @@ class BuyerSummaryView(APIView):
     @method_decorator(cache_page(page_expire_period()))
     def get(self, request):
         filter_args = {}
-        country = self.request.GET.get("country", None)
+        country_code = self.request.GET.get("country", None)
         result = {}
         trend = []
         current_time = datetime.datetime.now()
         previous_month_date = current_time - dateutil.relativedelta.relativedelta(months=-1)
         previous_month = previous_month_date.replace(day=1).date()
-        if country:
-            filter_args["country__country_code_alpha_2"] = country
+        if country_code:
+            country_code = str(country_code).upper()
+            filter_args["country__country_code_alpha_2"] = country_code
         buyer_details = (
             Tender.objects.filter(**filter_args)
             .exclude(buyer__isnull=True)

visualization/views/contract_red_flags_view.py

@@ -12,16 +12,17 @@ class ContractRedFlagsView(APIView):
     @method_decorator(cache_page(page_expire_period()))
     def get(self, request):
         filter_args = {}
-        country = self.request.GET.get("country", None)
-        supplier = self.request.GET.get("supplier", None)
-        buyer = self.request.GET.get("buyer", None)
+        country_code = self.request.GET.get("country", None)
+        supplier_id = self.request.GET.get("supplier", None)
+        buyer_id = self.request.GET.get("buyer", None)
         product = self.request.GET.get("product", None)
-        if country:
-            filter_args["country__country_code_alpha_2"] = country
-        if supplier:
-            filter_args = add_filter_args("supplier", supplier, filter_args)
-        if buyer:
-            filter_args = add_filter_args("buyer", buyer, filter_args)
+        if country_code:
+            country_code = str(country_code).upper()
+            filter_args["country__country_code_alpha_2"] = country_code
+        if supplier_id:
+            filter_args = add_filter_args("supplier", supplier_id, filter_args)
+        if buyer_id:
+            filter_args = add_filter_args("buyer", buyer_id, filter_args)
         if product:
             filter_args["goods_services__goods_services_category__id"] = product
         red_flags = RedFlag.objects.filter(implemented=True)

visualization/views/contract_status_view.py

@@ -22,14 +22,14 @@ def get(self, request):
         currency_code = ""
 
         country_code = self.request.GET.get("country", None)
-        buyer = self.request.GET.get("buyer")
+        buyer_id = self.request.GET.get("buyer")
 
         if country_code:
             country_code = str(country_code).upper()
             filter_args["country__country_code_alpha_2"] = country_code
 
-        if buyer:
-            filter_args = add_filter_args("buyer", buyer, filter_args)
+        if buyer_id:
+            filter_args = add_filter_args("buyer", buyer_id, filter_args)
 
         if country_code:
             try:

visualization/views/country_map_view.py

@@ -12,17 +12,17 @@
 class CountryMapView(APIView):
     @method_decorator(cache_page(page_expire_period()))
     def get(self, request):
-        country = self.request.GET.get("country", None)
+        country_code = self.request.GET.get("country", None)
 
         try:
-            country_instance = Country.objects.get(country_code_alpha_2=country)
+            country_instance = Country.objects.get(country_code_alpha_2=country_code)
 
-            if country is not None and country_instance is not None:
-                tender_instance = Tender.objects.filter(country__country_code_alpha_2=country).aggregate(
+            if country_code is not None and country_instance is not None:
+                tender_instance = Tender.objects.filter(country__country_code_alpha_2=country_code).aggregate(
                     total_usd=Sum("contract_value_usd"),
                     total_local=Sum("contract_value_local"),
                 )
-                count = Tender.objects.filter(country__country_code_alpha_2=country).count()
+                count = Tender.objects.filter(country__country_code_alpha_2=country_code).count()
                 final = {
                     "country_code": country_instance.country_code_alpha_2,
                     "country": country_instance.name,

visualization/views/country_suppliers_view.py

@@ -15,10 +15,11 @@ class CountrySuppliersView(APIView):
     @method_decorator(cache_page(page_expire_period()))
     def get(self, request):
         filter_args = {}
-        country = self.request.GET.get("country", None)
+        country_code = self.request.GET.get("country", None)
         count = int(self.request.GET.get("count", 5))
-        if country:
-            filter_args["country__country_code_alpha_2"] = country
+        if country_code:
+            country_code = str(country_code).upper()
+            filter_args["country__country_code_alpha_2"] = country_code
 
         usd_amountwise_sorted = (
             Tender.objects.filter(

visualization/views/data_provider_view.py

@@ -15,6 +15,7 @@ def get(self, request):
         country_code = self.request.GET.get("country", None)
 
         if country_code:
+            country_code = str(country_code).upper()
             filter_args["country__country_code_alpha_2"] = country_code
 
         try: