197-change-in-tender-table

[x] Added temp_table_id in tender table
[x] Added quantity and unit_price in goodsservices
[x] Added psycopg2==2.8.6 in requirement.txt file
[x] formatting applied
[x] Formatted in black
[x] Aded auto_optimize in serializer for optimization
[x] Resolved merge request issues

.gitignore

@@ -47,3 +47,6 @@ erl_crash.dump
 # Deployment scripts used by Young Innovations
  /script.sh
  /deploy.sh
+
+#Pycharm
+*.idea/

content/signals.py

@@ -1,12 +1,11 @@
-from django.db.models.signals import post_save
-from django.dispatch import receiver
+# from django.db.models.signals import post_save
+# from django.dispatch import receiver
 
-from content.models import DataImport
-from country.tasks import store_in_temp_table
+# from content.models import DataImport
+# from country.tasks import store_in_temp_table
 
-
-@receiver(post_save, sender=DataImport)
-def validation_and_temp_table_store(sender, created, instance, *args, **kwargs):
-    if created:
-        print("Data validation and temp table storage task started!")
-        store_in_temp_table.apply_async(args=(instance.id,), queue="covid19")
+# # @receiver(post_save, sender=DataImport)
+# # def validation_and_temp_table_store(sender,created ,instance, *args, **kwargs):
+# #     if created:
+# #         print("Data validation and temp table storage task started!")
+# #         store_in_temp_table.apply_async(args=(a.id,), queue='covid19')

country/admin.py

@@ -87,6 +87,22 @@ def import_status(self):
 
     import_status.short_description = "Import Status"
 
+    def validate(self):
+        data_import_id = str(self.page_ptr_id)
+        if self.validated:
+            return format_html(
+                f"""<a class="button" disabled="True" href="/data_validate?data_import_id={data_import_id}">
+                Validate</a>&nbsp;"""
+            )
+        else:
+            return format_html(
+                f"""<a class="button" onClick="this.disabled = true;" href="/data_validate?
+                data_import_id={data_import_id}">
+                Validate</a>&nbsp;"""
+            )
+
+    validate.short_description = "Validate"
+
     def import_actions(self):
         data_import_id = str(self.page_ptr_id)
         importbatch = ImportBatch.objects.get(data_import_id=data_import_id)
@@ -102,9 +118,7 @@ def import_actions(self):
                 <a class="button" href={file_source} download>Download Source File</a>&nbsp;"""
             )
 
-    import_actions.short_description = "Import Actions"
-
-    list_display = ("title", "description", "country", custom_title, import_status, import_actions)
+    list_display = ("title", "description", "country", custom_title, import_status, import_actions, validate)
 
 
 @admin.register(DataProvider)

country/migrations/0028_tender_temp_table_id.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.1.2 on 2021-03-25 10:31
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country", "0027_redflag_implemented"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="tender",
+            name="temp_table_id",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="tenders",
+                to="country.tempdataimporttable",
+            ),
+        ),
+    ]

country/migrations/0029_auto_20210325_1106.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.2 on 2021-03-25 11:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country", "0028_tender_temp_table_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="goodsservices",
+            name="ppu_including_vat",
+            field=models.CharField(max_length=1500, null=True, verbose_name="Price per units including VAT"),
+        ),
+        migrations.AddField(
+            model_name="goodsservices",
+            name="quantity_units",
+            field=models.CharField(max_length=1500, null=True, verbose_name="Quantity,units"),
+        ),
+    ]

country/models.py

@@ -158,8 +158,8 @@ def get_queryset(self):
             super()
             .get_queryset()
             .annotate(
-                amount_local=Sum("tenders__goods_services__contract_value_local", distinct=True),
-                amount_usd=Sum("tenders__goods_services__contract_value_usd", distinct=True),
+                amount_local=Sum("tenders__goods_services__contract_value_local"),
+                amount_usd=Sum("tenders__goods_services__contract_value_usd"),
                 country_name=F("tenders__country__name"),
                 red_flag_count=Count("tenders__red_flag", distinct=True),
                 product_category_count=Count("tenders__goods_services__goods_services_category", distinct=True),
@@ -175,8 +175,8 @@ def get_queryset(self):
             super()
             .get_queryset()
             .annotate(
-                amount_local=Sum("tenders__goods_services__contract_value_local", distinct=True),
-                amount_usd=Sum("tenders__goods_services__contract_value_usd", distinct=True),
+                amount_usd=Sum("tenders__goods_services__contract_value_usd"),
+                amount_local=Sum("tenders__goods_services__contract_value_local"),
                 country_name=F("tenders__country__name"),
                 red_flag_count=Count("tenders__red_flag", distinct=True),
                 product_category_count=Count("tenders__goods_services__goods_services_category", distinct=True),
@@ -188,20 +188,6 @@ def get_queryset(self):
         )
 
 
-class TenderManager(models.Manager):
-    def get_queryset(self):
-        return (
-            super()
-            .get_queryset()
-            .annotate(
-                amount_local=Sum("goods_services__contract_value_local"),
-                amount_usd=Sum("goods_services__contract_value_usd"),
-                red_flag_count=Count("red_flag"),
-                tender_count=Count("id", distinct=True),
-            )
-        )
-
-
 class Supplier(models.Model):
     supplier_id = models.CharField(verbose_name=_("Supplier ID"), max_length=50, null=True)
     supplier_name = models.CharField(
@@ -272,6 +258,50 @@ def __str__(self):
         return self.title
 
 
+class ImportBatch(models.Model):
+    country = models.ForeignKey(Country, on_delete=models.CASCADE, related_name="import_batch_country", null=True)
+    import_type = models.CharField(verbose_name=("Import Type"), max_length=150, null=False)
+    description = models.CharField(verbose_name=("Description"), max_length=500, null=False)
+    data_import_id = models.IntegerField(null=True)
+
+    def __str__(self):
+        return f"Import batch id: {str(self.id)}"
+
+
+class TempDataImportTable(models.Model):
+    import_batch = models.ForeignKey(ImportBatch, on_delete=models.CASCADE, related_name="import_batch", null=True)
+    contract_id = models.CharField(verbose_name=("Contract Id"), max_length=1500, null=True)
+    contract_date = models.CharField(verbose_name=("Contract date"), max_length=1500, null=True)
+    procurement_procedure = models.CharField(verbose_name=("Procurement procedure"), max_length=1500, null=True)
+    procurement_process = models.CharField(verbose_name=("Procurement process"), max_length=1500, null=True)
+    goods_services = models.CharField(verbose_name=("Goods/Services"), max_length=1500, null=True)
+    cpv_code_clear = models.CharField(verbose_name=("CPV code clear"), max_length=1500, null=True)
+    quantity_units = models.CharField(verbose_name=("Quantity,units"), max_length=1500, null=True)
+    ppu_including_vat = models.CharField(verbose_name=("Price per units including VAT"), max_length=1500, null=True)
+    tender_value = models.CharField(verbose_name=("Tender value"), max_length=1500, null=True)
+    award_value = models.CharField(verbose_name=("Award value"), max_length=1500, null=True)
+    contract_value = models.CharField(verbose_name=("Contract value"), max_length=1500, null=True)
+    contract_title = models.CharField(verbose_name=("Contract title"), max_length=1500, null=True)
+    contract_description = models.CharField(verbose_name=("Award value"), max_length=1500, null=True)
+    no_of_bidders = models.CharField(verbose_name=("No of bidders"), max_length=1500, null=True)
+    buyer = models.CharField(verbose_name=("Buyer"), max_length=1500, null=True)
+    buyer_id = models.CharField(verbose_name=("Buyer ID"), max_length=150, null=True)
+    buyer_address_as_an_object = models.CharField(
+        verbose_name=("Buyer Address(as an object)"), max_length=1500, null=True
+    )
+    supplier = models.CharField(verbose_name=("Supplier"), max_length=1500, null=True)
+    supplier_id = models.CharField(verbose_name=("Supplier ID"), max_length=1500, null=True)
+    supplier_address = models.CharField(verbose_name=("Supplier Address"), max_length=1500, null=True)
+    contract_status = models.CharField(verbose_name=("Contract Status"), max_length=1500, null=True)
+    contract_status_code = models.CharField(verbose_name=("Contract Status Code"), max_length=1500, null=True)
+    link_to_contract = models.CharField(verbose_name=("Link to Contract"), max_length=1500, null=True)
+    link_to_tender = models.CharField(verbose_name=("Link to Tender"), max_length=1500, null=True)
+    data_source = models.CharField(verbose_name=("Data Source"), max_length=1500, null=True)
+
+    def __str__(self):
+        return self.contract_id
+
+
 class Tender(models.Model):
     PROCUREMENT_METHOD_CHOICES = [
         ("open", "open"),
@@ -311,6 +341,9 @@ class Tender(models.Model):
     equity_categories = ArrayField(models.CharField(max_length=100, null=True, blank=True), default=list, null=True)
     equity_category = models.ManyToManyField(EquityCategory)
     red_flag = models.ManyToManyField(RedFlag)
+    temp_table_id = models.ForeignKey(
+        TempDataImportTable, on_delete=models.CASCADE, related_name="tenders", null=True, blank=True
+    )
 
     def __str__(self):
         return self.contract_id
@@ -346,6 +379,9 @@ class GoodsServices(models.Model):
     )
     buyer = models.ForeignKey(Buyer, on_delete=models.CASCADE, related_name="goods_services", null=True, blank=True)
 
+    quantity_units = models.CharField(verbose_name=("Quantity,units"), max_length=1500, null=True)
+    ppu_including_vat = models.CharField(verbose_name=("Price per units including VAT"), max_length=1500, null=True)
+
     tender_value_local = models.FloatField(verbose_name=_("Tender value local"), null=True, blank=True)
     tender_value_usd = models.FloatField(verbose_name=_("Tender value USD"), null=True, blank=True)
     award_value_local = models.FloatField(verbose_name=_("Award value local"), null=True, blank=True)
@@ -374,50 +410,6 @@ class EquityKeywords(models.Model):
     keyword = models.CharField(verbose_name=_("Keyword"), max_length=100, null=False)
 
 
-class ImportBatch(models.Model):
-    country = models.ForeignKey(Country, on_delete=models.CASCADE, related_name="import_batch_country", null=True)
-    import_type = models.CharField(verbose_name=("Import Type"), max_length=150, null=False)
-    description = models.CharField(verbose_name=("Description"), max_length=500, null=False)
-    data_import_id = models.IntegerField(null=True)
-
-    def __str__(self):
-        return f"Import batch id: {str(self.id)}"
-
-
-class TempDataImportTable(models.Model):
-    import_batch = models.ForeignKey(ImportBatch, on_delete=models.CASCADE, related_name="import_batch", null=True)
-    contract_id = models.CharField(verbose_name=("Contract Id"), max_length=1500, null=True)
-    contract_date = models.CharField(verbose_name=("Contract date"), max_length=1500, null=True)
-    procurement_procedure = models.CharField(verbose_name=("Procurement procedure"), max_length=1500, null=True)
-    procurement_process = models.CharField(verbose_name=("Procurement process"), max_length=1500, null=True)
-    goods_services = models.CharField(verbose_name=("Goods/Services"), max_length=1500, null=True)
-    cpv_code_clear = models.CharField(verbose_name=("CPV code clear"), max_length=1500, null=True)
-    quantity_units = models.CharField(verbose_name=("Quantity,units"), max_length=1500, null=True)
-    ppu_including_vat = models.CharField(verbose_name=("Price per units including VAT"), max_length=1500, null=True)
-    tender_value = models.CharField(verbose_name=("Tender value"), max_length=1500, null=True)
-    award_value = models.CharField(verbose_name=("Award value"), max_length=1500, null=True)
-    contract_value = models.CharField(verbose_name=("Contract value"), max_length=1500, null=True)
-    contract_title = models.CharField(verbose_name=("Contract title"), max_length=1500, null=True)
-    contract_description = models.CharField(verbose_name=("Award value"), max_length=1500, null=True)
-    no_of_bidders = models.CharField(verbose_name=("No of bidders"), max_length=1500, null=True)
-    buyer = models.CharField(verbose_name=("Buyer"), max_length=1500, null=True)
-    buyer_id = models.CharField(verbose_name=("Buyer ID"), max_length=150, null=True)
-    buyer_address_as_an_object = models.CharField(
-        verbose_name=("Buyer Address(as an object)"), max_length=1500, null=True
-    )
-    supplier = models.CharField(verbose_name=("Supplier"), max_length=1500, null=True)
-    supplier_id = models.CharField(verbose_name=("Supplier ID"), max_length=1500, null=True)
-    supplier_address = models.CharField(verbose_name=("Supplier Address"), max_length=1500, null=True)
-    contract_status = models.CharField(verbose_name=("Contract Status"), max_length=1500, null=True)
-    contract_status_code = models.CharField(verbose_name=("Contract Status Code"), max_length=1500, null=True)
-    link_to_contract = models.CharField(verbose_name=("Link to Contract"), max_length=1500, null=True)
-    link_to_tender = models.CharField(verbose_name=("Link to Tender"), max_length=1500, null=True)
-    data_source = models.CharField(verbose_name=("Data Source"), max_length=1500, null=True)
-
-    def __str__(self):
-        return self.contract_id
-
-
 class DataProvider(models.Model):
     alphaSpaces = RegexValidator(r"^[a-zA-Z ]+$", "Only letters and spaces are allowed in the Country Name")
     name = models.CharField(verbose_name=_("Name"), null=False, unique=True, max_length=50, validators=[alphaSpaces])

country/serializers.py

@@ -311,6 +311,7 @@ class TenderSerializer(serializers.ModelSerializer):
     equity_category = serializers.SerializerMethodField()
     # red_flag_count = serializers.SerializerMethodField()
     red_flag = RedFlagSerializer(many=True, read_only=True)
+    goods_services = serializers.SerializerMethodField()
 
     class Meta:
         model = Tender
@@ -348,6 +349,7 @@ class Meta:
             "equity_category",
             # 'red_flag_count',
             "red_flag",
+            "goods_services",
         )
         read_only_fields = (
             "contract_value_usd",
@@ -450,6 +452,17 @@ def get_buyer_code(self, obj):
         if obj.buyer:
             return obj.buyer.buyer_id
 
+    def get_goods_services(self, obj):
+        a = obj.goods_services.all().values(
+            "goods_services_category__category_name",
+            "contract_value_usd",
+            "quantity_units",
+            "ppu_including_vat",
+            "contract_value_local",
+            "classification_code",
+        )
+        return list(a)
+
 
 class OverallStatSummarySerializer(serializers.ModelSerializer):
     country = CountrySerializer(many=False, read_only=True)

country/tasks.py

@@ -149,15 +149,15 @@ def import_tender_from_batch_id(batch_id, country, currency):
             if row.tender_value:
                 tender_value_local = float(row.tender_value)
             else:
-                tender_value_local = ""
+                tender_value_local = 0
             if row.award_value:
                 award_value_local = float(row.award_value)
             else:
-                award_value_local = ""
+                award_value_local = 0
             if row.contract_value:
                 contract_value_local = float(row.contract_value)
             else:
-                contract_value_local = ""
+                contract_value_local = 0
 
             contract_title = row.contract_title
             contract_desc = row.contract_description
@@ -178,6 +178,9 @@ def import_tender_from_batch_id(batch_id, country, currency):
             if status == "Terminated" or status == "Canclled":
                 status = "canceled"
 
+            quantity_units = row.quantity_units
+            ppu_including_vat = row.ppu_including_vat
+
             link_to_contract = row.link_to_contract
             link_to_tender = row.link_to_tender
             data_source = row.data_source
@@ -238,6 +241,7 @@ def import_tender_from_batch_id(batch_id, country, currency):
                         contract_value_local=contract_value_local or None,
                         contract_desc=contract_desc,
                         no_of_bidders=no_of_bidders or None,
+                        temp_table_id=row,
                     )
                     tender_obj.save()
             else:
@@ -270,9 +274,11 @@ def import_tender_from_batch_id(batch_id, country, currency):
                     no_of_bidders=no_of_bidders or None,
                     contract_title=contract_title,
                     contract_desc=contract_desc,
-                    tender_value_local=tender_value_local or None,
-                    award_value_local=award_value_local or None,
-                    contract_value_local=contract_value_local or None,
+                    tender_value_local=tender_value_local or 0,
+                    award_value_local=award_value_local or 0,
+                    contract_value_local=contract_value_local or 0,
+                    quantity_units=quantity_units or None,
+                    ppu_including_vat=ppu_including_vat or None,
                 )
                 goods_services_obj.save()
 
@@ -507,10 +513,8 @@ def store_in_temp_table(instance_id):
         i = 0
 
         while i <= len(ws):
-            procurement_procedure_value = ws["Procurement procedure code"][i].strip().lower()
-            contract_status_value = ws["Contract Status Code"][i].strip().lower()
-            print(procurement_procedure_value)
-            print(contract_status_value)
+            procurement_procedure_value = str(ws["Procurement procedure code"][i]).strip().lower()
+            contract_status_value = str(ws["Contract Status Code"][i]).strip().lower()
 
             if contract_status_value in contract_status_option:
                 contract_status_lowered_value = contract_status_value.lower()

country/urls.py

@@ -14,5 +14,6 @@
 urlpatterns = [
     path("", include(router.urls)),
     path("data_import/", views.DataImportView.as_view(), name="data_imports"),
+    path("data_validate/", views.DataValidateView.as_view(), name="data_validate"),
     path("data_edit/", views.DataEditView.as_view(), name="data_edits"),
 ]