chore: Delete import_tender, since all imports should be through CMS

open-contracting-archive · Mar 30, 2021 · 4d73dad · 4d73dad
1 parent 371a204
commit 4d73dad
Show file tree

Hide file tree

Showing 6 changed files with 0 additions and 272 deletions.
diff --git a/country/management/commands/import_tender.py b/country/management/commands/import_tender.py
diff --git a/country/tasks.py b/country/tasks.py
@@ -7,7 +7,6 @@
 import dateutil.parser
 
 # from celery import shared_task
-import gspread
 import pandas as pd
 import requests
 import xlsxwriter
@@ -121,169 +120,6 @@ def convert_local_to_usd(conversion_date, source_currency, source_value, dst_cur
             return None
 
 
-def save_tender_data_to_db(gs_sheet_url):
-    gc = gspread.service_account(filename=settings.GOOGLE_SHEET_CREDENTIALS_JSON)
-    covid_sheets = gc.open_by_url(gs_sheet_url)
-
-    try:
-        worksheet_settings = covid_sheets.worksheet("settings")
-        worksheet_data = covid_sheets.worksheet("data")
-
-        # Get country and currency from worksheet:settings
-        country = worksheet_settings.cell(2, 3).value
-
-        data_all = worksheet_data.get_all_records()
-
-        data = []
-        for i in data_all:
-            if not i["Contract ID"] and not i["Contract date (yyyy-mm-dd)"] and not i["Contract value"]:
-                # detect end of rows
-                break
-            else:
-                data.append(i)
-
-        total_rows_imported_count = 0
-        errors = []
-
-        for index, row in enumerate(data):
-            # with transaction.atomic():
-            try:
-                # TODO: Check if row already exists in database
-
-                # Get Country
-                country_obj = Country.objects.filter(name=country).first()
-
-                # Get or Create Supplier
-                supplier_id = row["Supplier ID"]
-                supplier_name = row["Supplier"]
-                supplier_address = row["Supplier address"]
-
-                if supplier_id:
-                    supplier_id = str(supplier_id).strip()
-                    supplier_obj = Supplier.objects.filter(supplier_id=supplier_id).first()
-                    if not supplier_obj:
-                        supplier_obj = Supplier(
-                            supplier_id=supplier_id,
-                            supplier_name=supplier_name,
-                            supplier_address=supplier_address,
-                        )
-                        supplier_obj.save()
-                else:
-                    supplier_obj = None
-
-                # Get or Create Buyer
-                buyer_id = row["Buyer ID"]
-                buyer_name = row["Buyer"]
-                buyer_address = row["Buyer address (as an object)"]
-
-                if buyer_id:
-                    buyer_id = str(buyer_id).strip()
-                    buyer_obj = Buyer.objects.filter(buyer_id=buyer_id).first()
-                    if not buyer_obj:
-                        buyer_obj = Buyer(
-                            buyer_id=buyer_id,
-                            buyer_name=buyer_name,
-                            buyer_address=buyer_address,
-                        )
-                        buyer_obj.save()
-                else:
-                    buyer_obj = None
-
-                # Get or Create Tender Contract
-                contract_id = row["Contract ID"]
-
-                if contract_id:
-                    contract_id = str(contract_id).strip()
-                    tender_obj = Tender.objects.filter(contract_id=contract_id).first()
-                    if not tender_obj:
-                        tender_obj = Tender(
-                            country=country_obj,
-                            supplier=supplier_obj,
-                            buyer=buyer_obj,
-                            contract_id=row["Contract ID"],
-                            contract_date=row["Contract date (yyyy-mm-dd)"],
-                            procurement_procedure=row["Procurement procedure code"],
-                            status=row["Contract Status Code"],
-                            link_to_contract=row["Link to the contract"],
-                            link_to_tender=row["Link to the tender"],
-                            data_source=row["Data source"],
-                            # for viz api compatibility only; remove these later
-                            contract_title=row["Contract title"],
-                            contract_value_local=row["Contract value"] or None,
-                            contract_desc=row["Contract description"],
-                            no_of_bidders=row["Number of bidders"] or None,
-                        )
-                        tender_obj.save()
-                else:
-                    tender_obj = None
-
-                # Get or Create GoodsServicesCategory
-                goods_services_category_name = row["Goods/Services"].strip()
-                goods_services_category_desc = ""
-
-                if goods_services_category_name:
-                    goods_services_category_obj = GoodsServicesCategory.objects.filter(
-                        category_name=goods_services_category_name
-                    ).first()
-                    if not goods_services_category_obj:
-                        goods_services_category_obj = GoodsServicesCategory(
-                            category_name=goods_services_category_name, category_desc=goods_services_category_desc
-                        )
-                        goods_services_category_obj.save()
-                else:
-                    goods_services_category_obj = None
-
-                # Create GoodsServices...
-
-                if tender_obj:
-                    # ...only if there is a contract that it can be associated with
-                    goods_services_obj = GoodsServices(
-                        country=country_obj,
-                        goods_services_category=goods_services_category_obj,
-                        contract=tender_obj,
-                        classification_code=row["Classification Code (CPV or other)"],
-                        no_of_bidders=row["Number of bidders"] or None,
-                        contract_title=row["Contract title"],
-                        contract_desc=row["Contract description"],
-                        tender_value_local=row["Tender value"] or None,
-                        award_value_local=row["Award value"] or None,
-                        contract_value_local=row["Contract value"] or None,
-                    )
-                    goods_services_obj.save()
-
-                    print(tender_obj.id, goods_services_obj.id)
-
-                    # Execute local currency to USD conversion celery tasks
-                    conversion_date = row["Contract date (yyyy-mm-dd)"]
-                    source_currency = country_obj.currency
-                    source_values = {
-                        "tender_value_local": row["Tender value"] or None,
-                        "award_value_local": row["Award value"] or None,
-                        "contract_value_local": row["Contract value"] or None,
-                    }
-                    goods_services_row_id = goods_services_obj.id
-                    local_currency_to_usd.apply_async(
-                        args=(goods_services_row_id, conversion_date, source_currency, source_values), queue="covid19"
-                    )
-
-                total_rows_imported_count += 1
-            except:
-                # transaction.rollback()
-
-                contract_id = row["Contract ID"]
-                errors.append((index, contract_id))
-                print("------------------------------")
-                print(f"Error importing row {index}, contract id {contract_id}")
-                print(f"{row}\n")
-                traceback.print_exc(file=sys.stdout)
-                print("------------------------------")
-
-        print(f"Total rows imported: {total_rows_imported_count}")
-        print(errors)
-    except Exception as e:
-        print(e)
-
-
 @app.task(name="import_tender_from_batch_id")
 def import_tender_from_batch_id(batch_id, country, currency):
     print(f"import_tender_data from Batch_id {batch_id}")
@@ -471,12 +307,6 @@ def import_tender_from_batch_id(batch_id, country, currency):
     DataImport.objects.filter(page_ptr_id=data_import_id).update(imported=True)
 
 
-@app.task(name="import_tender_data")
-def import_tender_data(gs_sheet_url):
-    print(f"import_tender_data from {gs_sheet_url}")
-    save_tender_data_to_db(gs_sheet_url)
-
-
 @app.task(name="local_currency_to_usd")
 def local_currency_to_usd(goods_services_row_id, conversion_date, source_currency, source_values):
     dst_tender_value = convert_local_to_usd(

diff --git a/covidadmin/settings.py b/covidadmin/settings.py
@@ -207,7 +207,6 @@
 CELERY_TIMEZONE = env("CELERY_TIMEZONE")
 CELERY_TASK_ROUTES = {
     "country.tasks.fetch_covid_data": {"queue": "covid19", "exchange": "covid19", "routing_key": "covid19"},
-    "country.tasks.import_tender_data": {"queue": "covid19", "exchange": "covid19", "routing_key": "covid19"},
     "country.tasks.local_currency_to_usd": {"queue": "covid19", "exchange": "covid19", "routing_key": "covid19"},
 }
 CELERY_BEAT_SCHEDULE = {

diff --git a/requirements.in b/requirements.in
@@ -8,7 +8,6 @@ django-modelcluster
 django-taggit
 djangorestframework
 flower
-gspread
 pandas
 psycopg2
 python-dateutil

diff --git a/requirements.txt b/requirements.txt
@@ -14,8 +14,6 @@ beautifulsoup4==4.8.2
     # via wagtail
 billiard==3.6.3.0
     # via celery
-cachetools==4.1.1
-    # via google-auth
 celery==5.0.2
     # via
     #   -r requirements.in
@@ -76,14 +74,6 @@ et-xmlfile==1.0.1
     # via openpyxl
 flower==0.9.5
     # via -r requirements.in
-google-auth-oauthlib==0.4.2
-    # via gspread
-google-auth==1.23.0
-    # via
-    #   google-auth-oauthlib
-    #   gspread
-gspread==3.6.0
-    # via -r requirements.in
 html5lib==1.1
     # via wagtail
 humanize==3.1.0
@@ -100,8 +90,6 @@ l18n==2020.6.1
     # via wagtail
 numpy==1.19.4
     # via pandas
-oauthlib==3.1.0
-    # via requests-oauthlib
 openpyxl==3.0.5
     # via tablib
 pandas==1.1.5
@@ -114,12 +102,6 @@ prompt-toolkit==3.0.8
     # via click-repl
 psycopg2==2.8.6
     # via -r requirements.in
-pyasn1-modules==0.2.8
-    # via google-auth
-pyasn1==0.4.8
-    # via
-    #   pyasn1-modules
-    #   rsa
 python-dateutil==2.8.1
     # via
     #   -r requirements.in
@@ -132,22 +114,15 @@ pytz==2020.1
     #   flower
     #   l18n
     #   pandas
-requests-oauthlib==1.3.0
-    # via google-auth-oauthlib
 requests==2.24.0
     # via
     #   -r requirements.in
-    #   gspread
-    #   requests-oauthlib
     #   wagtail
-rsa==4.6
-    # via google-auth
 sentry-sdk==1.0.0
     # via -r requirements.in
 six==1.15.0
     # via
     #   click-repl
-    #   google-auth
     #   html5lib
     #   l18n
     #   python-dateutil

diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -34,10 +34,6 @@ billiard==3.6.3.0
     #   celery
 black==20.8b1
     # via -r requirements_dev.in
-cachetools==4.1.1
-    # via
-    #   -r requirements.txt
-    #   google-auth
 celery==5.0.2
     # via
     #   -r requirements.txt
@@ -140,17 +136,6 @@ gitdb==4.0.5
     # via gitpython
 gitpython==3.1.10
     # via transifex-client
-google-auth-oauthlib==0.4.2
-    # via
-    #   -r requirements.txt
-    #   gspread
-google-auth==1.23.0
-    # via
-    #   -r requirements.txt
-    #   google-auth-oauthlib
-    #   gspread
-gspread==3.6.0
-    # via -r requirements.txt
 html5lib==1.1
     # via
     #   -r requirements.txt
@@ -210,10 +195,6 @@ numpy==1.19.4
     # via
     #   -r requirements.txt
     #   pandas
-oauthlib==3.1.0
-    # via
-    #   -r requirements.txt
-    #   requests-oauthlib
 openpyxl==3.0.5
     # via
     #   -r requirements.txt
@@ -251,15 +232,6 @@ psycopg2==2.8.6
     # via -r requirements.txt
 ptyprocess==0.6.0
     # via pexpect
-pyasn1-modules==0.2.8
-    # via
-    #   -r requirements.txt
-    #   google-auth
-pyasn1==0.4.8
-    # via
-    #   -r requirements.txt
-    #   pyasn1-modules
-    #   rsa
 pycodestyle==2.6.0
     # via flake8
 pyflakes==2.2.0
@@ -285,29 +257,18 @@ pyyaml==5.4.1
     # via pre-commit
 regex==2021.3.17
     # via black
-requests-oauthlib==1.3.0
-    # via
-    #   -r requirements.txt
-    #   google-auth-oauthlib
 requests==2.24.0
     # via
     #   -r requirements.txt
     #   coveralls
-    #   gspread
-    #   requests-oauthlib
     #   transifex-client
     #   wagtail
-rsa==4.6
-    # via
-    #   -r requirements.txt
-    #   google-auth
 sentry-sdk==1.0.0
     # via -r requirements.txt
 six==1.15.0
     # via
     #   -r requirements.txt
     #   click-repl
-    #   google-auth
     #   html5lib
     #   l18n
     #   python-dateutil