Merge pull request #5424 from uktrade/fix/export-wins-migration-6

Improve adviser matching and is_active support for legacy export wins.

datahub/export_win/legacy_migration.py

@@ -1,7 +1,11 @@
 from datetime import datetime
 
+from logging import getLogger
+
 from dateutil.parser import parse
 
+from django.db.models import Q
+
 from datahub.company.models import (
     Advisor,
     Contact,
@@ -38,6 +42,9 @@
 from datahub.metadata.query_utils import get_sector_name_subquery
 
 
+logger = getLogger(__name__)
+
+
 def create_customer_response_from_legacy(win, item):
     must_resolvers = {
         'access_to_contacts': resolve_legacy_field(
@@ -204,6 +211,10 @@ def create_export_win_from_legacy(item):
         'migrated_on': lambda item, context: datetime.now(),
         'created_on': lambda item, context: parse(item['created']),
         'audit': lambda item, context: '' if item['audit'] is None else item['audit'],
+        'is_deleted': lambda item, context: False if item.get(
+            'is_active',
+            None,
+        ) is None else (not item['is_active']),
     }
 
     # fields to be copied over directly
@@ -226,7 +237,6 @@ def create_export_win_from_legacy(item):
     win = {
         'id': item.get('id'),
     }
-
     for field, resolver in must_resolvers.items():
         resolved = resolver(item, win)
         if isinstance(resolved, dict):
@@ -262,7 +272,11 @@ def migrate_legacy_win(item):
         for field in many_to_many_fields if field in win_data
     }
     win_id = win_data.pop('id')
-    win, created = Win.objects.update_or_create(
+    if win_data['country'] is None:
+        logger.warning(f'Country not found for {win_id}.')
+        return
+
+    win, created = Win.objects.all_wins().update_or_create(
         id=win_id,
         defaults=win_data,
     )
@@ -290,7 +304,7 @@ def migrate_legacy_win(item):
 
 
 def create_breakdown_from_legacy(item):
-    win = Win.objects.get(id=item['win__id'])
+    win = Win.objects.all_wins().get(id=item['win__id'])
     breakdown_type = BreakdownType.objects.get(export_win_id=item['type'])
     year = item['year'] - get_financial_year(win.date) + 1
     return {
@@ -366,16 +380,19 @@ def migrate_legacy_win_adviser(item):
     return adviser
 
 
-def resolve_legacy_field(model, source_field_name, lookup_field=None, annotate=None):
+def resolve_legacy_field(model, source_field_name, lookup_field=None, annotate=None, remap=None):
     if not lookup_field:
         lookup_field = 'export_win_id'
     if not annotate:
         annotate = {}
+    if not remap:
+        remap = {}
 
     def resolver(data, context=None):
-        field_value = data.get(source_field_name)
-        if field_value == '':
+        _field_value = data.get(source_field_name)
+        if _field_value == '':
             return None
+        field_value = remap.get(_field_value, _field_value)
         try:
             obj = model.objects.annotate(**annotate).get(**{lookup_field: field_value})
             return obj
@@ -413,9 +430,9 @@ def resolve_company(data, context=None):
 
 
 def resolve_adviser(data, context=None):
+    email = data.get('user__email').strip()
     adviser = Advisor.objects.filter(
-        contact_email__iexact=data.get('user__email').strip(),
-        is_active=True,
+        Q(contact_email__iexact=email) | Q(email__iexact=email),
     ).order_by('-date_joined').first()
     if adviser is None:
         return {
@@ -426,23 +443,22 @@ def resolve_adviser(data, context=None):
 
 
 def resolve_lead_officer(data, context=None):
-    criteria = {'is_active': True}
+    filters = Q()
     email = data.get('lead_officer_email_address').strip()
     if email != '':
-        criteria.update({
-            'contact_email__iexact': email,
-        })
+        filters = Q(email__iexact=email) | Q(contact_email__iexact=email)
     else:
         parts = data.get('lead_officer_name').split()
         # In case name is written as "Joe M. Doe"
         first_name = parts[0]
         last_name = parts[-1]
-        criteria.update({
-            'first_name__iexact': first_name.strip(),
-            'last_name__iexact': last_name.strip(),
-        })
+        filters = Q(
+            first_name__iexact=first_name.strip(),
+        ) & Q(
+            last_name__iexact=last_name.strip(),
+        )
     adviser = Advisor.objects.filter(
-        **criteria,
+        filters,
     ).order_by('-date_joined').first()
     if adviser is None:
         return {
@@ -460,7 +476,6 @@ def resolve_line_manager(data, context=None):
     adviser = Advisor.objects.filter(
         first_name__iexact=first_name.strip(),
         last_name__iexact=last_name.strip(),
-        is_active=True,
     ).order_by('-date_joined').first()
     if adviser is None:
         return {

datahub/export_win/models.py

@@ -46,6 +46,9 @@ def soft_deleted(self, *args, **kwargs):
             .filter(is_deleted=True)
         )
 
+    def all_wins(self, *args, **kwargs):
+        return super().get_queryset(*args, **kwargs)
+
 
 class BaseCustomerResponseSoftDeleteManager(models.Manager):
     """Base class for Customer response soft delete manager."""

datahub/export_win/test/test_legacy_migration.py

@@ -287,11 +287,118 @@
             'confirmation__comments': None,
             'confirmation__name': None,
             'confirmation__other_marketing_source': None,
+        }, {
+            'associated_programme_1': None,
+            'associated_programme_2': None,
+            'associated_programme_3': None,
+            'associated_programme_4': None,
+            'associated_programme_5': None,
+            'audit': None,
+            'business_potential': None,
+            'business_type': '8',
+            'cdms_reference': 'cdms reference',
+            'company_name': 'company name',
+            'complete': False,
+            'country': 'AL',
+            'country_name': 'abc',
+            'created': '2016-06-24T11:32:42.134323Z',
+            'customer_email_address': 'test@test.com',
+            'customer_job_title': 'customer job title',
+            'customer_location': 3,
+            'customer_name': 'customer name',
+            'date': '2016-01-01',
+            'description': 'asdf',
+            'export_experience': None,
+            'goods_vs_services': 1,
+            'has_hvo_specialist_involvement': False,
+            'confirmation__comments': None,
+            'hq_team': 'team:1',
+            'hvc': None,
+            'hvo_programme': 'AER-01',
+            'id': 'c84caade-4fae-4af0-816f-e27a8c5ded95',
+            'is_e_exported': True,
+            'is_line_manager_confirmed': True,
+            'is_personally_confirmed': True,
+            'is_prosperity_fund_related': True,
+            'lead_officer_email_address': '',
+            'lead_officer_name': 'Lead officer name',
+            'line_manager_name': 'line manager',
+            'name_of_customer': '',
+            'name_of_export': '',
+            'other_official_email_address': '',
+            'sector_display': None,
+            'team_type': 'team',
+            'type_of_support_1': 1,
+            'type_of_support_2': None,
+            'type_of_support_3': None,
+            'user__email': 'abc@test',
+            'user__name': 'Abc Def',
+            'confirmation__comments': None,
+            'confirmation__name': None,
+            'confirmation__other_marketing_source': None,
+        }, {
+            'associated_programme_1': None,
+            'associated_programme_2': None,
+            'associated_programme_3': None,
+            'associated_programme_4': None,
+            'associated_programme_5': None,
+            'audit': None,
+            'business_potential': None,
+            'business_type': '8',
+            'cdms_reference': 'cdms reference',
+            'company_name': 'company name',
+            'complete': False,
+            'country': 'AL',
+            'country_name': 'Albania',
+            'created': '2016-06-24T11:32:42.134323Z',
+            'customer_email_address': 'test@test.com',
+            'customer_job_title': 'customer job title',
+            'customer_location': 3,
+            'customer_name': 'customer name',
+            'date': '2016-01-01',
+            'description': 'asdf',
+            'export_experience': None,
+            'goods_vs_services': 1,
+            'has_hvo_specialist_involvement': False,
+            'confirmation__comments': None,
+            'hq_team': 'team:1',
+            'hvc': None,
+            'hvo_programme': 'AER-01',
+            'id': '5778b485-1060-46e2-b411-772cd0f76d79',
+            'is_e_exported': True,
+            'is_line_manager_confirmed': True,
+            'is_personally_confirmed': True,
+            'is_prosperity_fund_related': True,
+            'lead_officer_email_address': 'user.email2@trade.gov.uk',
+            'lead_officer_name': 'Lead officer name',
+            'line_manager_name': 'line manager',
+            'name_of_customer': '',
+            'name_of_export': '',
+            'other_official_email_address': '',
+            'sector_display': None,
+            'team_type': 'team',
+            'type_of_support_1': 1,
+            'type_of_support_2': None,
+            'type_of_support_3': None,
+            'user__email': 'abc@test',
+            'user__name': 'Abc Def',
+            'confirmation__comments': None,
+            'confirmation__name': None,
+            'confirmation__other_marketing_source': None,
+            'is_active': False,
         }],
     },
     mock_legacy_wins_page_urls['breakdowns'][0]: {
         'next': mock_legacy_wins_page_urls['breakdowns'][1],
         'results': [
+            {
+                'id': 11,
+                'win__id': '5778b485-1060-46e2-b411-772cd0f76d79',
+                'type': 1,
+                'year': 2016,
+                'value': 3000,
+                'is_active': False,
+            },
             {
                 'id': 5,
                 'win__id': '4c90a214-035f-4445-b6a1-ca7af3486f8c',
@@ -443,8 +550,8 @@ def test_legacy_migration(mock_legacy_wins_pages):
         contact_email='user.email@trade.gov.uk',
         is_active=True,
     )
-    AdviserFactory(
-        contact_email='user.email@trade.gov.uk',
+    adviser2 = AdviserFactory(
+        email='user.email2@trade.gov.uk',
         is_active=False,
     )
     current_date = datetime.utcnow()
@@ -550,3 +657,31 @@ def test_legacy_migration(mock_legacy_wins_pages):
     assert win_4.customer_response.comments == ''
     assert win_4.customer_response.other_marketing_source == ''
     assert win_4.line_manager == line_manager
+
+    assert Win.objects.filter(id='c84caade-4fae-4af0-816f-e27a8c5ded95').exists() is False
+
+    win_6 = Win.objects.all_wins().get(id='5778b485-1060-46e2-b411-772cd0f76d79')
+    assert win_6.company_contacts.count() == 0
+    assert win_6.company is None
+    assert win_6.company_name == 'company name'
+    assert win_6.customer_name == 'customer name'
+    assert win_6.lead_officer == adviser2
+    assert win_6.lead_officer_name == ''
+    assert win_6.adviser_email_address == 'abc@test'
+    assert win_6.adviser is None
+    assert win_6.total_expected_export_value == 3000
+    assert win_6.total_expected_non_export_value == 0
+    assert win_6.total_expected_odi_value == 0
+    assert win_6.breakdowns.count() == 1
+    assert win_6.advisers.count() == 0
+    assert win_6.migrated_on == current_date
+    win_6_created_on = parser.parse('2016-06-24T11:32:42.134323Z').astimezone(timezone.utc)
+    assert win_6.created_on == win_6_created_on
+    assert win_6.customer_response.created_on == win_6_created_on
+    assert win_6.customer_response.responded_on is None
+    assert win_6.sector_id is None
+    assert win_6.customer_response.name == ''
+    assert win_6.customer_response.comments == ''
+    assert win_6.customer_response.other_marketing_source == ''
+    assert win_6.line_manager == line_manager
+    assert win_6.is_deleted is True