diff --git a/seed/data_importer/tests/data/example-data-properties-postal.xlsx b/seed/data_importer/tests/data/example-data-properties-postal.xlsx new file mode 100644 index 0000000000..1eaeefc57c Binary files /dev/null and b/seed/data_importer/tests/data/example-data-properties-postal.xlsx differ diff --git a/seed/data_importer/tests/integration/test_data_import.py b/seed/data_importer/tests/integration/test_data_import.py index 7078049365..2e3311bbcb 100644 --- a/seed/data_importer/tests/integration/test_data_import.py +++ b/seed/data_importer/tests/integration/test_data_import.py @@ -389,3 +389,57 @@ def import_exported_data(self, filename): # call the mapping function from the tasks file map_data(self.import_file.id) + + +class TestPostalCode(DataMappingBaseTestCase): + def setUp(self): + filename = getattr(self, 'filename', 'example-data-properties-postal.xlsx') + import_file_source_type = ASSESSED_RAW + self.fake_mappings = FAKE_MAPPINGS + selfvars = self.set_up(import_file_source_type) + self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars + filepath = osp.join(osp.dirname(__file__), '..', 'data', filename) + self.import_file.file = SimpleUploadedFile( + name=filename, + content=open(filepath, 'rb').read() + ) + self.import_file.save() + + def test_postal_code_property(self): + + new_mappings = copy.deepcopy(self.fake_mappings['portfolio']) + + tasks.save_raw_data(self.import_file.pk) + Column.create_mappings(new_mappings, self.org, self.user, self.import_file.pk) + tasks.map_data(self.import_file.pk) + + # get mapped property postal_code + ps = PropertyState.objects.filter(address_line_1='11 Ninth Street')[0] + self.assertEqual(ps.postal_code, '00340') + + ps = PropertyState.objects.filter(address_line_1='20 Tenth Street')[0] + self.assertEqual(ps.postal_code, '00000') + + ps = PropertyState.objects.filter(address_line_1='93029 Wellington Blvd')[0] + self.assertEqual(ps.postal_code, '00001-0002') + + def test_postal_code_taxlot(self): + + new_mappings = copy.deepcopy(self.fake_mappings['taxlot']) + + tasks.save_raw_data(self.import_file.pk) + Column.create_mappings(new_mappings, self.org, self.user, self.import_file.pk) + tasks.map_data(self.import_file.pk) + + # get mapped taxlot postal_code + ts = TaxLotState.objects.filter(address_line_1='35 Tenth Street').first() + + if ts is None: + raise TypeError("Invalid Taxlot Address!") + self.assertEqual(ts.postal_code, '00333') + + ts = TaxLotState.objects.filter(address_line_1='93030 Wellington Blvd').first() + + if ts is None: + raise TypeError("Invalid Taxlot Address!") + self.assertEqual(ts.postal_code, '00000-0000') diff --git a/seed/data_importer/tests/util.py b/seed/data_importer/tests/util.py index 36978dd288..2f09cc4d57 100644 --- a/seed/data_importer/tests/util.py +++ b/seed/data_importer/tests/util.py @@ -35,6 +35,11 @@ "to_table_name": 'TaxLotState', "to_field": 'block_number' }, + { + 'from_field': 'postal code', + 'to_table_name': 'TaxLotState', + 'to_field': 'postal_code', + } ] PROPERTIES_MAPPING = [ @@ -122,6 +127,10 @@ "from_field": 'recent sale date', "to_table_name": 'PropertyState', "to_field": 'recent_sale_date' + }, { + 'from_field': 'postal code', + 'to_table_name': 'PropertyState', + 'to_field': 'postal_code', } ] @@ -165,10 +174,6 @@ 'from_field': 'City', # raw field in import file 'to_field': 'city', 'to_table_name': 'PropertyState', - }, { - 'from_field': 'Zip', # raw field in import file - 'to_field': 'postal_code', - 'to_table_name': 'PropertyState', }, { 'from_field': 'GBA', # raw field in import file 'to_field': 'gross_floor_area', diff --git a/seed/lib/mcm/mapper.py b/seed/lib/mcm/mapper.py index 32972b9458..1b9ebcf425 100644 --- a/seed/lib/mcm/mapper.py +++ b/seed/lib/mcm/mapper.py @@ -99,6 +99,14 @@ def apply_column_value(raw_column_name, column_value, model, mapping, is_extra_d if raw_column_name in mapping: table_name, mapped_column_name, display_name, is_extra_data = mapping.get(raw_column_name) + # special postal case: + if mapped_column_name in ['postal_code', 'owner_postal_code']: + if '-' in str(column_value): + postal = str(column_value).split('-')[0].zfill(5) + ext = str(column_value).split('-')[1].zfill(4) + column_value = postal + '-' + ext + column_value = str(column_value).zfill(5) + cleaned_value = None if cleaner: # Get the list of Quantity fields from the Column object in SEED. This is non-ideal, since the diff --git a/seed/migrations/0115_rehash_postal_code.py b/seed/migrations/0115_rehash_postal_code.py new file mode 100644 index 0000000000..4ed83e5735 --- /dev/null +++ b/seed/migrations/0115_rehash_postal_code.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This rehashing file should be used in the future if needed as it has been optimized. Rehashing takes +# awhile and should be avoided if possible. +from __future__ import unicode_literals + +import re + +from django.db import migrations, transaction +from django.db.models import Q + + +def zero_fill(postal): + if postal: + if bool(re.compile(r'(\b\d{1,5}-\d{1,4}\b)').match(postal)): + return postal.split('-')[0].zfill(5) + '-' + postal.split('-')[1].zfill(4) + elif bool(re.compile(r'(\b\d{1,5}\b)').match(postal)): + return postal.zfill(5) + + +# Go through every property and tax lot and simply save it to create the hash_object +def update_postal_codes(apps, schema_editor): + PropertyState = apps.get_model('seed', 'PropertyState') + TaxLotState = apps.get_model('seed', 'TaxLotState') + + with transaction.atomic(): + print('Checking for short postal codes in property states') + objs = PropertyState.objects.filter( + Q(postal_code__iregex=r'^\d{4}-\d{3,4}$') | Q(postal_code__iregex=r'^\d{4}$') | + Q(owner_postal_code__iregex=r'^\d{4}-\d{3,4}$') | Q(owner_postal_code__iregex=r'^\d{4}$') + ) + for obj in objs: + # print( + # f'fixing zip for {obj} -- postal_code | owner_postal_code = {obj.postal_code} | {obj.owner_postal_code}') + obj.postal_code = zero_fill(obj.postal_code) + obj.owner_postal_code = zero_fill(obj.owner_postal_code) + obj.save() + + print('Checking for short postal codes in tax lot states') + objs = TaxLotState.objects.filter( + Q(postal_code__iregex=r'^\d{4}-\d{3,4}$') | Q(postal_code__iregex=r'^\d{4}$') + ) + for obj in objs: + # print( + # f'fixing zip for {obj} -- postal_code | owner_postal_code = {obj.postal_code} | {obj.owner_postal_code}') + obj.postal_code = zero_fill(obj.postal_code) + obj.save() + + +class Migration(migrations.Migration): + dependencies = [ + ('seed', '0114_auto_20191211_0958'), + ] + + operations = [ + migrations.RunPython(update_postal_codes), + ]