feat: buildingsync column mapping presets (#2213)

* feat: column mapping presets for buildingsync

* fix: address PR comments

* chore: add units for bsync columns

* chore: fix migrations

* chore: address PR comments

* chore: allow changing from_units for bsync custom presets

Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com>

seed/api/v2_1/views.py

@@ -24,6 +24,7 @@
     PropertyState,
     BuildingFile,
     Cycle,
+    ColumnMappingPreset,
 )
 from seed.serializers.properties import (
     PropertyViewAsStateSerializer,
@@ -169,6 +170,23 @@ def building_sync(self, request, pk):
               required: true
               paramType: query
         """
+        preset_pk = request.GET.get('preset_id')
+        try:
+            preset_pk = int(preset_pk)
+            column_mapping_preset = ColumnMappingPreset.objects.get(
+                pk=preset_pk,
+                preset_type__in=[ColumnMappingPreset.BUILDINGSYNC_DEFAULT, ColumnMappingPreset.BUILDINGSYNC_CUSTOM])
+        except TypeError:
+            return JsonResponse({
+                'success': False,
+                'message': 'Query param `preset_id` is either missing or invalid'
+            }, status=status.HTTP_400_BAD_REQUEST)
+        except ColumnMappingPreset.DoesNotExist:
+            return JsonResponse({
+                'success': False,
+                'message': f'Cannot find a BuildingSync ColumnMappingPreset with pk={preset_pk}'
+            }, status=status.HTTP_400_BAD_REQUEST)
+
         try:
             # TODO: not checking organization? Is that right?
             # TODO: this needs to call _get_property_view and use the property pk, not the property_view pk.
@@ -178,19 +196,22 @@ def building_sync(self, request, pk):
             return JsonResponse({
                 'success': False,
                 'message': 'Cannot match a PropertyView with pk=%s' % pk
-            })
+            }, status=status.HTTP_400_BAD_REQUEST)
 
         bs = BuildingSync()
         # Check if there is an existing BuildingSync XML file to merge
         bs_file = property_view.state.building_files.order_by('created').last()
         if bs_file is not None and os.path.exists(bs_file.file.path):
             bs.import_file(bs_file.file.path)
-            xml = bs.export(property_view.state)
-            return HttpResponse(xml, content_type='application/xml')
-        else:
-            # create a new XML from the record, do not import existing XML
-            xml = bs.export(property_view.state)
+
+        try:
+            xml = bs.export_using_preset(property_view.state, column_mapping_preset.mappings)
             return HttpResponse(xml, content_type='application/xml')
+        except Exception as e:
+            return JsonResponse({
+                'success': False,
+                'message': str(e)
+            }, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     @action(detail=True, methods=['GET'])
     def hpxml(self, request, pk):

seed/building_sync/building_sync.py

@@ -117,12 +117,21 @@ def init_tree(self, version=BUILDINGSYNC_V2_0):
         self.element_tree = etree.parse(StringIO(xml_string))
         self.version = version
 
-    def export(self, property_state, custom_mapping=None):
+    def export_using_preset(self, property_state, column_mapping_preset=None):
         """Export BuildingSync file from an existing BuildingSync file (from import), property_state and
         a custom mapping.
 
+        expected column_mapping_preset structure
+        [
+            {from_field: <absolute xpath>, from_field_value: 'text' | @<attr> | ..., to_field: <db_column>},
+            {from_field: <absolute xpath>, from_field_value: 'text' | @<attr> | ..., to_field: <db_column>},
+            .
+            .
+            .
+        ]
+
         :param property_state: object, PropertyState to merge into BuildingSync
-        :param custom_mapping: dict, user-defined mapping (used with higher priority over the default mapping)
+        :param column_mapping_preset: list, mappings from ColumnMappingPreset
         :return: string, as XML
         """
         if not property_state:
@@ -131,39 +140,33 @@ def export(self, property_state, custom_mapping=None):
         if not self.element_tree:
             self.init_tree(version=BuildingSync.BUILDINGSYNC_V2_0)
 
-        merged_mappings = merge_mappings(self.VERSION_MAPPINGS_DICT[self.version], custom_mapping)
         schema = self.get_schema(self.version)
 
-        # iterate through the 'property' field mappings doing the following
+        # iterate through the mappings doing the following
         # - if the property_state has the field, update the xml with that value
         # - else, ignore it
-        base_path = merged_mappings['property']['xpath']
-        field_mappings = merged_mappings['property']['properties']
-        for field, mapping in field_mappings.items():
-            value = None
+        for mapping in column_mapping_preset:
+            field = mapping['to_field']
+            xml_element_xpath = mapping['from_field']
+            xml_element_value = mapping['from_field_value']
+            seed_value = None
             try:
                 property_state._meta.get_field(field)
-                value = getattr(property_state, field)
+                seed_value = getattr(property_state, field)
             except FieldDoesNotExist:
                 _log.debug("Field {} is not a db field, trying read from extra data".format(field))
-                value = property_state.extra_data.get(field, None)
+                seed_value = property_state.extra_data.get(field, None)
 
-            if value is None:
+            if seed_value is None:
                 continue
-            if isinstance(value, ureg.Quantity):
-                value = value.magnitude
-
-            if mapping['xpath'].startswith('./'):
-                mapping_path = mapping['xpath'][2:]
-            else:
-                mapping_path = mapping['xpath']
-            absolute_xpath = os.path.join(base_path, mapping_path)
+            if isinstance(seed_value, ureg.Quantity):
+                seed_value = seed_value.magnitude
 
-            update_tree(schema, self.element_tree, absolute_xpath,
-                        mapping['value'], str(value), NAMESPACES)
+            update_tree(schema, self.element_tree, xml_element_xpath,
+                        xml_element_value, str(seed_value), NAMESPACES)
 
         # Not sure why, but lxml was not pretty printing if the tree was updated
-        # a hack to fix this, we just export the tree, parse it, then export again
+        # As a hack to fix this, we just export the tree, parse it, then export again
         xml_bytes = etree.tostring(self.element_tree, pretty_print=True)
         tree = etree.parse(BytesIO(xml_bytes))
         return etree.tostring(tree, pretty_print=True).decode()

seed/building_sync/mappings.py

@@ -507,6 +507,7 @@ def update_tree(schema, tree, xpath, target, value, namespaces):
                 'type': 'value',
                 'value': 'text',
                 'formatter': to_float,
+                'units': 'ft**2',
             },
             'net_floor_area': {
                 'xpath': './auc:Buildings/auc:Building/auc:FloorAreas/auc:FloorArea[auc:FloorAreaType="Net"]/auc:FloorAreaValue',

seed/building_sync/tests/test_buildingsync_views.py

@@ -16,6 +16,7 @@
 from seed.models import (
     PropertyView,
     StatusLabel,
+    ColumnMappingPreset,
 )
 from seed.test_helpers.fake import (
     FakeCycleFactory, FakeColumnFactory,
@@ -50,6 +51,8 @@ def setUp(self):
             start=datetime(2010, 10, 10, tzinfo=timezone.get_current_timezone())
         )
 
+        self.default_bsync_preset = ColumnMappingPreset.objects.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT)
+
         self.client.login(**user_details)
 
     def test_get_building_sync(self):
@@ -61,7 +64,8 @@ def test_get_building_sync(self):
 
         # go to buildingsync endpoint
         params = {
-            'organization_id': self.org.pk
+            'organization_id': self.org.pk,
+            'preset_id': self.default_bsync_preset.id
         }
         url = reverse('api:v2.1:properties-building-sync', args=[pv.id])
         response = self.client.get(url, params)
@@ -90,7 +94,7 @@ def test_upload_and_get_building_sync(self):
         # now get the building sync that was just uploaded
         property_id = result['data']['property_view']['id']
         url = reverse('api:v2.1:properties-building-sync', args=[property_id])
-        response = self.client.get(url)
+        response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id})
         self.assertIn('<auc:YearOfConstruction>1967</auc:YearOfConstruction>',
                       response.content.decode("utf-8"))
 
@@ -182,6 +186,6 @@ def test_upload_and_get_building_sync_diff_ns(self):
         # now get the building sync that was just uploaded
         property_id = result['data']['property_view']['id']
         url = reverse('api:v2.1:properties-building-sync', args=[property_id])
-        response = self.client.get(url)
+        response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id})
         self.assertIn('<auc:YearOfConstruction>1889</auc:YearOfConstruction>',
                       response.content.decode('utf-8'))

seed/data_importer/tests/integration/test_data_import.py

@@ -26,7 +26,6 @@
     FAKE_EXTRA_DATA,
     FAKE_MAPPINGS,
     FAKE_ROW,
-    mock_buildingsync_mapping
 )
 from seed.models import (
     ASSESSED_RAW,
@@ -43,6 +42,7 @@
     BuildingFile,
 )
 from seed.tests.util import DataMappingBaseTestCase
+from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings
 
 _log = logging.getLogger(__name__)
 
@@ -240,7 +240,7 @@ def test_map_data_zip(self):
         self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 2)
 
         # make the column mappings
-        self.fake_mappings = mock_buildingsync_mapping()
+        self.fake_mappings = default_buildingsync_preset_mappings()
         Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk)
 
         # -- Act
@@ -260,7 +260,7 @@ def test_map_all_models_zip(self):
         self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 2)
 
         # make the column mappings
-        self.fake_mappings = mock_buildingsync_mapping()
+        self.fake_mappings = default_buildingsync_preset_mappings()
         Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk)
 
         # map the data
@@ -327,7 +327,7 @@ def test_map_all_models_xml(self):
         self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 1)
 
         # make the column mappings
-        self.fake_mappings = mock_buildingsync_mapping()
+        self.fake_mappings = default_buildingsync_preset_mappings()
         Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk)
 
         # map the data
@@ -386,7 +386,7 @@ def test_map_all_models_xml(self):
         self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 1)
 
         # make the column mappings
-        self.fake_mappings = mock_buildingsync_mapping()
+        self.fake_mappings = default_buildingsync_preset_mappings()
         Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk)
 
         # map the data

seed/data_importer/tests/util.py

@@ -7,7 +7,6 @@
 
 import logging
 
-from seed.building_sync.mappings import BASE_MAPPING_V2_0, xpath_to_column_map
 
 logger = logging.getLogger(__name__)
 
@@ -269,18 +268,3 @@
     "to_table_name": 'PropertyState',
     "to_field": 'property_footprint',
 }
-
-
-def mock_buildingsync_mapping():
-    # returns a column mapping for bsync files
-    xpath_to_col = xpath_to_column_map(BASE_MAPPING_V2_0)
-    result = []
-    for xpath, col in xpath_to_col.items():
-        result.append(
-            {
-                'from_field': xpath,
-                'to_field': col,
-                'to_table_name': 'PropertyState'
-            }
-        )
-    return result

seed/lib/xml_mapping/mapper.py

@@ -9,7 +9,7 @@
 import logging
 
 from seed.building_sync.building_sync import BuildingSync
-from seed.building_sync.mappings import merge_mappings, xpath_to_column_map
+from seed.building_sync.mappings import merge_mappings, xpath_to_column_map, BASE_MAPPING_V2_0
 
 _log = logging.getLogger(__name__)
 
@@ -23,3 +23,41 @@ def build_column_mapping(base_mapping=None, custom_mapping=None):
         xpath: ('PropertyState', db_column, 100)
         for xpath, db_column in column_mapping.items()
     }
+
+
+def default_buildingsync_preset_mappings():
+    """Returns the default ColumnMappingPreset mappings for BuildingSync
+    :return: list
+    """
+    # taken from mapping partial (./static/seed/partials/mapping.html)
+    valid_units = [
+        # area units
+        "ft**2",
+        "m**2",
+        # eui_units
+        "kBtu/ft**2/year",
+        "kWh/m**2/year",
+        "GJ/m**2/year",
+        "MJ/m**2/year",
+        "kBtu/m**2/year",
+    ]
+
+    mapping = BASE_MAPPING_V2_0.copy()
+    base_path = mapping['property']['xpath'].rstrip('/')
+    result = []
+    for col_name, col_info in mapping['property']['properties'].items():
+        from_units = col_info.get('units')
+        if from_units not in valid_units:
+            from_units = None
+
+        sub_path = col_info['xpath'].replace('./', '')
+        absolute_xpath = f'{base_path}/{sub_path}'
+        result.append({
+            'from_field': absolute_xpath,
+            'from_field_value': col_info['value'],
+            'from_units': from_units,
+            'to_field': col_name,
+            'to_table_name': 'PropertyState'
+        })
+
+    return result

seed/migrations/0126_columnmappingpreset_preset_type.py

@@ -0,0 +1,34 @@
+# Generated by Django 2.2.10 on 2020-05-01 21:24
+
+from django.db import migrations, models
+
+from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings
+
+
+def create_default_bsync_presets(apps, schema_editor):
+    """create a default BuildingSync column mapping preset for each organization"""
+    Organization = apps.get_model("orgs", "Organization")
+
+    for org in Organization.objects.all():
+        bsync_mapping_name = 'BuildingSync v2.0 Defaults'
+        org.columnmappingpreset_set.create(
+            name=bsync_mapping_name,
+            mappings=default_buildingsync_preset_mappings(),
+            preset_type=1
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('seed', '0125_dq_refactor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='columnmappingpreset',
+            name='preset_type',
+            field=models.IntegerField(choices=[(0, 'Normal'), (1, 'BuildingSync Default'), (2, 'BuildingSync Custom')], default=0),
+        ),
+        migrations.RunPython(create_default_bsync_presets),
+    ]

seed/models/column_mapping_presets.py

@@ -8,10 +8,37 @@
 
 
 class ColumnMappingPreset(models.Model):
+    NORMAL = 0
+    BUILDINGSYNC_DEFAULT = 1
+    BUILDINGSYNC_CUSTOM = 2
+
+    COLUMN_MAPPING_PRESET_TYPES = (
+        (NORMAL, 'Normal'),
+        (BUILDINGSYNC_DEFAULT, 'BuildingSync Default'),
+        (BUILDINGSYNC_CUSTOM, 'BuildingSync Custom')
+    )
+
     name = models.CharField(max_length=255, blank=False)
     mappings = JSONField(default=list, blank=True)
 
     organizations = models.ManyToManyField(Organization)
 
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
+
+    preset_type = models.IntegerField(choices=COLUMN_MAPPING_PRESET_TYPES, default=NORMAL)
+
+    @classmethod
+    def get_preset_type(cls, preset_type):
+        """Returns the integer value for a preset type. Raises exception when
+        preset_type is invalid.
+
+        :param preset_type: int | str
+        :return: str
+        """
+        if isinstance(preset_type, int):
+            return preset_type
+        types_dict = dict((v, k) for k, v in cls.COLUMN_MAPPING_PRESET_TYPES)
+        if preset_type in types_dict:
+            return types_dict[preset_type]
+        raise Exception(f'Invalid preset type "{preset_type}"')