Merge pull request #3211 from SEED-platform/asset-extractor

BuildingSync Asset Extractor Functionality

requirements/base.txt

@@ -55,6 +55,9 @@ lark==0.11.3
 # Parsing and managing geojson data (this is only used in managed tasks at the moment)
 geojson==2.5.0
 
+# BuildingSync Asset Extractor
+buildingsync-asset-extractor==0.1.7
+
 # pnnl/buildingid-py
 -e git+https://github.com/SEED-platform/buildingid.git@f68219df82191563cc2aca818e0b0fa1b32dd52d#egg=pnnl-buildingid
 

seed/building_sync/building_sync.py

@@ -11,6 +11,8 @@
 import re
 from io import StringIO, BytesIO
 
+from buildingsync_asset_extractor.processor import BSyncProcessor as BAE
+
 from django.core.exceptions import FieldDoesNotExist
 from quantityfield.units import ureg
 from lxml import etree
@@ -66,6 +68,10 @@ def import_file(self, source):
         """
         parser = etree.XMLParser(remove_blank_text=True)
         etree.set_default_parser(parser)
+
+        # save filename
+        self.source_filename = source
+
         # save element tree
         if isinstance(source, str):
             if not os.path.isfile(source):
@@ -162,12 +168,15 @@ def export_using_profile(self, property_state, column_mapping_profile=None):
             xml_element_xpath = mapping['from_field']
             xml_element_value = mapping['from_field_value']
             seed_value = None
-            try:
-                property_state._meta.get_field(field)
-                seed_value = getattr(property_state, field)
-            except FieldDoesNotExist:
-                _log.debug("Field {} is not a db field, trying read from extra data".format(field))
-                seed_value = property_state.extra_data.get(field, None)
+            if mapping['to_field'] != mapping['from_field']:
+                # only do this for non BAE assets
+                try:
+                    property_state._meta.get_field(field)
+                    seed_value = getattr(property_state, field)
+                except FieldDoesNotExist:
+                    _log.debug("Field {} is not a db field, trying read from extra data".format(field))
+                    seed_value = property_state.extra_data.get(field, None)
+                    continue
 
             if seed_value is None:
                 continue
@@ -380,13 +389,23 @@ def _process_struct(self, base_mapping, custom_mapping=None):
         :param custom_mapping: dict, another mapping object which is given higher priority over base_mapping
         :return: list, [dict, dict], [results, dict of errors and warnings]
         """
+
         merged_mappings = merge_mappings(base_mapping, custom_mapping)
         messages = {'warnings': [], 'errors': []}
         result = apply_mapping(self.element_tree, merged_mappings, messages, NAMESPACES)
 
         # turn result into SEED structure
         seed_result = self.restructure_mapped_result(result, messages)
 
+        # BuildingSync Asset Extractor
+        bae = BAE(self.source_filename)
+        bae.extract()
+        assets = bae.get_assets()
+
+        # add to data and column headers
+        for item in assets:
+            seed_result[item['name']] = item['value']
+
         return seed_result, messages
 
     def process(self, table_mappings=None):

seed/lib/xml_mapping/mapper.py

@@ -7,10 +7,11 @@
 
 from __future__ import absolute_import
 import logging
-
+from buildingsync_asset_extractor.processor import BSyncProcessor as BAE
 from seed.building_sync.building_sync import BuildingSync
 from seed.building_sync.mappings import merge_mappings, xpath_to_column_map, BASE_MAPPING_V2
 
+
 _log = logging.getLogger(__name__)
 
 
@@ -40,6 +41,8 @@ def default_buildingsync_profile_mappings():
         "GJ/m**2/year",
         "MJ/m**2/year",
         "kBtu/m**2/year",
+        # from BAE
+        "F"
     ]
 
     mapping = BASE_MAPPING_V2.copy()
@@ -60,4 +63,30 @@ def default_buildingsync_profile_mappings():
             'to_table_name': 'PropertyState'
         })
 
+    # BAE results
+    bsync_assets = BAE.get_default_asset_defs()
+    for item in bsync_assets:
+        from_units = item['units']
+        # only add units defined above
+        if from_units not in valid_units:
+            from_units = None
+        if item['type'] == 'sqft':
+            # these types need 2 different entries: 1 for "primary" and 1 for "secondary"
+            for i in ['Primary', 'Secondary']:
+                result.append({
+                    'from_field': i + ' ' + item['export_name'],
+                    'from_field_value': 'text',  # hard code this for now
+                    'from_units': from_units,
+                    'to_field': i + ' ' + item['export_name'],
+                    'to_table_name': 'PropertyState'
+                })
+        else:
+            result.append({
+                'from_field': item['export_name'],
+                'from_field_value': 'text',  # hard code this for now
+                'from_units': from_units,
+                'to_field': item['export_name'],
+                'to_table_name': 'PropertyState'
+            })
+
     return result

seed/lib/xml_mapping/reader.py

@@ -4,10 +4,10 @@
 :copyright (c) 2014 - 2022, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved.  # NOQA
 :author
 """
-
 from io import BytesIO
 import os
 import zipfile
+from buildingsync_asset_extractor.processor import BSyncProcessor as BAE
 
 from seed.building_sync.building_sync import BuildingSync
 from seed.building_sync.mappings import xpath_to_column_map
@@ -23,6 +23,7 @@ def __init__(self, file_):
         self._xpath_col_dict = {}
 
         filename = file_.name
+
         _, file_extension = os.path.splitext(filename)
         # grab the data from the zip or xml file
         self.data = []
@@ -54,6 +55,19 @@ def _add_property_to_data(self, bsync_file, file_name):
             self.headers = list(self._xpath_col_dict.keys())
 
         property_ = bs.process_property_xpaths(self._xpath_col_dict)
+
+        # BuildingSync Asset Extractor (BAE) - automatically extract assets from BuildingSync file
+        bae = BAE(data=bsync_file)
+        bae.extract()
+        assets = bae.get_assets()
+
+        # add to data and column headers
+        for item in assets:
+            property_[item['name']] = item['value']
+            # only append if not already there (when processing a zip of xmls)
+            if item['name'] not in self.headers:
+                self.headers.append(item['name'])
+
         # When importing zip files, we need to be able to determine which .xml file
         # a certain PropertyState came from (because of the linked BuildingFile model).
         # For this reason, we add this extra information here for later use in

seed/migrations/0163_add_bae_assets_to_buildingsync_default_profile.py

@@ -0,0 +1,39 @@
+# Generated by Django 3.2.12 on 2022-04-14 20:57
+
+from django.db import migrations
+
+from seed.lib.xml_mapping.mapper import default_buildingsync_profile_mappings
+
+
+def recreate_default_bsync_presets(apps, schema_editor):
+    """create a default BuildingSync column mapping preset for each organization"""
+    Organization = apps.get_model("orgs", "Organization")
+
+    # profile number for 'BuildingSync Default' profile is 1
+    prof_type = 1
+
+    for org in Organization.objects.all():
+        bsync_mapping_name = 'BuildingSync v2.0 Defaults'
+        # first find current BuildingSync mapping and delete
+        profiles = org.columnmappingprofile_set.filter(profile_type=prof_type)
+
+        for prof in profiles:
+            prof.delete()
+
+        # then recreate including BAE fields with updated "default_buildingsync_profile_mappings" method
+        org.columnmappingprofile_set.create(
+            name=bsync_mapping_name,
+            mappings=default_buildingsync_profile_mappings(),
+            profile_type=prof_type
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('seed', '0162_auto_20220418_2257'),
+    ]
+
+    operations = [
+        migrations.RunPython(recreate_default_bsync_presets)
+    ]

seed/models/building_file.py

@@ -161,6 +161,7 @@ def process(self, organization_id, cycle, property_view=None, promote_property_s
             parser_kwargs = {}
             # TODO: use table_mappings for BuildingSync process method
             data, messages = parser.process(*parser_args, **parser_kwargs)
+
         except ParsingError as e:
             return False, None, None, [str(e)]
 

seed/static/seed/js/controllers/data_upload_modal_controller.js

@@ -548,6 +548,7 @@ angular.module('BE.seed.controller.data_upload_modal', [])
           } else {
             // successfully passed validation, save the data
             save_raw_assessed_data(file_id, cycle_id, false);
+
           }
         };
 

seed/static/seed/js/services/uploader_service.js

@@ -46,6 +46,7 @@ angular.module('BE.seed.service.uploader', []).factory('uploader_service', [
      * @param file_id: the pk of a ImportFile object we're going to save raw.
      */
     uploader_factory.validate_use_cases = function (file_id) {
+
       var org_id = user_service.get_organization().id;
       return $http.post('/api/v3/import_files/' + file_id + '/validate_use_cases/?organization_id=' + org_id.toString())
         .then(function (response) {