Merge branch 'develop' into 2114

config/settings/common.py

@@ -76,7 +76,6 @@
     'django.middleware.locale.LocaleMiddleware',
     'django.middleware.common.BrokenLinkEmailsMiddleware',
     'seed.utils.api.APIBypassCSRFMiddleware',
-    'seed.utils.nosniff.DisableMIMESniffingMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',

docker/nginx-seed.conf

@@ -1,3 +1,34 @@
+# https://gist.github.com/plentz/6737338
+# config to disallow the browser to render the page inside a frame or iframe
+# and avoid clickjacking http://en.wikipedia.org/wiki/Clickjacking
+# if you need to allow [i]frames, you can use SAMEORIGIN or even set an uri with ALLOW-FROM uri
+# https://developer.mozilla.org/en-US/docs/HTTP/X-Frame-Options
+add_header X-Frame-Options SAMEORIGIN;
+
+# when serving user-supplied content, include a X-Content-Type-Options: nosniff header along with the Content-Type: header,
+# to disable content-type sniffing on some browsers.
+# https://www.owasp.org/index.php/List_of_useful_HTTP_headers
+# currently supported in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx
+# https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/compatibility/gg622941(v=vs.85)
+# 'soon' on Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=471020
+add_header X-Content-Type-Options nosniff;
+
+# This header enables the Cross-site scripting (XSS) filter built into most recent web browsers.
+# It's usually enabled by default anyway, so the role of this header is to re-enable the filter for
+# this particular website if it was disabled by the user.
+# https://www.owasp.org/index.php/List_of_useful_HTTP_headers
+add_header X-XSS-Protection "1; mode=block";
+
+# with Content Security Policy (CSP) enabled (and a browser that supports it (http://caniuse.com/#feat=contentsecuritypolicy)),
+# you can tell the browser that it can only download content from the domains you explicitly allow
+# https://www.html5rocks.com/en/tutorials/security/content-security-policy/
+# https://www.owasp.org/index.php/Content_Security_Policy
+# https://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful
+add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; img-src 'self' data: https://stamen-tiles-a.a.ssl.fastly.net https://stamen-tiles-b.a.ssl.fastly.net https://stamen-tiles-c.a.ssl.fastly.net https://stamen-tiles-d.a.ssl.fastly.net; style-src 'self' 'unsafe-inline'; frame-src 'self'; object-src 'none'";
+
+# HSTS
+add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
+
 # the upstream component nginx needs to connect to
 upstream seed_upsteam {
     server unix:///tmp/uwsgi-seed.sock;
@@ -9,7 +40,7 @@ server {
     server_name localhost;
     charset     utf-8;
 
-    # increase the timeouts (large files can take awhile to upload)
+    # increase the timeouts (large files can take a while to upload)
     # These are probably not needed, but increasing anyway
     proxy_connect_timeout       600;
     proxy_send_timeout          600;

docs/source/docker.rst

@@ -79,7 +79,7 @@ Ubuntu server 18.04 or newer with a m5ad.xlarge (if using in Production instance
 
     # The admin user is only valid only until the database is restored
     export SEED_ADMIN_USER=user@seed-platform.org
-    export SEED_ADMIN_PASSWORD=7FeBWal38*&k3jlfa92lakj8ih4
+    export SEED_ADMIN_PASSWORD="7FeBWal38*&k3jlfa92lakj8ih4"
     export SEED_ADMIN_ORG=default
 
     # For SES

requirements/base.txt

@@ -1,5 +1,5 @@
 # Django
-django==2.2.10
+django==2.2.13
 
 # NL 12/30/2019 - DJ database url is not longer used?
 #dj-database-url==0.5.0

seed/api/v2_1/views.py

@@ -24,6 +24,7 @@
     PropertyState,
     BuildingFile,
     Cycle,
+    ColumnMappingPreset,
 )
 from seed.serializers.properties import (
     PropertyViewAsStateSerializer,
@@ -169,6 +170,23 @@ def building_sync(self, request, pk):
               required: true
               paramType: query
         """
+        preset_pk = request.GET.get('preset_id')
+        try:
+            preset_pk = int(preset_pk)
+            column_mapping_preset = ColumnMappingPreset.objects.get(
+                pk=preset_pk,
+                preset_type__in=[ColumnMappingPreset.BUILDINGSYNC_DEFAULT, ColumnMappingPreset.BUILDINGSYNC_CUSTOM])
+        except TypeError:
+            return JsonResponse({
+                'success': False,
+                'message': 'Query param `preset_id` is either missing or invalid'
+            }, status=status.HTTP_400_BAD_REQUEST)
+        except ColumnMappingPreset.DoesNotExist:
+            return JsonResponse({
+                'success': False,
+                'message': f'Cannot find a BuildingSync ColumnMappingPreset with pk={preset_pk}'
+            }, status=status.HTTP_400_BAD_REQUEST)
+
         try:
             # TODO: not checking organization? Is that right?
             # TODO: this needs to call _get_property_view and use the property pk, not the property_view pk.
@@ -178,19 +196,22 @@ def building_sync(self, request, pk):
             return JsonResponse({
                 'success': False,
                 'message': 'Cannot match a PropertyView with pk=%s' % pk
-            })
+            }, status=status.HTTP_400_BAD_REQUEST)
 
         bs = BuildingSync()
         # Check if there is an existing BuildingSync XML file to merge
         bs_file = property_view.state.building_files.order_by('created').last()
         if bs_file is not None and os.path.exists(bs_file.file.path):
             bs.import_file(bs_file.file.path)
-            xml = bs.export(property_view.state)
-            return HttpResponse(xml, content_type='application/xml')
-        else:
-            # create a new XML from the record, do not import existing XML
-            xml = bs.export(property_view.state)
+
+        try:
+            xml = bs.export_using_preset(property_view.state, column_mapping_preset.mappings)
             return HttpResponse(xml, content_type='application/xml')
+        except Exception as e:
+            return JsonResponse({
+                'success': False,
+                'message': str(e)
+            }, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     @action(detail=True, methods=['GET'])
     def hpxml(self, request, pk):

seed/api/v3/urls.py

@@ -3,13 +3,28 @@
 from django.conf.urls import url, include
 from rest_framework import routers
 
+from seed.views.v3.columns import ColumnViewSet
+from seed.views.v3.cycles import CycleViewSet
 from seed.views.v3.data_quality import DataQualityViews
 from seed.views.v3.datasets import DatasetViewSet
+from seed.views.v3.labels import LabelViewSet
+from seed.views.v3.import_files import ImportFileViewSet
+from seed.views.v3.organizations import OrganizationViewSet
+from seed.views.v3.properties import PropertyViewSet
+from seed.views.v3.taxlots import TaxlotViewSet
+from seed.views.v3.users import UserViewSet
 
 api_v3_router = routers.DefaultRouter()
-api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks')
+api_v3_router.register(r'columns', ColumnViewSet, base_name='columns')
+api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles')
 api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets')
-
+api_v3_router.register(r'labels', LabelViewSet, base_name='labels')
+api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks')
+api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files')
+api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations')
+api_v3_router.register(r'properties', PropertyViewSet, base_name='properties')
+api_v3_router.register(r'taxlots', TaxlotViewSet, base_name='taxlots')
+api_v3_router.register(r'users', UserViewSet, base_name='user')
 
 urlpatterns = [
     url(r'^', include(api_v3_router.urls)),

seed/building_sync/building_sync.py

@@ -117,12 +117,21 @@ def init_tree(self, version=BUILDINGSYNC_V2_0):
         self.element_tree = etree.parse(StringIO(xml_string))
         self.version = version
 
-    def export(self, property_state, custom_mapping=None):
+    def export_using_preset(self, property_state, column_mapping_preset=None):
         """Export BuildingSync file from an existing BuildingSync file (from import), property_state and
         a custom mapping.
 
+        expected column_mapping_preset structure
+        [
+            {from_field: <absolute xpath>, from_field_value: 'text' | @<attr> | ..., to_field: <db_column>},
+            {from_field: <absolute xpath>, from_field_value: 'text' | @<attr> | ..., to_field: <db_column>},
+            .
+            .
+            .
+        ]
+
         :param property_state: object, PropertyState to merge into BuildingSync
-        :param custom_mapping: dict, user-defined mapping (used with higher priority over the default mapping)
+        :param column_mapping_preset: list, mappings from ColumnMappingPreset
         :return: string, as XML
         """
         if not property_state:
@@ -131,39 +140,33 @@ def export(self, property_state, custom_mapping=None):
         if not self.element_tree:
             self.init_tree(version=BuildingSync.BUILDINGSYNC_V2_0)
 
-        merged_mappings = merge_mappings(self.VERSION_MAPPINGS_DICT[self.version], custom_mapping)
         schema = self.get_schema(self.version)
 
-        # iterate through the 'property' field mappings doing the following
+        # iterate through the mappings doing the following
         # - if the property_state has the field, update the xml with that value
         # - else, ignore it
-        base_path = merged_mappings['property']['xpath']
-        field_mappings = merged_mappings['property']['properties']
-        for field, mapping in field_mappings.items():
-            value = None
+        for mapping in column_mapping_preset:
+            field = mapping['to_field']
+            xml_element_xpath = mapping['from_field']
+            xml_element_value = mapping['from_field_value']
+            seed_value = None
             try:
                 property_state._meta.get_field(field)
-                value = getattr(property_state, field)
+                seed_value = getattr(property_state, field)
             except FieldDoesNotExist:
                 _log.debug("Field {} is not a db field, trying read from extra data".format(field))
-                value = property_state.extra_data.get(field, None)
+                seed_value = property_state.extra_data.get(field, None)
 
-            if value is None:
+            if seed_value is None:
                 continue
-            if isinstance(value, ureg.Quantity):
-                value = value.magnitude
-
-            if mapping['xpath'].startswith('./'):
-                mapping_path = mapping['xpath'][2:]
-            else:
-                mapping_path = mapping['xpath']
-            absolute_xpath = os.path.join(base_path, mapping_path)
+            if isinstance(seed_value, ureg.Quantity):
+                seed_value = seed_value.magnitude
 
-            update_tree(schema, self.element_tree, absolute_xpath,
-                        mapping['value'], str(value), NAMESPACES)
+            update_tree(schema, self.element_tree, xml_element_xpath,
+                        xml_element_value, str(seed_value), NAMESPACES)
 
         # Not sure why, but lxml was not pretty printing if the tree was updated
-        # a hack to fix this, we just export the tree, parse it, then export again
+        # As a hack to fix this, we just export the tree, parse it, then export again
         xml_bytes = etree.tostring(self.element_tree, pretty_print=True)
         tree = etree.parse(BytesIO(xml_bytes))
         return etree.tostring(tree, pretty_print=True).decode()

seed/building_sync/mappings.py

@@ -507,6 +507,7 @@ def update_tree(schema, tree, xpath, target, value, namespaces):
                 'type': 'value',
                 'value': 'text',
                 'formatter': to_float,
+                'units': 'ft**2',
             },
             'net_floor_area': {
                 'xpath': './auc:Buildings/auc:Building/auc:FloorAreas/auc:FloorArea[auc:FloorAreaType="Net"]/auc:FloorAreaValue',

seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml

@@ -1915,6 +1915,7 @@
                     </auc:Modeled>
                   </auc:CalculationMethod>
                   <auc:AnnualSavingsSiteEnergy>0.0</auc:AnnualSavingsSiteEnergy>
+                  <auc:AnnualSavingsSourceEnergy>0.0</auc:AnnualSavingsSourceEnergy>
                   <auc:AnnualSavingsCost>0</auc:AnnualSavingsCost>
                   <auc:AnnualSavingsByFuels>
                     <auc:AnnualSavingsByFuel>
@@ -1936,6 +1937,7 @@
                   <auc:ResourceUnits>kBtu</auc:ResourceUnits>
                   <auc:AnnualFuelUseNativeUnits>2235075.9859244428</auc:AnnualFuelUseNativeUnits>
                   <auc:AnnualFuelUseConsistentUnits>2235.076</auc:AnnualFuelUseConsistentUnits>
+                  <auc:AnnualPeakConsistentUnits>110.75706</auc:AnnualPeakConsistentUnits>
                 </auc:ResourceUse>
                 <auc:ResourceUse ID="Baseline-Resource2">
                   <auc:EnergyResource>Natural gas</auc:EnergyResource>
@@ -2166,6 +2168,9 @@
                 <auc:AllResourceTotal>
                   <auc:EndUse>All end uses</auc:EndUse>
                   <auc:SiteEnergyUse>3902.655949232491</auc:SiteEnergyUse>
+                  <auc:SiteEnergyUseIntensity>52.8533358512101</auc:SiteEnergyUseIntensity>
+                  <auc:SourceEnergyUse>2051986.1527934822</auc:SourceEnergyUse>
+                  <auc:SourceEnergyUseIntensity>132.24988549512202</auc:SourceEnergyUseIntensity>
                 </auc:AllResourceTotal>
               </auc:AllResourceTotals>
             </auc:Scenario>
@@ -2183,6 +2188,7 @@
                     </auc:Modeled>
                   </auc:CalculationMethod>
                   <auc:AnnualSavingsSiteEnergy>758.5569977291548</auc:AnnualSavingsSiteEnergy>
+                  <auc:AnnualSavingsSourceEnergy>1772.1336698497769</auc:AnnualSavingsSourceEnergy>
                   <auc:AnnualSavingsCost>15183</auc:AnnualSavingsCost>
                   <auc:AnnualSavingsByFuels>
                     <auc:AnnualSavingsByFuel>
@@ -2204,6 +2210,7 @@
                   <auc:ResourceUnits>kBtu</auc:ResourceUnits>
                   <auc:AnnualFuelUseNativeUnits>3206417.9271639367</auc:AnnualFuelUseNativeUnits>
                   <auc:AnnualFuelUseConsistentUnits>2064.179</auc:AnnualFuelUseConsistentUnits>
+                  <auc:AnnualPeakConsistentUnits>110.75706</auc:AnnualPeakConsistentUnits>
                 </auc:ResourceUse>
                 <auc:ResourceUse ID="LED_Only-Resource2">
                   <auc:EnergyResource>Natural gas</auc:EnergyResource>
@@ -2434,6 +2441,9 @@
                 <auc:AllResourceTotal>
                   <auc:EndUse>All end uses</auc:EndUse>
                   <auc:SiteEnergyUse>3144.098951503336</auc:SiteEnergyUse>
+                  <auc:SiteEnergyUseIntensity>52.8533358512101</auc:SiteEnergyUseIntensity>
+                  <auc:SourceEnergyUse>2051986.1527934822</auc:SourceEnergyUse>
+                  <auc:SourceEnergyUseIntensity>132.24988549512202</auc:SourceEnergyUseIntensity>
                 </auc:AllResourceTotal>
               </auc:AllResourceTotals>
               <auc:LinkedPremises>
@@ -2456,6 +2466,7 @@
                     </auc:Modeled>
                   </auc:CalculationMethod>
                   <auc:AnnualSavingsSiteEnergy>284.11765998512055</auc:AnnualSavingsSiteEnergy>
+                  <auc:AnnualSavingsSourceEnergy>1772.1336698497769</auc:AnnualSavingsSourceEnergy>
                   <auc:AnnualSavingsCost>5194</auc:AnnualSavingsCost>
                   <auc:AnnualSavingsByFuels>
                     <auc:AnnualSavingsByFuel>
@@ -2477,6 +2488,7 @@
                   <auc:ResourceUnits>kBtu</auc:ResourceUnits>
                   <auc:AnnualFuelUseNativeUnits>2913836.260294419</auc:AnnualFuelUseNativeUnits>
                   <auc:AnnualFuelUseConsistentUnits>2913.836</auc:AnnualFuelUseConsistentUnits>
+                  <auc:AnnualPeakConsistentUnits>110.75706</auc:AnnualPeakConsistentUnits>
                 </auc:ResourceUse>
                 <auc:ResourceUse ID="Electric_Appliance_30pct_Reduction-Resource2">
                   <auc:EnergyResource>Natural gas</auc:EnergyResource>
@@ -2707,6 +2719,9 @@
                 <auc:AllResourceTotal>
                   <auc:EndUse>All end uses</auc:EndUse>
                   <auc:SiteEnergyUse>3618.5382892473704</auc:SiteEnergyUse>
+                  <auc:SiteEnergyUseIntensity>52.8533358512101</auc:SiteEnergyUseIntensity>
+                  <auc:SourceEnergyUse>2051986.1527934822</auc:SourceEnergyUse>
+                  <auc:SourceEnergyUseIntensity>132.24988549512202</auc:SourceEnergyUseIntensity>
                 </auc:AllResourceTotal>
               </auc:AllResourceTotals>
               <auc:LinkedPremises>

seed/building_sync/tests/test_buildingsync_views.py

@@ -16,6 +16,7 @@
 from seed.models import (
     PropertyView,
     StatusLabel,
+    ColumnMappingPreset,
 )
 from seed.test_helpers.fake import (
     FakeCycleFactory, FakeColumnFactory,
@@ -50,6 +51,8 @@ def setUp(self):
             start=datetime(2010, 10, 10, tzinfo=timezone.get_current_timezone())
         )
 
+        self.default_bsync_preset = ColumnMappingPreset.objects.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT)
+
         self.client.login(**user_details)
 
     def test_get_building_sync(self):
@@ -61,7 +64,8 @@ def test_get_building_sync(self):
 
         # go to buildingsync endpoint
         params = {
-            'organization_id': self.org.pk
+            'organization_id': self.org.pk,
+            'preset_id': self.default_bsync_preset.id
         }
         url = reverse('api:v2.1:properties-building-sync', args=[pv.id])
         response = self.client.get(url, params)
@@ -90,7 +94,7 @@ def test_upload_and_get_building_sync(self):
         # now get the building sync that was just uploaded
         property_id = result['data']['property_view']['id']
         url = reverse('api:v2.1:properties-building-sync', args=[property_id])
-        response = self.client.get(url)
+        response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id})
         self.assertIn('<auc:YearOfConstruction>1967</auc:YearOfConstruction>',
                       response.content.decode("utf-8"))
 
@@ -182,6 +186,6 @@ def test_upload_and_get_building_sync_diff_ns(self):
         # now get the building sync that was just uploaded
         property_id = result['data']['property_view']['id']
         url = reverse('api:v2.1:properties-building-sync', args=[property_id])
-        response = self.client.get(url)
+        response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id})
         self.assertIn('<auc:YearOfConstruction>1889</auc:YearOfConstruction>',
                       response.content.decode('utf-8'))