Merge pull request #1889 from SEED-platform/buildingsync-xlsx-export

Export BuildingSync Data in Excel Format

locale/en_US/LC_MESSAGES/django.po

@@ -788,12 +788,18 @@ msgstr "Export"
 msgid "Export Buildings"
 msgstr "Export Buildings"
 
+msgid "Export BuildingSync"
+msgstr "Export BuildingSync"
+
 msgid "Export Name"
 msgstr "Export Name"
 
 msgid "Export Selected"
 msgstr "Export Selected"
 
+msgid "Export Spreadsheet"
+msgstr "Export Spreadsheet"
+
 msgid "Export your Properties and Tax Lots"
 msgstr "Export your Properties and Tax Lots"
 

locale/fr_CA/LC_MESSAGES/django.po

@@ -792,12 +792,18 @@ msgstr "Exportation"
 msgid "Export Buildings"
 msgstr "Exporter les bâtiments"
 
+msgid "Export BuildingSync"
+msgstr "Exporter BuildingSync"
+
 msgid "Export Name"
 msgstr "Nom d'exportation"
 
 msgid "Export Selected"
 msgstr "Exporter la sélection"
 
+msgid "Export Spreadsheet"
+msgstr "Exporter la feuille de calcul"
+
 msgid "Export your Properties and Tax Lots"
 msgstr "Exportez vos propriétés et vos lots d'impôt"
 

requirements/base.txt

@@ -48,6 +48,7 @@ unidecode==1.0.22
 usaddress==0.5.10
 xlwt==1.3.0
 xlrd==1.1.0
+xlsxwriter==1.1.8
 xmltodict==0.11.0
 requests==2.20.0
 lxml==4.2.5

seed/static/seed/js/controllers/export_inventory_modal_controller.js

@@ -21,7 +21,6 @@ angular.module('BE.seed.controller.export_inventory_modal', []).controller('expo
       var ext = '.' + export_type;
       if (!_.endsWith(filename, ext)) filename += ext;
 
-
       return $http.post('/api/v2.1/tax_lot_properties/export/', {
         ids: ids,
         filename: filename,
@@ -32,18 +31,20 @@ angular.module('BE.seed.controller.export_inventory_modal', []).controller('expo
           organization_id: user_service.get_organization().id,
           cycle_id: cycle_id,
           inventory_type: inventory_type
-        }
+        },
+        responseType: export_type === 'xlsx' ? 'arraybuffer' : undefined
       }).then(function (response) {
         var blob_type = response.headers()['content-type'];
-        var blob_data;
-
-        if (blob_type === 'application/json') {
-          blob_data = JSON.stringify(response.data, null, '    ');
+        var data;
+        if (export_type === 'xlsx') {
+          data = response.data;
+        } else if (blob_type === 'application/json') {
+          data = JSON.stringify(response.data, null, '    ');
         } else if (blob_type === 'text/csv') {
-          blob_data = response.data;
+          data = response.data;
         }
 
-        var blob = new Blob([blob_data], {type: blob_type});
+        var blob = new Blob([data], {type: blob_type});
         saveAs(blob, filename);
 
         $scope.close();

seed/static/seed/js/controllers/inventory_detail_controller.js

@@ -95,7 +95,7 @@ angular.module('BE.seed.controller.inventory_detail', [])
       $scope.item_state = inventory_payload.state;
 
       // item_parent is the property or the tax lot instead of the PropertyState / TaxLotState
-      if($scope.inventory_type === 'properties') {
+      if ($scope.inventory_type === 'properties') {
         $scope.item_parent = inventory_payload.property;
       } else {
         $scope.item_parent = inventory_payload.taxlot;
@@ -371,7 +371,7 @@ angular.module('BE.seed.controller.inventory_detail', [])
         var the_url = '/api/v2_1/properties/' + $stateParams.view_id + '/building_sync/';
         $http.get(the_url, {})
           .then(function (response) {
-            var blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;' });
+            var blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'});
             var downloadLink = angular.element('<a></a>');
             var filename = 'buildingsync_property_' + $stateParams.view_id + '.xml';
             downloadLink.attr('href', $window.URL.createObjectURL(blob));
@@ -380,6 +380,33 @@ angular.module('BE.seed.controller.inventory_detail', [])
           });
       };
 
+      $scope.export_building_sync_xlsx = function () {
+        var filename = 'buildingsync_property_' + $stateParams.view_id + '.xlsx';
+        var profileId = null;
+        if ($scope.currentProfile) {
+          profileId = $scope.currentProfile.id;
+        }
+
+        $http.post('/api/v2.1/tax_lot_properties/export/', {
+          ids: [$stateParams.view_id],
+          filename: filename,
+          profile_id: null, // TODO: reconfigure backend to handle detail settings profiles
+          export_type: 'xlsx'
+        }, {
+          params: {
+            organization_id: $scope.organization.id,
+            cycle_id: $scope.cycle.id,
+            inventory_type: $scope.inventory_type
+          },
+          responseType: 'arraybuffer'
+        }).then(function (response) {
+          var blob_type = response.headers()['content-type'];
+
+          var blob = new Blob([response.data], {type: blob_type});
+          saveAs(blob, filename);
+        });
+      };
+
       $scope.unpair_property_from_taxlot = function (property_id) {
         pairing_service.unpair_property_from_taxlot($scope.inventory.view_id, property_id);
         $state.reload();

seed/static/seed/locales/en_US.json

@@ -246,8 +246,10 @@
     "Existing Tax Lots": "Existing Tax Lots",
     "Export": "Export",
     "Export Buildings": "Export Buildings",
+    "Export BuildingSync": "Export BuildingSync",
     "Export Name": "Export Name",
     "Export Selected": "Export Selected",
+    "Export Spreadsheet": "Export Spreadsheet",
     "Export your Properties and Tax Lots": "Export your Properties and Tax Lots",
     "Failed to delete inventory": "Failed to delete inventory",
     "Field": "Field",

seed/static/seed/locales/fr_CA.json

@@ -246,8 +246,10 @@
     "Existing Tax Lots": "Lots d'impôt existants",
     "Export": "Exportation",
     "Export Buildings": "Exporter les bâtiments",
+    "Export BuildingSync": "Exporter BuildingSync",
     "Export Name": "Nom d'exportation",
     "Export Selected": "Exporter la sélection",
+    "Export Spreadsheet": "Exporter la feuille de calcul",
     "Export your Properties and Tax Lots": "Exportez vos propriétés et vos lots d'impôt",
     "Failed to delete inventory": "Échec de la suppression de l'inventaire",
     "Field": "Champ",

seed/static/seed/partials/export_inventory_modal.html

@@ -10,7 +10,11 @@ <h4 class="modal-title" id="exportModalLabel" translate>Export your Properties a
     </div>
 </div>
 <div class="modal-footer">
-    <button type="button" class="btn btn-default" ng-click="cancel()" translate>Cancel</button>
     <button type="button" class="btn btn-primary" ng-click="export_selected('csv')" ng-disabled="!export_name.length" translate>Export CSV</button>
+    <button type="button" class="btn btn-primary" ng-click="export_selected('xlsx')" ng-disabled="!export_name.length" translate>Export BuildingSync in Excel format</button>
     <button type="button" class="btn btn-primary" ng-click="export_selected('geojson')" ng-disabled="!export_name.length" translate>Export GeoJSON</button>
+
+</div>
+<div class="modal-footer">
+    <button type="button" class="btn btn-default" ng-click="cancel()" translate>Cancel</button>
 </div>

seed/static/seed/partials/inventory_detail.html

@@ -59,6 +59,10 @@ <h2>
                             <a ng-click="export_building_sync()">{$:: 'Export BuildingSync' | translate $}</a>
                         </li>
 
+                        <li ng-if="::inventory_type === 'properties'" role="menuitem">
+                            <a ng-click="export_building_sync_xlsx()">Export BuildingSync (Excel)</a>
+                        </li>
+
                         <li role="menuitem">
                             <a ng-click="open_update_labels_modal()" ng-disabled="selectedCount === 0">{$:: 'Add/Remove Labels' | translate $}</a>
                         </li>

seed/tests/test_tax_lot_property.py

@@ -110,6 +110,40 @@ def test_csv_export(self):
         # last row should be blank
         self.assertEqual(data[52], '')
 
+    def test_xlxs_export(self):
+        for i in range(50):
+            p = self.property_view_factory.get_property_view()
+            self.properties.append(p.id)
+
+        columns = []
+        for c in Column.retrieve_all(self.org.id, 'property', False):
+            columns.append(c['name'])
+
+        # call the API
+        url = reverse_lazy('api:v2.1:tax_lot_properties-export')
+        response = self.client.post(
+            url + '?{}={}&{}={}&{}={}'.format(
+                'organization_id', self.org.pk,
+                'cycle_id', self.cycle,
+                'inventory_type', 'properties'
+            ),
+            data=json.dumps({'columns': columns, 'export_type': 'xlsx'}),
+            content_type='application/x-www-form-urlencoded'
+        )
+
+        print(response.content)
+
+        # parse the content as array
+        data = response.content.decode('utf-8').split('\n')
+
+        self.assertTrue('Address Line 1' in data[0].split(','))
+        self.assertTrue('Property Labels\r' in data[0].split(','))
+
+        self.assertEqual(len(data), 53)
+
+        # last row should be blank
+        self.assertEqual(data[52], '')
+
     def test_json_export(self):
         """Test to make sure get_related returns the fields"""
         for i in range(50):

seed/views/tax_lot_properties.py

@@ -10,8 +10,10 @@
 
 import csv
 import datetime
+import io
 from collections import OrderedDict
 
+import xlsxwriter
 from django.http import JsonResponse, HttpResponse
 from quantityfield import ureg
 from rest_framework.decorators import list_route
@@ -26,6 +28,12 @@
     TaxLotView,
     ColumnListSetting,
 )
+from seed.models.property_measures import (
+    PropertyMeasure
+)
+from seed.models.scenarios import (
+    Scenario
+)
 from seed.serializers.tax_lot_properties import (
     TaxLotPropertySerializer
 )
@@ -130,7 +138,8 @@ def export(self, request):
             # always export the labels
             column_name_mappings['taxlot_labels'] = 'Tax Lot Labels'
 
-        model_views = view_klass.objects.select_related(*select_related).prefetch_related(*prefetch_related).filter(**filter_str).order_by('id')
+        model_views = view_klass.objects.select_related(*select_related).prefetch_related(
+            *prefetch_related).filter(**filter_str).order_by('id')
 
         # get the data in a dict which includes the related data
         data = TaxLotProperty.get_related(model_views, column_ids, columns_from_database)
@@ -161,6 +170,8 @@ def export(self, request):
             return self._csv_response(filename, data, column_name_mappings)
         elif export_type == "geojson":
             return self._json_response(filename, data, column_name_mappings)
+        elif export_type == "xlsx":
+            return self._spreadsheet_response(filename, data, column_name_mappings)
 
     def _csv_response(self, filename, data, column_name_mappings):
         response = HttpResponse(content_type='text/csv')
@@ -198,8 +209,131 @@ def _csv_response(self, filename, data, column_name_mappings):
 
         return response
 
+    def _spreadsheet_response(self, filename, data, column_name_mappings):
+        response = HttpResponse(
+            content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+        response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename)
+
+        scenario_keys = (
+            'id', 'name', 'description', 'annual_site_energy_savings',
+            'annual_source_energy_savings', 'annual_cost_savings', 'summer_peak_load_reduction',
+            'winter_peak_load_reduction', 'hdd', 'cdd', 'analysis_state', 'analysis_state_message'
+        )
+        property_measure_keys = (
+            'id', 'property_measure_name', 'measure_id', 'cost_mv', 'cost_total_first',
+            'cost_installation', 'cost_material', 'cost_capital_replacement', 'cost_residual_value'
+        )
+        measure_keys = ('name', 'display_name', 'category', 'category_display_name')
+        # find measures and scenarios
+        for i, record in enumerate(data):
+            measures = PropertyMeasure.objects.filter(property_state_id=record['property_state_id'])
+            record['measures'] = measures
+
+            scenarios = Scenario.objects.filter(property_state_id=record['property_state_id'])
+            record['scenarios'] = scenarios
+
+        output = io.BytesIO()
+        wb = xlsxwriter.Workbook(output)
+
+        # add tabs
+        ws1 = wb.add_worksheet('Properties')
+        ws2 = wb.add_worksheet('Measures')
+        ws3 = wb.add_worksheet('Scenarios')
+        ws4 = wb.add_worksheet('Scenario Measure Join Table')
+        bold = wb.add_format({'bold': True})
+
+        row = 0
+        row2 = 0
+        col2 = 0
+        row3 = 0
+        col3 = 0
+        row4 = 0
+
+        for index, val in enumerate(list(column_name_mappings.values())):
+            # Do not write the first element as ID, this causes weird issues with Excel.
+            if index == 0 and val == 'ID':
+                ws1.write(row, index, 'id', bold)
+            else:
+                ws1.write(row, index, val, bold)
+
+        # iterate over the results to preserve column order and write row.
+        for datum in data:
+            row += 1
+            id = None
+            for index, column in enumerate(column_name_mappings):
+                if column == 'id':
+                    id = datum.get(column, None)
+
+                row_result = datum.get(column, None)
+
+                # Try grabbing the value out of the related field if not found yet.
+                if row_result is None and datum.get('related'):
+                    row_result = datum['related'][0].get(column, None)
+
+                # Convert quantities (this is typically handled in the JSON Encoder, but that isn't here).
+                if isinstance(row_result, ureg.Quantity):
+                    row_result = row_result.magnitude
+                elif isinstance(row_result, datetime.datetime):
+                    row_result = row_result.strftime("%Y-%m-%d %H:%M:%S")
+                elif isinstance(row_result, datetime.date):
+                    row_result = row_result.strftime("%Y-%m-%d")
+                ws1.write(row, index, row_result)
+
+            # measures
+            for index, m in enumerate(datum['measures']):
+                if index == 0:
+                    # grab headers
+                    for key in property_measure_keys:
+                        ws2.write(row2, col2, key, bold)
+                        col2 += 1
+                    for key in measure_keys:
+                        ws2.write(row2, col2, 'measure ' + key, bold)
+                        col2 += 1
+
+                row2 += 1
+                col2 = 0
+                for key in property_measure_keys:
+                    ws2.write(row2, col2, getattr(m, key))
+                    col2 += 1
+                for key in measure_keys:
+                    ws2.write(row2, col2, getattr(m.measure, key))
+                    col2 += 1
+
+            # scenarios (and join table)
+            # join table
+            ws4.write('A1', 'Property ID', bold)
+            ws4.write('B1', 'Scenario ID', bold)
+            ws4.write('C1', 'Measure ID', bold)
+            for index, s in enumerate(datum['scenarios']):
+                scenario_id = s.id
+                if index == 0:
+                    # grab headers
+                    for key in scenario_keys:
+                        ws3.write(row3, col3, key, bold)
+                        col3 += 1
+                row3 += 1
+                col3 = 0
+                for key in scenario_keys:
+                    ws3.write(row3, col3, getattr(s, key))
+                    col3 += 1
+
+                for sm in s.measures.all():
+                    row4 += 1
+                    ws4.write(row4, 0, id)
+                    ws4.write(row4, 1, scenario_id)
+                    ws4.write(row4, 2, sm.id)
+
+        wb.close()
+
+        # xlsx_data contains the Excel file
+        xlsx_data = output.getvalue()
+
+        response.write(xlsx_data)
+        return response
+
     def _json_response(self, filename, data, column_name_mappings):
-        polygon_fields = ["bounding_box", "centroid", "property_footprint", "taxlot_footprint", "long_lat"]
+        polygon_fields = ["bounding_box", "centroid", "property_footprint", "taxlot_footprint",
+                          "long_lat"]
         features = []
 
         # extract related records