diff --git a/config/settings/common.py b/config/settings/common.py index 4588a4455f..fcbfe20137 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -76,7 +76,6 @@ 'django.middleware.locale.LocaleMiddleware', 'django.middleware.common.BrokenLinkEmailsMiddleware', 'seed.utils.api.APIBypassCSRFMiddleware', - 'seed.utils.nosniff.DisableMIMESniffingMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', diff --git a/docker/nginx-seed.conf b/docker/nginx-seed.conf index 2e5af7db90..b409dc30b6 100644 --- a/docker/nginx-seed.conf +++ b/docker/nginx-seed.conf @@ -1,3 +1,34 @@ +# https://gist.github.com/plentz/6737338 +# config to disallow the browser to render the page inside a frame or iframe +# and avoid clickjacking http://en.wikipedia.org/wiki/Clickjacking +# if you need to allow [i]frames, you can use SAMEORIGIN or even set an uri with ALLOW-FROM uri +# https://developer.mozilla.org/en-US/docs/HTTP/X-Frame-Options +add_header X-Frame-Options SAMEORIGIN; + +# when serving user-supplied content, include a X-Content-Type-Options: nosniff header along with the Content-Type: header, +# to disable content-type sniffing on some browsers. +# https://www.owasp.org/index.php/List_of_useful_HTTP_headers +# currently supported in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx +# https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/compatibility/gg622941(v=vs.85) +# 'soon' on Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=471020 +add_header X-Content-Type-Options nosniff; + +# This header enables the Cross-site scripting (XSS) filter built into most recent web browsers. +# It's usually enabled by default anyway, so the role of this header is to re-enable the filter for +# this particular website if it was disabled by the user. +# https://www.owasp.org/index.php/List_of_useful_HTTP_headers +add_header X-XSS-Protection "1; mode=block"; + +# with Content Security Policy (CSP) enabled (and a browser that supports it (http://caniuse.com/#feat=contentsecuritypolicy)), +# you can tell the browser that it can only download content from the domains you explicitly allow +# https://www.html5rocks.com/en/tutorials/security/content-security-policy/ +# https://www.owasp.org/index.php/Content_Security_Policy +# https://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful +add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; img-src 'self' data: https://stamen-tiles-a.a.ssl.fastly.net https://stamen-tiles-b.a.ssl.fastly.net https://stamen-tiles-c.a.ssl.fastly.net https://stamen-tiles-d.a.ssl.fastly.net; style-src 'self' 'unsafe-inline'; frame-src 'self'; object-src 'none'"; + +# HSTS +add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; + # the upstream component nginx needs to connect to upstream seed_upsteam { server unix:///tmp/uwsgi-seed.sock; @@ -9,7 +40,7 @@ server { server_name localhost; charset utf-8; - # increase the timeouts (large files can take awhile to upload) + # increase the timeouts (large files can take a while to upload) # These are probably not needed, but increasing anyway proxy_connect_timeout 600; proxy_send_timeout 600; diff --git a/docs/source/docker.rst b/docs/source/docker.rst index e2aa676334..697955f697 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -79,7 +79,7 @@ Ubuntu server 18.04 or newer with a m5ad.xlarge (if using in Production instance # The admin user is only valid only until the database is restored export SEED_ADMIN_USER=user@seed-platform.org - export SEED_ADMIN_PASSWORD=7FeBWal38*&k3jlfa92lakj8ih4 + export SEED_ADMIN_PASSWORD="7FeBWal38*&k3jlfa92lakj8ih4" export SEED_ADMIN_ORG=default # For SES diff --git a/requirements/base.txt b/requirements/base.txt index be1837d9d2..318532423a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # Django -django==2.2.10 +django==2.2.13 # NL 12/30/2019 - DJ database url is not longer used? #dj-database-url==0.5.0 diff --git a/seed/api/v2_1/views.py b/seed/api/v2_1/views.py index e20f6266ee..694741da29 100644 --- a/seed/api/v2_1/views.py +++ b/seed/api/v2_1/views.py @@ -24,6 +24,7 @@ PropertyState, BuildingFile, Cycle, + ColumnMappingPreset, ) from seed.serializers.properties import ( PropertyViewAsStateSerializer, @@ -169,6 +170,23 @@ def building_sync(self, request, pk): required: true paramType: query """ + preset_pk = request.GET.get('preset_id') + try: + preset_pk = int(preset_pk) + column_mapping_preset = ColumnMappingPreset.objects.get( + pk=preset_pk, + preset_type__in=[ColumnMappingPreset.BUILDINGSYNC_DEFAULT, ColumnMappingPreset.BUILDINGSYNC_CUSTOM]) + except TypeError: + return JsonResponse({ + 'success': False, + 'message': 'Query param `preset_id` is either missing or invalid' + }, status=status.HTTP_400_BAD_REQUEST) + except ColumnMappingPreset.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': f'Cannot find a BuildingSync ColumnMappingPreset with pk={preset_pk}' + }, status=status.HTTP_400_BAD_REQUEST) + try: # TODO: not checking organization? Is that right? # TODO: this needs to call _get_property_view and use the property pk, not the property_view pk. @@ -178,19 +196,22 @@ def building_sync(self, request, pk): return JsonResponse({ 'success': False, 'message': 'Cannot match a PropertyView with pk=%s' % pk - }) + }, status=status.HTTP_400_BAD_REQUEST) bs = BuildingSync() # Check if there is an existing BuildingSync XML file to merge bs_file = property_view.state.building_files.order_by('created').last() if bs_file is not None and os.path.exists(bs_file.file.path): bs.import_file(bs_file.file.path) - xml = bs.export(property_view.state) - return HttpResponse(xml, content_type='application/xml') - else: - # create a new XML from the record, do not import existing XML - xml = bs.export(property_view.state) + + try: + xml = bs.export_using_preset(property_view.state, column_mapping_preset.mappings) return HttpResponse(xml, content_type='application/xml') + except Exception as e: + return JsonResponse({ + 'success': False, + 'message': str(e) + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @action(detail=True, methods=['GET']) def hpxml(self, request, pk): diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 1882f228c8..81020182fa 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -3,13 +3,28 @@ from django.conf.urls import url, include from rest_framework import routers +from seed.views.v3.columns import ColumnViewSet +from seed.views.v3.cycles import CycleViewSet from seed.views.v3.data_quality import DataQualityViews from seed.views.v3.datasets import DatasetViewSet +from seed.views.v3.labels import LabelViewSet +from seed.views.v3.import_files import ImportFileViewSet +from seed.views.v3.organizations import OrganizationViewSet +from seed.views.v3.properties import PropertyViewSet +from seed.views.v3.taxlots import TaxlotViewSet +from seed.views.v3.users import UserViewSet api_v3_router = routers.DefaultRouter() -api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') +api_v3_router.register(r'columns', ColumnViewSet, base_name='columns') +api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') - +api_v3_router.register(r'labels', LabelViewSet, base_name='labels') +api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') +api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files') +api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations') +api_v3_router.register(r'properties', PropertyViewSet, base_name='properties') +api_v3_router.register(r'taxlots', TaxlotViewSet, base_name='taxlots') +api_v3_router.register(r'users', UserViewSet, base_name='user') urlpatterns = [ url(r'^', include(api_v3_router.urls)), diff --git a/seed/building_sync/building_sync.py b/seed/building_sync/building_sync.py index f5cbeb9826..5f5417fdfe 100644 --- a/seed/building_sync/building_sync.py +++ b/seed/building_sync/building_sync.py @@ -117,12 +117,21 @@ def init_tree(self, version=BUILDINGSYNC_V2_0): self.element_tree = etree.parse(StringIO(xml_string)) self.version = version - def export(self, property_state, custom_mapping=None): + def export_using_preset(self, property_state, column_mapping_preset=None): """Export BuildingSync file from an existing BuildingSync file (from import), property_state and a custom mapping. + expected column_mapping_preset structure + [ + {from_field: , from_field_value: 'text' | @ | ..., to_field: }, + {from_field: , from_field_value: 'text' | @ | ..., to_field: }, + . + . + . + ] + :param property_state: object, PropertyState to merge into BuildingSync - :param custom_mapping: dict, user-defined mapping (used with higher priority over the default mapping) + :param column_mapping_preset: list, mappings from ColumnMappingPreset :return: string, as XML """ if not property_state: @@ -131,39 +140,33 @@ def export(self, property_state, custom_mapping=None): if not self.element_tree: self.init_tree(version=BuildingSync.BUILDINGSYNC_V2_0) - merged_mappings = merge_mappings(self.VERSION_MAPPINGS_DICT[self.version], custom_mapping) schema = self.get_schema(self.version) - # iterate through the 'property' field mappings doing the following + # iterate through the mappings doing the following # - if the property_state has the field, update the xml with that value # - else, ignore it - base_path = merged_mappings['property']['xpath'] - field_mappings = merged_mappings['property']['properties'] - for field, mapping in field_mappings.items(): - value = None + for mapping in column_mapping_preset: + field = mapping['to_field'] + xml_element_xpath = mapping['from_field'] + xml_element_value = mapping['from_field_value'] + seed_value = None try: property_state._meta.get_field(field) - value = getattr(property_state, field) + seed_value = getattr(property_state, field) except FieldDoesNotExist: _log.debug("Field {} is not a db field, trying read from extra data".format(field)) - value = property_state.extra_data.get(field, None) + seed_value = property_state.extra_data.get(field, None) - if value is None: + if seed_value is None: continue - if isinstance(value, ureg.Quantity): - value = value.magnitude - - if mapping['xpath'].startswith('./'): - mapping_path = mapping['xpath'][2:] - else: - mapping_path = mapping['xpath'] - absolute_xpath = os.path.join(base_path, mapping_path) + if isinstance(seed_value, ureg.Quantity): + seed_value = seed_value.magnitude - update_tree(schema, self.element_tree, absolute_xpath, - mapping['value'], str(value), NAMESPACES) + update_tree(schema, self.element_tree, xml_element_xpath, + xml_element_value, str(seed_value), NAMESPACES) # Not sure why, but lxml was not pretty printing if the tree was updated - # a hack to fix this, we just export the tree, parse it, then export again + # As a hack to fix this, we just export the tree, parse it, then export again xml_bytes = etree.tostring(self.element_tree, pretty_print=True) tree = etree.parse(BytesIO(xml_bytes)) return etree.tostring(tree, pretty_print=True).decode() diff --git a/seed/building_sync/mappings.py b/seed/building_sync/mappings.py index 0557349a87..23d2268785 100644 --- a/seed/building_sync/mappings.py +++ b/seed/building_sync/mappings.py @@ -507,6 +507,7 @@ def update_tree(schema, tree, xpath, target, value, namespaces): 'type': 'value', 'value': 'text', 'formatter': to_float, + 'units': 'ft**2', }, 'net_floor_area': { 'xpath': './auc:Buildings/auc:Building/auc:FloorAreas/auc:FloorArea[auc:FloorAreaType="Net"]/auc:FloorAreaValue', diff --git a/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml b/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml index bf07ba1827..6364088d14 100644 --- a/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml +++ b/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml @@ -1915,6 +1915,7 @@ 0.0 + 0.0 0 @@ -1936,6 +1937,7 @@ kBtu 2235075.9859244428 2235.076 + 110.75706 Natural gas @@ -2166,6 +2168,9 @@ All end uses 3902.655949232491 + 52.8533358512101 + 2051986.1527934822 + 132.24988549512202 @@ -2183,6 +2188,7 @@ 758.5569977291548 + 1772.1336698497769 15183 @@ -2204,6 +2210,7 @@ kBtu 3206417.9271639367 2064.179 + 110.75706 Natural gas @@ -2434,6 +2441,9 @@ All end uses 3144.098951503336 + 52.8533358512101 + 2051986.1527934822 + 132.24988549512202 @@ -2456,6 +2466,7 @@ 284.11765998512055 + 1772.1336698497769 5194 @@ -2477,6 +2488,7 @@ kBtu 2913836.260294419 2913.836 + 110.75706 Natural gas @@ -2707,6 +2719,9 @@ All end uses 3618.5382892473704 + 52.8533358512101 + 2051986.1527934822 + 132.24988549512202 diff --git a/seed/building_sync/tests/test_buildingsync_views.py b/seed/building_sync/tests/test_buildingsync_views.py index e98444d767..9d9ee80aa7 100644 --- a/seed/building_sync/tests/test_buildingsync_views.py +++ b/seed/building_sync/tests/test_buildingsync_views.py @@ -16,6 +16,7 @@ from seed.models import ( PropertyView, StatusLabel, + ColumnMappingPreset, ) from seed.test_helpers.fake import ( FakeCycleFactory, FakeColumnFactory, @@ -50,6 +51,8 @@ def setUp(self): start=datetime(2010, 10, 10, tzinfo=timezone.get_current_timezone()) ) + self.default_bsync_preset = ColumnMappingPreset.objects.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + self.client.login(**user_details) def test_get_building_sync(self): @@ -61,7 +64,8 @@ def test_get_building_sync(self): # go to buildingsync endpoint params = { - 'organization_id': self.org.pk + 'organization_id': self.org.pk, + 'preset_id': self.default_bsync_preset.id } url = reverse('api:v2.1:properties-building-sync', args=[pv.id]) response = self.client.get(url, params) @@ -90,7 +94,7 @@ def test_upload_and_get_building_sync(self): # now get the building sync that was just uploaded property_id = result['data']['property_view']['id'] url = reverse('api:v2.1:properties-building-sync', args=[property_id]) - response = self.client.get(url) + response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id}) self.assertIn('1967', response.content.decode("utf-8")) @@ -182,6 +186,6 @@ def test_upload_and_get_building_sync_diff_ns(self): # now get the building sync that was just uploaded property_id = result['data']['property_view']['id'] url = reverse('api:v2.1:properties-building-sync', args=[property_id]) - response = self.client.get(url) + response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id}) self.assertIn('1889', response.content.decode('utf-8')) diff --git a/seed/building_sync/tests/test_validation_client.py b/seed/building_sync/tests/test_validation_client.py new file mode 100644 index 0000000000..17118f8b18 --- /dev/null +++ b/seed/building_sync/tests/test_validation_client.py @@ -0,0 +1,226 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2019, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import os +from unittest.mock import patch + +from django.test import TestCase +from requests.models import Response +import json + +from config.settings.common import BASE_DIR +from seed.building_sync.validation_client import validate_use_case, DEFAULT_USE_CASE + + +def responseFactory(status_code, body_dict): + the_response = Response() + the_response.status_code = status_code + the_response._content = json.dumps(body_dict).encode() + return the_response + + +class TestValidationClient(TestCase): + def setUp(self): + # NOTE: the contents of these files are not actually used, it's just convenient + # to use these files so we don't have to create tmp ones and clean them up + self.single_file = open(os.path.join(BASE_DIR, 'seed', 'building_sync', 'tests', 'data', 'buildingsync_v2_0_bricr_workflow.xml')) + self.zip_file = open(os.path.join(BASE_DIR, 'seed', 'building_sync', 'tests', 'data', 'ex_1_and_buildingsync_ex01_measures.zip')) + + def test_validation_single_file_ok(self): + good_body = { + 'success': True, + 'validation_results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, good_body)): + all_files_valid, file_summaries = validate_use_case(self.single_file) + + self.assertTrue(all_files_valid) + self.assertEqual([], file_summaries) + + def test_validation_zip_file_ok(self): + good_body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + }, { + 'file': 'file2.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + } + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, good_body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertTrue(all_files_valid) + self.assertEqual([], file_summaries) + + def test_validation_fails_when_one_file_has_bad_schema(self): + bad_file_result = { + 'file': 'bad.xml', + 'results': { + 'schema': { + # Set the schema as NOT valid + 'valid': False, + 'errors': ['schema was bad'] + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + } + + body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + }, + bad_file_result, + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertFalse(all_files_valid) + bad_file_names = [f['file'] for f in file_summaries] + self.assertEqual([bad_file_result['file']], bad_file_names) + + def test_validation_fails_when_one_file_fails_use_case(self): + bad_file_result = { + 'file': 'bad.xml', + 'results': { + 'schema': { + 'valid': True, + }, + 'use_cases': { + DEFAULT_USE_CASE: { + # Include a use case error + 'errors': ['something was wrong'], + 'warnings': [], + } + } + } + } + + body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + }, + bad_file_result, + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertFalse(all_files_valid) + bad_file_names = [f['file'] for f in file_summaries] + self.assertEqual([bad_file_result['file']], bad_file_names) + + def test_validation_zip_file_ok_when_warnings(self): + good_body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + # Include a warning + 'warnings': ['This is a warning!'], + } + } + } + }, { + 'file': 'file2.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + # Include a warning + 'warnings': ['This is another warning!'], + } + } + } + } + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, good_body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertTrue(all_files_valid) + file_names = [f['file'] for f in file_summaries] + self.assertEqual(['file1.xml', 'file2.xml'], file_names) diff --git a/seed/building_sync/validation_client.py b/seed/building_sync/validation_client.py new file mode 100644 index 0000000000..23ddf91cc5 --- /dev/null +++ b/seed/building_sync/validation_client.py @@ -0,0 +1,134 @@ +import os + +import requests + + +VALIDATION_API_URL = "https://selectiontool.buildingsync.net/api/validate" +DEFAULT_SCHEMA_VERSION = '2.0.0' +DEFAULT_USE_CASE = 'SEED' + + +class ValidationClientException(Exception): + pass + + +def _validation_api_post(file_, schema_version, use_case_name): + payload = {'schema_version': schema_version} + files = [ + ('file', file_) + ] + + return requests.request( + "POST", + VALIDATION_API_URL, + data=payload, + files=files, + timeout=60 * 2, # timeout after two minutes (it can take a long time for zips) + ) + + +def validate_use_case(file_, filename=None, schema_version=DEFAULT_SCHEMA_VERSION, use_case_name=DEFAULT_USE_CASE): + """calls Selection Tool's validation API + + :param file_: File, the file to validate; can be single xml or zip + :param filename: string, (optional) name of the file, useful if file_.name is not user friendly (e.g. a Django SimpleUploadedFile). Not used if file_ is a zip + :param schema_version: string + :param use_case_name: string + :return: tuple, (bool, list), bool indicates if the file passes validation, + the list is a collection of files and their errors, warnings, and infos + """ + + try: + response = _validation_api_post(file_, schema_version, use_case_name) + except requests.exceptions.Timeout: + raise ValidationClientException( + "Request to Selection Tool timed out. SEED may need to increase the timeout", + ) + except Exception as e: + raise ValidationClientException( + f"Failed to make request to selection tool: {e}", + ) + + if response.status_code != 200: + raise ValidationClientException( + f"Received bad response from Selection Tool: {response.text}", + ) + + try: + response_body = response.json() + except ValueError: + raise ValidationClientException( + f"Expected JSON response from Selection Tool: {response.text}", + ) + + if response_body.get('success', False) is not True: + ValidationClientException( + f"Selection Tool request was not successful: {response.text}", + ) + + response_schema_version = response_body.get('schema_version') + if response_schema_version != schema_version: + ValidationClientException( + f"Expected schema_version to be '{schema_version}' but it was '{response_schema_version}'", + ) + + _, file_extension = os.path.splitext(file_.name) + validation_results = response_body.get('validation_results') + # check the returned type and make validation_results a list if it's not already + if file_extension == '.zip': + if type(validation_results) is not list: + raise ValidationClientException( + f"Expected response validation_results to be list for zip file: {response.text}", + ) + else: + if type(validation_results) is not dict: + raise ValidationClientException( + f"Expected response validation_results to be dict for single xml file: {response.text}", + ) + # turn the single file result into the same structure as zip file result + if filename is None: + filename = os.path.basename(file_.name) + validation_results = [{ + 'file': filename, + 'results': validation_results, + }] + + # check that the schema and use case is valid for every file + file_summaries = [] + all_files_valid = True + for validation_result in validation_results: + results = validation_result['results'] + filename = validation_result['file'] + + # it's possible there's no use_cases key - occurs when schema validation fails + if results['schema']['valid'] is False: + use_case_result = {} + else: + if use_case_name not in results.get('use_cases', []): + raise ValidationClientException( + f'Expected use case "{use_case_name}" to exist in result\'s uses cases.', + ) + use_case_result = results['use_cases'][use_case_name] + + file_summary = { + 'file': filename, + 'schema_errors': results['schema'].get('errors', []), + 'use_case_errors': use_case_result.get('errors', []), + 'use_case_warnings': use_case_result.get('warnings', []), + } + + file_has_errors = ( + len(file_summary['schema_errors']) > 0 + or len(file_summary['use_case_errors']) > 0 + ) + if file_has_errors: + all_files_valid = False + + file_has_errors_or_warnings = ( + file_has_errors + or len(file_summary['use_case_warnings']) > 0 + ) + if file_has_errors_or_warnings: + file_summaries.append(file_summary) + + return all_files_valid, file_summaries diff --git a/seed/data_importer/tasks.py b/seed/data_importer/tasks.py index bfe849b8d0..1b7d116835 100644 --- a/seed/data_importer/tasks.py +++ b/seed/data_importer/tasks.py @@ -33,6 +33,7 @@ from past.builtins import basestring from unidecode import unidecode +from seed.building_sync import validation_client from seed.data_importer.equivalence_partitioner import EquivalencePartitioner from seed.data_importer.match import ( match_and_link_incoming_properties_and_taxlots, @@ -1527,3 +1528,53 @@ def pair_new_states(merged_property_views, merged_taxlot_views): m2m_join.save() return + + +@shared_task +def _validate_use_cases(file_pk, progress_key): + import_file = ImportFile.objects.get(pk=file_pk) + progress_data = ProgressData.from_key(progress_key) + + progress_data.step('validating data with Selection Tool') + try: + all_files_valid, file_summaries = validation_client.validate_use_case( + import_file.file, + filename=import_file.uploaded_filename + ) + if all_files_valid is False: + import_file.delete() + progress_data.finish_with_success( + message=json.dumps({ + 'valid': all_files_valid, + 'issues': file_summaries, + }), + ) + except validation_client.ValidationClientException as e: + _log.debug(f'ValidationClientException while validating import_file `{file_pk}`: {e}') + progress_data.finish_with_error(message=str(e)) + progress_data.save() + import_file.delete() + except Exception as e: + _log.debug(f'Unexpected Exception while validating import_file `{file_pk}`: {e}') + progress_data.finish_with_error(message=str(e)) + progress_data.save() + import_file.delete() + + +def validate_use_cases(file_pk): + """ + Kicks off task for validating BuildingSync files for use cases + + :param file_pk: ImportFile Primary Key + :return: + """ + progress_data = ProgressData(func_name='validate_use_cases', unique_id=file_pk) + # break progress into two steps: + # 1. started job + # 2. finished request + progress_data.total = 2 + progress_data.save() + + _validate_use_cases.s(file_pk, progress_data.key).apply_async() + _log.debug(progress_data.result()) + return progress_data.result() diff --git a/seed/data_importer/tests/integration/test_data_import.py b/seed/data_importer/tests/integration/test_data_import.py index 29d336dd13..5b24f266b2 100644 --- a/seed/data_importer/tests/integration/test_data_import.py +++ b/seed/data_importer/tests/integration/test_data_import.py @@ -26,7 +26,6 @@ FAKE_EXTRA_DATA, FAKE_MAPPINGS, FAKE_ROW, - mock_buildingsync_mapping ) from seed.models import ( ASSESSED_RAW, @@ -43,6 +42,7 @@ BuildingFile, ) from seed.tests.util import DataMappingBaseTestCase +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings _log = logging.getLogger(__name__) @@ -240,7 +240,7 @@ def test_map_data_zip(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 2) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # -- Act @@ -260,7 +260,7 @@ def test_map_all_models_zip(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 2) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # map the data @@ -327,7 +327,7 @@ def test_map_all_models_xml(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 1) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # map the data @@ -386,7 +386,7 @@ def test_map_all_models_xml(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 1) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # map the data diff --git a/seed/data_importer/tests/test_mapping.py b/seed/data_importer/tests/test_mapping.py index 070818401c..3f4f1f7754 100644 --- a/seed/data_importer/tests/test_mapping.py +++ b/seed/data_importer/tests/test_mapping.py @@ -94,6 +94,105 @@ def test_mapping(self): # for p in props: # pp(p) + def test_remapping_with_and_without_unit_aware_columns_doesnt_lose_data(self): + """ + During import, when the initial -State objects are created from the extra_data values, + ColumnMapping objects are used to take the extra_data dictionary values and create the + -State objects, setting the DB-level values as necessary - e.g. taking a raw + "Site EUI (kBtu/ft2)" value and inserting it into the DB field "site_eui". + + Previously, remapping could cause extra Column objects to be created, and subsequently, + this created extra ColumnMapping objects. These extra ColumnMapping objects could cause + raw values to be inserted into the wrong DB field on -State creation. + """ + # Just as in the previous test, build extra_data PropertyState + state = self.property_state_factory.get_property_state_as_extra_data( + import_file_id=self.import_file.id, + source_type=ASSESSED_RAW, + data_state=DATA_STATE_IMPORT, + random_extra=42, + ) + + # Replace the site_eui key-value that gets autogenerated by get_property_state_as_extra_data + del state.extra_data['site_eui'] + state.extra_data['Site EUI (kBtu/ft2)'] = 123 + state.save() + + self.import_file.raw_save_done = True + self.import_file.save() + + # Build 2 sets of mappings - with and without a unit-aware destination site_eui data + suggested_mappings = mapper.build_column_mapping( + list(state.extra_data.keys()), + Column.retrieve_all_by_tuple(self.org), + previous_mapping=get_column_mapping, + map_args=[self.org], + thresh=80 + ) + + ed_site_eui_mappings = [] + unit_aware_site_eui_mappings = [] + for raw_column, suggestion in suggested_mappings.items(): + if raw_column == 'Site EUI (kBtu/ft2)': + # Make this an extra_data field (without from_units) + ed_site_eui_mappings.append({ + "from_field": raw_column, + "from_units": None, + "to_table_name": 'PropertyState', + "to_field": raw_column, + "to_field_display_name": raw_column, + }) + + unit_aware_site_eui_mappings.append({ + "from_field": raw_column, + "from_units": 'kBtu/ft**2/year', + "to_table_name": 'PropertyState', + "to_field": 'site_eui', + "to_field_display_name": 'Site EUI', + }) + else: + other_mapping = { + "from_field": raw_column, + "from_units": None, + "to_table_name": suggestion[0], + "to_field": suggestion[1], + "to_field_display_name": suggestion[1], + } + ed_site_eui_mappings.append(other_mapping) + unit_aware_site_eui_mappings.append(other_mapping) + + # Map and remap the file multiple times with different mappings each time. + # Round 1 - Map site_eui data into Extra Data + Column.create_mappings(ed_site_eui_mappings, self.org, self.user, self.import_file.id) + tasks.map_data(self.import_file.id) + + # There should only be one raw 'Site EUI (kBtu/ft2)' Column object + self.assertEqual(1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) + # The one propertystate should have site eui info in extra_data + prop = self.import_file.find_unmatched_property_states().get() + self.assertIsNone(prop.site_eui) + self.assertIsNotNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) + + # Round 2 - Map site_eui data into the PropertyState attribute "site_eui" + Column.create_mappings(unit_aware_site_eui_mappings, self.org, self.user, self.import_file.id) + tasks.map_data(self.import_file.id, remap=True) + + self.assertEqual(1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) + # The one propertystate should have site eui info in site_eui + prop = self.import_file.find_unmatched_property_states().get() + self.assertIsNotNone(prop.site_eui) + self.assertIsNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) + + # Round 3 - Map site_eui data into Extra Data + Column.create_mappings(ed_site_eui_mappings, self.org, self.user, self.import_file.id) + tasks.map_data(self.import_file.id, remap=True) + + self.assertEqual(1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) + # The one propertystate should have site eui info in extra_data + prop = self.import_file.find_unmatched_property_states().get() + self.assertIsNone(prop.site_eui) + self.assertIsNotNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) + class TestDuplicateFileHeaders(DataMappingBaseTestCase): def setUp(self): diff --git a/seed/data_importer/tests/util.py b/seed/data_importer/tests/util.py index e741260550..25ec0c4735 100644 --- a/seed/data_importer/tests/util.py +++ b/seed/data_importer/tests/util.py @@ -7,7 +7,6 @@ import logging -from seed.building_sync.mappings import BASE_MAPPING_V2_0, xpath_to_column_map logger = logging.getLogger(__name__) @@ -269,18 +268,3 @@ "to_table_name": 'PropertyState', "to_field": 'property_footprint', } - - -def mock_buildingsync_mapping(): - # returns a column mapping for bsync files - xpath_to_col = xpath_to_column_map(BASE_MAPPING_V2_0) - result = [] - for xpath, col in xpath_to_col.items(): - result.append( - { - 'from_field': xpath, - 'to_field': col, - 'to_table_name': 'PropertyState' - } - ) - return result diff --git a/seed/data_importer/views.py b/seed/data_importer/views.py index bba10c134f..075da67fe6 100644 --- a/seed/data_importer/views.py +++ b/seed/data_importer/views.py @@ -34,6 +34,7 @@ map_additional_models as task_map_additional_models, match_buildings as task_match_buildings, save_raw_data as task_save_raw, + validate_use_cases as task_validate_use_cases, ) from seed.decorators import ajax_request, ajax_request_class from seed.decorators import get_prog_key @@ -901,6 +902,43 @@ def data_quality_progress(self, request, pk=None): cache = get_cache(prog_key) return HttpResponse(cache['progress']) + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def validate_use_cases(self, request, pk=None): + """ + Starts a background task to call BuildingSync's use case validation + tool. + --- + type: + status: + required: true + type: string + description: either success or error + message: + required: false + type: string + description: error message, if any + progress_key: + type: integer + description: ID of background job, for retrieving job progress + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must include pk of import_file to validate' + }, status=status.HTTP_400_BAD_REQUEST) + + return task_validate_use_cases(import_file_id) + @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') diff --git a/seed/landing/forms.py b/seed/landing/forms.py index 4a07f54de6..3fbde8cc24 100644 --- a/seed/landing/forms.py +++ b/seed/landing/forms.py @@ -19,7 +19,7 @@ class LoginForm(forms.Form): password = forms.CharField( label=_("Password"), widget=forms.PasswordInput( - attrs={'class': 'field', 'placeholder': _('Password')} + attrs={'class': 'field', 'placeholder': _('Password'), 'autocomplete': 'off'} ), required=True ) diff --git a/seed/landing/models.py b/seed/landing/models.py index be5d2b9dc2..7cfa94c298 100644 --- a/seed/landing/models.py +++ b/seed/landing/models.py @@ -124,6 +124,10 @@ def process_header_request(cls, request): def get_absolute_url(self): return "/users/%s/" % urlquote(self.username) + def deactivate_user(self): + self.is_active = False + self.save() + def get_full_name(self): """ Returns the first_name plus the last_name, with a space in between. diff --git a/seed/lib/xml_mapping/mapper.py b/seed/lib/xml_mapping/mapper.py index 32cfc94203..51733df26b 100644 --- a/seed/lib/xml_mapping/mapper.py +++ b/seed/lib/xml_mapping/mapper.py @@ -9,7 +9,7 @@ import logging from seed.building_sync.building_sync import BuildingSync -from seed.building_sync.mappings import merge_mappings, xpath_to_column_map +from seed.building_sync.mappings import merge_mappings, xpath_to_column_map, BASE_MAPPING_V2_0 _log = logging.getLogger(__name__) @@ -23,3 +23,41 @@ def build_column_mapping(base_mapping=None, custom_mapping=None): xpath: ('PropertyState', db_column, 100) for xpath, db_column in column_mapping.items() } + + +def default_buildingsync_preset_mappings(): + """Returns the default ColumnMappingPreset mappings for BuildingSync + :return: list + """ + # taken from mapping partial (./static/seed/partials/mapping.html) + valid_units = [ + # area units + "ft**2", + "m**2", + # eui_units + "kBtu/ft**2/year", + "kWh/m**2/year", + "GJ/m**2/year", + "MJ/m**2/year", + "kBtu/m**2/year", + ] + + mapping = BASE_MAPPING_V2_0.copy() + base_path = mapping['property']['xpath'].rstrip('/') + result = [] + for col_name, col_info in mapping['property']['properties'].items(): + from_units = col_info.get('units') + if from_units not in valid_units: + from_units = None + + sub_path = col_info['xpath'].replace('./', '') + absolute_xpath = f'{base_path}/{sub_path}' + result.append({ + 'from_field': absolute_xpath, + 'from_field_value': col_info['value'], + 'from_units': from_units, + 'to_field': col_name, + 'to_table_name': 'PropertyState' + }) + + return result diff --git a/seed/migrations/0126_columnmappingpreset_preset_type.py b/seed/migrations/0126_columnmappingpreset_preset_type.py new file mode 100644 index 0000000000..471196d9b7 --- /dev/null +++ b/seed/migrations/0126_columnmappingpreset_preset_type.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.10 on 2020-05-01 21:24 + +from django.db import migrations, models + +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings + + +def create_default_bsync_presets(apps, schema_editor): + """create a default BuildingSync column mapping preset for each organization""" + Organization = apps.get_model("orgs", "Organization") + + for org in Organization.objects.all(): + bsync_mapping_name = 'BuildingSync v2.0 Defaults' + org.columnmappingpreset_set.create( + name=bsync_mapping_name, + mappings=default_buildingsync_preset_mappings(), + preset_type=1 + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('seed', '0125_dq_refactor'), + ] + + operations = [ + migrations.AddField( + model_name='columnmappingpreset', + name='preset_type', + field=models.IntegerField(choices=[(0, 'Normal'), (1, 'BuildingSync Default'), (2, 'BuildingSync Custom')], default=0), + ), + migrations.RunPython(create_default_bsync_presets), + ] diff --git a/seed/models/column_mapping_presets.py b/seed/models/column_mapping_presets.py index 6badf1dec5..a65b04d249 100644 --- a/seed/models/column_mapping_presets.py +++ b/seed/models/column_mapping_presets.py @@ -8,6 +8,16 @@ class ColumnMappingPreset(models.Model): + NORMAL = 0 + BUILDINGSYNC_DEFAULT = 1 + BUILDINGSYNC_CUSTOM = 2 + + COLUMN_MAPPING_PRESET_TYPES = ( + (NORMAL, 'Normal'), + (BUILDINGSYNC_DEFAULT, 'BuildingSync Default'), + (BUILDINGSYNC_CUSTOM, 'BuildingSync Custom') + ) + name = models.CharField(max_length=255, blank=False) mappings = JSONField(default=list, blank=True) @@ -15,3 +25,20 @@ class ColumnMappingPreset(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + + preset_type = models.IntegerField(choices=COLUMN_MAPPING_PRESET_TYPES, default=NORMAL) + + @classmethod + def get_preset_type(cls, preset_type): + """Returns the integer value for a preset type. Raises exception when + preset_type is invalid. + + :param preset_type: int | str + :return: str + """ + if isinstance(preset_type, int): + return preset_type + types_dict = dict((v, k) for k, v in cls.COLUMN_MAPPING_PRESET_TYPES) + if preset_type in types_dict: + return types_dict[preset_type] + raise Exception(f'Invalid preset type "{preset_type}"') diff --git a/seed/models/columns.py b/seed/models/columns.py index aa41fe9be1..611c60c004 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -982,22 +982,32 @@ def select_col_obj(column_name, table_name, organization_column): organization=organization, table_name__in=[None, ''], column_name=field['from_field'], - units_pint=field.get('from_units'), # might be None - is_extra_data=False # data from header rows in the files are NEVER extra data + is_extra_data=False # Column objects representing raw/header rows are NEVER extra data ) except Column.MultipleObjectsReturned: + # We want to avoid the ambiguity of having multiple Column objects for a specific raw column. + # To do that, delete all multiples along with any associated ColumnMapping objects. _log.debug( "More than one from_column found for {}.{}".format(field['to_table_name'], field['to_field'])) - # TODO: write something to remove the duplicate columns - from_org_col = Column.objects.filter(organization=organization, - table_name__in=[None, ''], - column_name=field['from_field'], - units_pint=field.get('from_units'), - # might be None - is_extra_data=is_extra_data).first() - _log.debug("Grabbing the first from_column") + all_from_cols = Column.objects.filter( + organization=organization, + table_name__in=[None, ''], + column_name=field['from_field'], + is_extra_data=False + ) + + ColumnMapping.objects.filter(column_raw__id__in=models.Subquery(all_from_cols.values('id'))).delete() + all_from_cols.delete() + + from_org_col = Column.objects.create( + organization=organization, + table_name__in=[None, ''], + column_name=field['from_field'], + is_extra_data=False # Column objects representing raw/header rows are NEVER extra data + ) + _log.debug("Creating a new from_column") new_field['to_column_object'] = select_col_obj(field['to_field'], field['to_table_name'], to_org_col) diff --git a/seed/models/data_quality.py b/seed/models/data_quality.py index ebbcc4a0e7..d22652df65 100644 --- a/seed/models/data_quality.py +++ b/seed/models/data_quality.py @@ -161,6 +161,7 @@ class Rule(models.Model): }, { 'table_name': 'TaxLotState', 'field': 'address_line_1', + 'data_type': TYPE_STRING, 'not_null': True, 'rule_type': RULE_TYPE_DEFAULT, 'severity': SEVERITY_ERROR, diff --git a/seed/serializers/column_mapping_presets.py b/seed/serializers/column_mapping_presets.py index 9d14aaa584..b9b466ab93 100644 --- a/seed/serializers/column_mapping_presets.py +++ b/seed/serializers/column_mapping_presets.py @@ -3,10 +3,30 @@ from rest_framework import serializers +from seed.serializers.base import ChoiceField from seed.models import ColumnMappingPreset class ColumnMappingPresetSerializer(serializers.ModelSerializer): + preset_type = ChoiceField(choices=ColumnMappingPreset.COLUMN_MAPPING_PRESET_TYPES, default=ColumnMappingPreset.NORMAL) + class Meta: model = ColumnMappingPreset fields = '__all__' + + def validate_mappings(self, mappings): + """if the preset is for BuildingSync, make sure it has valid mappings""" + preset_types_dict = dict(ColumnMappingPreset.COLUMN_MAPPING_PRESET_TYPES) + bsync_presets = [ + preset_types_dict[ColumnMappingPreset.BUILDINGSYNC_DEFAULT], + preset_types_dict[ColumnMappingPreset.BUILDINGSYNC_CUSTOM] + ] + preset_type = self.initial_data.get('preset_type') + if preset_type is None or preset_type not in bsync_presets: + return mappings + + for mapping in mappings: + if mapping.get('from_field_value') is None: + raise serializers.ValidationError(f'All BuildingSync mappings must include "from_field_value": for mapping {mapping["from_field"]}') + + return mappings diff --git a/seed/serializers/column_mappings.py b/seed/serializers/column_mappings.py index 0dfb5c9104..cbee425c22 100644 --- a/seed/serializers/column_mappings.py +++ b/seed/serializers/column_mappings.py @@ -33,3 +33,38 @@ def to_representation(self, obj): result['column_mapped'] = ColumnSerializer(obj.column_mapped.first()).data return result + + +class ImportMappingSerializer(serializers.Serializer): + from_field = serializers.CharField() + from_units = serializers.CharField() + to_field = serializers.CharField() + to_field_display_name = serializers.CharField() + to_table_name = serializers.CharField() + + +class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): + """ + Note that this is _not_ a model serializer, but used only for saving mappings + + Example: + { + "mappings": [ + { + 'from_field': 'eui', # raw field in import file + 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional + 'to_field': 'energy_use_intensity', + 'to_field_display_name': 'Energy Use Intensity', + 'to_table_name': 'PropertyState', + }, + { + 'from_field': 'gfa', + 'from_units': 'ft**2', # pint-parsable units, optional + 'to_field': 'gross_floor_area', + 'to_field_display_name': 'Gross Floor Area', + 'to_table_name': 'PropertyState', + } + ] + } + """ + mappings = serializers.ListField(child=ImportMappingSerializer()) diff --git a/seed/static/seed/js/constants.js b/seed/static/seed/js/constants.js new file mode 100644 index 0000000000..de19976095 --- /dev/null +++ b/seed/static/seed/js/constants.js @@ -0,0 +1,8 @@ +/** + * :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. + * :author + */ +angular.module('BE.seed.constants', []) + .constant('COLUMN_MAPPING_PRESET_TYPE_NORMAL', 'Normal') + .constant('COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', 'BuildingSync Default') + .constant('COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', 'BuildingSync Custom'); diff --git a/seed/static/seed/js/controllers/admin_controller.js b/seed/static/seed/js/controllers/admin_controller.js index 0c1d1e7bc2..d70e1ed84a 100644 --- a/seed/static/seed/js/controllers/admin_controller.js +++ b/seed/static/seed/js/controllers/admin_controller.js @@ -188,7 +188,7 @@ angular.module('BE.seed.controller.admin', []) } }; - function confirm_remove_user(user, org_id) { + function confirm_remove_user (user, org_id) { organization_service.remove_user(user.user_id, org_id).then(function () { $scope.get_organizations_users($scope.org_user.organization); get_users(); @@ -239,12 +239,12 @@ angular.module('BE.seed.controller.admin', []) .then(function (data) { // resolve promise uploader_service.check_progress_loop(data.progress_key, 0, 1, function () { - org.remove_message = 'success'; - get_organizations(); - }, function () { - // Do nothing - }, - org); + org.remove_message = 'success'; + get_organizations(); + }, function () { + // Do nothing + }, + org); }); }; diff --git a/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js b/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js index 36dfbff8cb..97c8fd1733 100644 --- a/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js +++ b/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js @@ -18,7 +18,7 @@ angular.module('BE.seed.controller.column_mapping_preset_modal', []) $scope.rename_preset = function () { if (!$scope.disabled()) { var preset_id = $scope.data.id; - var updated_data = {name: $scope.newName} + var updated_data = {name: $scope.newName}; column_mappings_service.update_column_mapping_preset($scope.org_id, preset_id, updated_data).then(function (result) { $uibModalInstance.close(result.data.name); }); @@ -36,6 +36,7 @@ angular.module('BE.seed.controller.column_mapping_preset_modal', []) column_mappings_service.new_column_mapping_preset_for_org($scope.org_id, { name: $scope.newName, mappings: $scope.data.mappings, + preset_type: $scope.data.preset_type }).then(function (result) { $uibModalInstance.close(result.data); }); diff --git a/seed/static/seed/js/controllers/column_mappings_controller.js b/seed/static/seed/js/controllers/column_mappings_controller.js index ff9aaac5fa..e289e74e1a 100644 --- a/seed/static/seed/js/controllers/column_mappings_controller.js +++ b/seed/static/seed/js/controllers/column_mappings_controller.js @@ -6,6 +6,7 @@ angular.module('BE.seed.controller.column_mappings', []) .controller('column_mappings_controller', [ '$scope', '$state', + '$log', '$uibModal', 'Notification', 'auth_payload', @@ -16,9 +17,13 @@ angular.module('BE.seed.controller.column_mappings', []) 'mappable_taxlot_columns_payload', 'organization_payload', 'urls', + 'COLUMN_MAPPING_PRESET_TYPE_NORMAL', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', function ( $scope, $state, + $log, $uibModal, Notification, auth_payload, @@ -28,7 +33,10 @@ angular.module('BE.seed.controller.column_mappings', []) mappable_property_columns_payload, mappable_taxlot_columns_payload, organization_payload, - urls + urls, + COLUMN_MAPPING_PRESET_TYPE_NORMAL, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM ) { $scope.org = organization_payload.organization; $scope.auth = auth_payload.auth; @@ -39,12 +47,12 @@ angular.module('BE.seed.controller.column_mappings', []) $scope.mappable_taxlot_columns = mappable_taxlot_columns_payload; // Helpers to convert to and from DB column names and column display names - var mapping_db_to_display = function(mapping) { + var mapping_db_to_display = function (mapping) { var mappable_column; - if (mapping.to_table_name === "PropertyState") { + if (mapping.to_table_name === 'PropertyState') { mappable_column = _.find($scope.mappable_property_columns, {column_name: mapping.to_field}); - } else if (mapping.to_table_name === "TaxLotState") { + } else if (mapping.to_table_name === 'TaxLotState') { mappable_column = _.find($scope.mappable_taxlot_columns, {column_name: mapping.to_field}); } @@ -53,16 +61,16 @@ angular.module('BE.seed.controller.column_mappings', []) } }; - var mapping_display_to_db = function(mapping) { + var mapping_display_to_db = function (mapping) { // Also, clear from_units if mapping is not for units col if (!$scope.is_eui_column(mapping) && !$scope.is_area_column(mapping)) { mapping.from_units = null; } var mappable_column; - if (mapping.to_table_name === "PropertyState") { + if (mapping.to_table_name === 'PropertyState') { mappable_column = _.find($scope.mappable_property_columns, {displayName: mapping.to_field}); - } else if (mapping.to_table_name === "TaxLotState") { + } else if (mapping.to_table_name === 'TaxLotState') { mappable_column = _.find($scope.mappable_taxlot_columns, {displayName: mapping.to_field}); } @@ -72,7 +80,7 @@ angular.module('BE.seed.controller.column_mappings', []) }; // On page load, convert DB field names to display names - _.forEach(column_mapping_presets_payload, function(preset) { + _.forEach(column_mapping_presets_payload, function (preset) { _.forEach(preset.mappings, mapping_db_to_display); }); @@ -101,7 +109,7 @@ angular.module('BE.seed.controller.column_mappings', []) }; // On load... - analyze_chosen_inventory_types() + analyze_chosen_inventory_types(); $scope.updateSingleInventoryTypeDropdown = function () { analyze_chosen_inventory_types(); @@ -120,13 +128,19 @@ angular.module('BE.seed.controller.column_mappings', []) // Preset-level CRUD modal-rending actions $scope.new_preset = function () { + var presetData = JSON.parse(JSON.stringify($scope.current_preset)); + // change the preset type to custom if we've edited a default preset + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT) { + presetData.preset_type = COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM; + } + var modalInstance = $uibModal.open({ templateUrl: urls.static_url + 'seed/partials/column_mapping_preset_modal.html', controller: 'column_mapping_preset_modal_controller', resolve: { action: _.constant('new'), - data: _.constant($scope.current_preset), - org_id: _.constant($scope.org.id), + data: _.constant(presetData), + org_id: _.constant($scope.org.id) } }); @@ -148,7 +162,7 @@ angular.module('BE.seed.controller.column_mappings', []) resolve: { action: _.constant('rename'), data: _.constant($scope.current_preset), - org_id: _.constant($scope.org.id), + org_id: _.constant($scope.org.id) } }); @@ -169,7 +183,7 @@ angular.module('BE.seed.controller.column_mappings', []) resolve: { action: _.constant('remove'), data: _.constant($scope.current_preset), - org_id: _.constant($scope.org.id), + org_id: _.constant($scope.org.id) } }); @@ -236,7 +250,7 @@ angular.module('BE.seed.controller.column_mappings', []) // Add and remove column methods $scope.add_new_column = function () { - var empty_row = {from_field: "", from_units: null, to_field: "", to_table_name: ""}; + var empty_row = {from_field: '', from_units: null, to_field: '', to_table_name: ''}; if ($scope.current_preset.mappings[0]) { $scope.current_preset.mappings.push(empty_row); @@ -257,9 +271,9 @@ angular.module('BE.seed.controller.column_mappings', []) }; // Copy Comma-delimited list into headers - $scope.csv_headers = ""; + $scope.csv_headers = ''; - $scope.copy_csv_headers = function() { + $scope.copy_csv_headers = function () { $uibModal.open({ template: '
' + '

Replacing Existing Columns

' + @@ -311,10 +325,36 @@ angular.module('BE.seed.controller.column_mappings', []) }; $scope.header_duplicates_present = function () { - var grouped_by_from_field = _.groupBy($scope.current_preset.mappings, 'from_field') + var grouped_by_from_field = _.groupBy($scope.current_preset.mappings, 'from_field'); return Boolean(_.find( - _.values(grouped_by_from_field), function (group) {return group.length > 1}) + _.values(grouped_by_from_field), function (group) { + return group.length > 1; + }) ); }; + + $scope.preset_action_ok = function (action) { + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_NORMAL) { + return true; + } + + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT) { + return false; + } + + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM) { + var allowed_actions = [ + 'update', + 'rename', + 'delete', + 'change_to_field', + 'change_from_units' + ]; + return allowed_actions.includes(action); + } + + $log.warn('Unknown preset type "' + $scope.current_preset.preset_type + '"'); + return false; + }; }]); diff --git a/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js b/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js index b390be6ba7..e1b8fcff0f 100644 --- a/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js +++ b/seed/static/seed/js/controllers/confirm_column_settings_modal_controller.js @@ -33,8 +33,8 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) $scope.all_columns = all_columns; $scope.step = { - number: 1, - } + number: 1 + }; $scope.goto_step = function (step) { $scope.step.number = step; @@ -45,7 +45,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) }); // parse proposed changes to create change summary to be presented to user - var all_changed_settings = ["column_name"]; // add column_name to describe each row + var all_changed_settings = ['column_name']; // add column_name to describe each row $scope.change_summary_data = _.reduce(proposed_changes, function (summary, value, key) { var column = _.find($scope.all_columns, {id: parseInt(key)}); var change = _.pick(_.cloneDeep(column), ['column_name']); @@ -64,7 +64,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) }, { field: 'display_name', - displayName: 'Display Name Change', + displayName: 'Display Name Change' }, { field: 'geocoding_order', @@ -76,34 +76,34 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) '' + '' + '' + - '
', + '' }, { field: 'data_type', - displayName: 'Data Type Change', + displayName: 'Data Type Change' }, { field: 'merge_protection', displayName: 'Merge Protection Change', cellTemplate: '
' + '' + - '
', + '' }, { field: 'recognize_empty', displayName: 'Recognize Empty', cellTemplate: '
' + '' + - '
', + '' }, { field: 'is_matching_criteria', displayName: 'Matching Criteria Change', cellTemplate: '
' + '' + - '
', - }, - ] + '' + } + ]; var unique_summary_columns = _.uniq(all_changed_settings); $scope.change_summary_column_defs = _.filter(base_summary_column_defs, function (column_def) { @@ -115,25 +115,25 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) columnDefs: $scope.change_summary_column_defs, enableColumnResizing: true, rowHeight: 40, - minRowsToShow: Math.min($scope.change_summary_data.length, 5), + minRowsToShow: Math.min($scope.change_summary_data.length, 5) }; // By default, assume matching criteria isn't being updated to exclude PM Property ID // And since warning wouldn't be shown in that case, set "acknowledged" to true. $scope.checks = { matching_criteria_excludes_pm_property_id: false, - warnings_acknowledged: true, - } + warnings_acknowledged: true + }; // Check if PM Property ID is actually being removed from matching criteria - if (_.find($scope.change_summary_data, {column_name: "pm_property_id", is_matching_criteria: false})) { + if (_.find($scope.change_summary_data, {column_name: 'pm_property_id', is_matching_criteria: false})) { $scope.checks.matching_criteria_excludes_pm_property_id = true; $scope.checks.warnings_acknowledged = false; } // Preview // Agg function returning last value of matching criteria field (all should be the same if they match) - $scope.matching_field_value = function(aggregation, fieldValue) { + $scope.matching_field_value = function (aggregation, fieldValue) { aggregation.value = fieldValue; }; @@ -143,7 +143,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) // Lastly, non-matching columns are given next priority so that users can sort within a grouped set. if (sortColumns.length > 1) { var matching_cols = _.filter(sortColumns, function (col) { - return col.colDef.is_matching_criteria; + return col.colDef.is_matching_criteria; }); var linking_id_col = _.find(sortColumns, ['name', 'id']); var remaining_cols = _.filter(sortColumns, function (col) { @@ -152,20 +152,20 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) sortColumns = matching_cols.concat(linking_id_col).concat(remaining_cols); _.forEach(sortColumns, function (col, index) { col.sort.priority = index; - }) + }); } }; // Takes raw cycle-partitioned records and returns array of cycle-aware records - var format_preview_records = function(raw_inventory) { - return _.reduce(raw_inventory, function(all_records, records, cycle_id) { + var format_preview_records = function (raw_inventory) { + return _.reduce(raw_inventory, function (all_records, records, cycle_id) { var cycle = _.find($scope.cycles, { id: parseInt(cycle_id) }); - _.forEach(records, function(record) { + _.forEach(records, function (record) { record.cycle_name = cycle.name; record.cycle_start = cycle.start; - all_records.push(record) - }) - return all_records + all_records.push(record); + }); + return all_records; }, []); }; @@ -179,7 +179,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) headerCellFilter: 'translate', minWidth: default_min_width, width: 125, - groupingShowAggregationMenu: false, + groupingShowAggregationMenu: false }; _.map(preview_column_defs, function (col) { @@ -197,7 +197,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) // Help indicate matching columns are given preferred sort priority col.displayName = col.displayName + '*'; - options.headerCellClass = "matching-column-header"; + options.headerCellClass = 'matching-column-header'; options.customTreeAggregationFn = $scope.matching_field_value; options.width = autopin_width; @@ -216,23 +216,23 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) visible: false, suppressRemoveSort: true, // since grouping relies on sorting minWidth: default_min_width, - width: autopin_width, + width: autopin_width }, { - name: "cycle_name", - displayName: "Cycle", + name: 'cycle_name', + displayName: 'Cycle', pinnedLeft: true, treeAggregationType: uiGridGroupingConstants.aggregation.COUNT, customTreeAggregationFinalizerFn: function (aggregation) { - aggregation.rendered = "total cycles: " + aggregation.value; + aggregation.rendered = 'total cycles: ' + aggregation.value; }, minWidth: default_min_width, width: autopin_width, - groupingShowAggregationMenu: false, + groupingShowAggregationMenu: false }, { - name: "cycle_start", - displayName: "Cycle Start", + name: 'cycle_start', + displayName: 'Cycle Start', cellFilter: 'date:\'yyyy-MM-dd\'', filter: inventory_service.dateFilter(), type: 'date', @@ -240,11 +240,11 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) pinnedLeft: true, minWidth: default_min_width, width: autopin_width, - groupingShowAggregationMenu: false, - }, - ) + groupingShowAggregationMenu: false + } + ); - return preview_column_defs; + return preview_column_defs; }; // Initialize preview table as empty for now. @@ -260,31 +260,31 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) $scope.gridApi.core.on.filterChanged($scope, function () { // This is a workaround for losing the state of expanded rows during filtering. - _.delay($scope.gridApi.treeBase.expandAllRows, 500); - }) + _.delay($scope.gridApi.treeBase.expandAllRows, 500); + }); // Prioritized to maintain grouping. $scope.gridApi.core.on.sortChanged($scope, prioritize_sort); - }, + } }; // Preview Loading Helpers var build_proposed_matching_columns = function (result) { // Summarize proposed matching_criteria_columns for pinning and to create preview var criteria_additions = _.filter($scope.change_summary_data, function (change) { - return change.is_matching_criteria + return change.is_matching_criteria; }); var criteria_removals = _.filter($scope.change_summary_data, function (change) { - return change.is_matching_criteria === false; + return change.is_matching_criteria === false; }); $scope.criteria_changes = { add: _.map(criteria_additions, 'column_name'), - remove: _.map(criteria_removals, 'column_name'), - } + remove: _.map(criteria_removals, 'column_name') + }; var base_and_add; - if ($scope.inventory_type == "properties") { + if ($scope.inventory_type == 'properties') { base_and_add = _.union(result.PropertyState, $scope.criteria_changes.add); } else { base_and_add = _.union(result.TaxLotState, $scope.criteria_changes.add); @@ -307,24 +307,24 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) // Use new proposed matching_criteria_columns to request a preview then render this preview. var spinner_options = { scale: 0.40, - position: "relative", - left: "100%", - } + position: 'relative', + left: '100%' + }; spinner_utility.show(spinner_options, $('#spinner_placeholder')[0]); organization_service.match_merge_link_preview($scope.org_id, $scope.inventory_type, $scope.criteria_changes) .then(function (response) { organization_service.check_match_merge_link_status(response.progress_key) - .then(function (completion_notice) { - organization_service.get_match_merge_link_result($scope.org_id, completion_notice.unique_id) - .then(build_preview) - .then(preview_loading_complete); - }); + .then(function (completion_notice) { + organization_service.get_match_merge_link_result($scope.org_id, completion_notice.unique_id) + .then(build_preview) + .then(preview_loading_complete); + }); }); }; // Get and Show Preview (If matching criteria changes exist.) - $scope.matching_criteria_exists = _.find(_.values($scope.change_summary_data), function(delta) { + $scope.matching_criteria_exists = _.find(_.values($scope.change_summary_data), function (delta) { return _.has(delta, 'is_matching_criteria'); }); diff --git a/seed/static/seed/js/controllers/data_quality_admin_controller.js b/seed/static/seed/js/controllers/data_quality_admin_controller.js index 63f873aa49..47ad29f0d7 100644 --- a/seed/static/seed/js/controllers/data_quality_admin_controller.js +++ b/seed/static/seed/js/controllers/data_quality_admin_controller.js @@ -141,10 +141,10 @@ angular.module('BE.seed.controller.data_quality_admin', []) $scope.ruleGroups = ruleGroups; $scope.rule_count_property = 0; $scope.rule_count_taxlot = 0; - _.map($scope.ruleGroups['properties'], function (rule) { + _.map($scope.ruleGroups.properties, function (rule) { $scope.rule_count_property += rule.length; }); - _.map($scope.ruleGroups['taxlots'], function (rule) { + _.map($scope.ruleGroups.taxlots, function (rule) { $scope.rule_count_taxlot += rule.length; }); }; @@ -422,8 +422,7 @@ angular.module('BE.seed.controller.data_quality_admin', []) $scope.delete_rule = function (rule, index) { if ($scope.ruleGroups[$scope.inventory_type][rule.field].length === 1) { delete $scope.ruleGroups[$scope.inventory_type][rule.field]; - } - else $scope.ruleGroups[$scope.inventory_type][rule.field].splice(index, 1); + } else $scope.ruleGroups[$scope.inventory_type][rule.field].splice(index, 1); $scope.change_rules(); if ($scope.inventory_type === 'properties') $scope.rule_count_property -= 1; else $scope.rule_count_taxlot -= 1; diff --git a/seed/static/seed/js/controllers/data_upload_modal_controller.js b/seed/static/seed/js/controllers/data_upload_modal_controller.js index aa8bd5440d..1b50afb83b 100644 --- a/seed/static/seed/js/controllers/data_upload_modal_controller.js +++ b/seed/static/seed/js/controllers/data_upload_modal_controller.js @@ -21,7 +21,6 @@ * ng-switch-when="11" == Confirm Save Mappings? * ng-switch-when="12" == Error Processing Data * ng-switch-when="13" == Portfolio Manager Import - * ng-switch-when="14" == Successful upload! [BuildingSync] */ angular.module('BE.seed.controller.data_upload_modal', []) .controller('data_upload_modal_controller', [ @@ -318,34 +317,20 @@ angular.module('BE.seed.controller.data_upload_modal', []) case 'upload_error': $scope.step_12_error_message = file.error; $scope.step.number = 12; - // add variables to identify buildingsync bulk uploads - $scope.building_sync_files = (file.source_type === 'BuildingSync'); - $scope.bulk_upload = (_.last(file.filename.split('.')) === 'zip'); break; case 'upload_in_progress': $scope.uploader.in_progress = true; - if (file.source_type === 'BuildingSync') { - $scope.uploader.progress = 100 * progress.loaded / progress.total; - } else { - $scope.uploader.progress = 25 * progress.loaded / progress.total; - } + $scope.uploader.progress = 25 * progress.loaded / progress.total; break; case 'upload_complete': var current_step = $scope.step.number; $scope.uploader.status_message = 'upload complete'; $scope.dataset.filename = file.filename; - $scope.step_14_message = null; $scope.source_type = file.source_type; - if (file.source_type === 'BuildingSync') { - $scope.uploader.complete = true; - $scope.uploader.in_progress = false; - $scope.uploader.progress = 100; - $scope.step.number = 14; - $scope.step_14_message = (_.size(file.message.warnings) > 0) ? file.message.warnings : null; - } else if (file.source_type === 'PM Meter Usage') { + if (file.source_type === 'PM Meter Usage') { $scope.cycle_id = file.cycle_id; $scope.file_id = file.file_id; @@ -361,7 +346,12 @@ angular.module('BE.seed.controller.data_upload_modal', []) // Assessed Data; upload is step 2; PM import is currently treated as such, and is step 13 if (current_step === 2 || current_step === 13) { - save_raw_assessed_data(file.file_id, file.cycle_id, false); + // if importing BuildingSync, validate then save, otherwise just save + if (file.source_type === "BuildingSync Raw") { + validate_use_cases_then_save(file.file_id, file.cycle_id) + } else { + save_raw_assessed_data(file.file_id, file.cycle_id, false) + } } // Portfolio Data if (current_step === 4) { @@ -496,6 +486,58 @@ angular.module('BE.seed.controller.data_upload_modal', []) }; }; + /** + * validate_use_cases_then_save: validates BuildingSync files for use cases + * before saving the data + * + * @param {string} file_id: the id of the import file + * @param cycle_id + */ + var validate_use_cases_then_save = function (file_id, cycle_id) { + $scope.uploader.status_message = 'validating data'; + $scope.uploader.progress = 0; + + const successHandler = (progress_data) => { + $scope.uploader.complete = false + $scope.uploader.in_progress = true + $scope.uploader.status_message = 'validation complete; starting to save data'; + $scope.uploader.progress = 100; + + const result = JSON.parse(progress_data.message) + $scope.buildingsync_valid = result.valid + $scope.buildingsync_issues = result.issues + + // if validation failed, end the import flow here; otherwise continue + if ($scope.buildingsync_valid !== true) { + $scope.step_12_error_message = 'Failed to validate uploaded BuildingSync file(s)' + $scope.step_12_buildingsync_validation_error = true + $scope.step.number = 12 + } else { + // successfully passed validation, save the data + save_raw_assessed_data(file_id, cycle_id, false) + } + } + + const errorHandler = (data) => { + $log.error(data.message); + if (data.hasOwnProperty('stacktrace')) $log.error(data.stacktrace); + $scope.step_12_error_message = data.data ? data.data.message : data.message; + $scope.step.number = 12; + } + + uploader_service.validate_use_cases(file_id) + .then(data => { + const progress = _.clamp(data.progress, 0, 100); + uploader_service.check_progress_loop( + data.progress_key, + progress, 1 - (progress / 100), + successHandler, + errorHandler, + $scope.uploader, + ) + }) + }; + /** * save_raw_assessed_data: saves Assessed data * @@ -578,7 +620,7 @@ angular.module('BE.seed.controller.data_upload_modal', []) // this only occurs in buildingsync, where we are not actually merging properties // thus we will always end up at step 10 $scope.step_10_style = 'danger'; - $scope.step_10_file_message = 'Warning(s)/Error(s) occurred while processing the file(s):\n' + JSON.stringify(progress_data.file_info, null, 2) + $scope.step_10_file_message = 'Warning(s)/Error(s) occurred while processing the file(s):\n' + JSON.stringify(progress_data.file_info, null, 2); } // If merges against existing exist, provide slightly different feedback diff --git a/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js b/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js new file mode 100644 index 0000000000..fb02d8d1f4 --- /dev/null +++ b/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js @@ -0,0 +1,49 @@ +/** + * :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. + * :author + */ +angular.module('BE.seed.controller.export_buildingsync_modal', []) + .controller('export_buildingsync_modal_controller', [ + '$http', + '$window', + '$scope', + '$uibModalInstance', + 'property_view_id', + 'column_mapping_presets', + function ( + $http, + $window, + $scope, + $uibModalInstance, + property_view_id, + column_mapping_presets + ) { + $scope.column_mapping_presets = column_mapping_presets; + $scope.current_column_mapping_preset = column_mapping_presets[0]; + + $scope.download_file = function () { + var the_url = '/api/v2_1/properties/' + property_view_id + '/building_sync/'; + $http.get( + the_url, + { params: { preset_id: $scope.current_column_mapping_preset.id } } + ).then(function (response) { + var blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'}); + var downloadLink = angular.element(''); + var filename = 'buildingsync_property_' + property_view_id + '.xml'; + downloadLink.attr('href', $window.URL.createObjectURL(blob)); + downloadLink.attr('download', filename); + downloadLink[0].click(); + $uibModalInstance.close(); + }, function (err) { + $scope.download_error_message = err.data ? err.data : err.toString(); + }); + }; + + $scope.close = function () { + $uibModalInstance.close(); + }; + + $scope.cancel = function () { + $uibModalInstance.dismiss(); + }; + }]); diff --git a/seed/static/seed/js/controllers/inventory_cycles_controller.js b/seed/static/seed/js/controllers/inventory_cycles_controller.js index 8642ce7b91..706ca429f9 100644 --- a/seed/static/seed/js/controllers/inventory_cycles_controller.js +++ b/seed/static/seed/js/controllers/inventory_cycles_controller.js @@ -44,7 +44,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.currentProfile = current_profile; // Scope columns/data to only those of the given inventory_type - var state_type = $scope.inventory_type == "properties" ? "PropertyState" : "TaxLotState"; + var state_type = $scope.inventory_type == 'properties' ? 'PropertyState' : 'TaxLotState'; $scope.all_columns = _.filter(all_columns, {table_name: state_type}); // set up i18n @@ -61,13 +61,13 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.included_cycle_ids = _.map(_.keys(inventory), function (cycle_id) { return parseInt(cycle_id); }); - $scope.cycle_options = _.map(cycles.cycles, function(cycle) { + $scope.cycle_options = _.map(cycles.cycles, function (cycle) { var selected = $scope.included_cycle_ids.includes(cycle.id); return { selected: selected, label: cycle.name, value: cycle.id, - start: cycle.start, + start: cycle.start }; }); @@ -84,15 +84,15 @@ angular.module('BE.seed.controller.inventory_cycles', []) }; // Takes raw cycle-partitioned records and returns array of cycle-aware records - $scope.format_records = function(raw_inventory) { - return _.reduce(raw_inventory, function(all_records, records, cycle_id) { + $scope.format_records = function (raw_inventory) { + return _.reduce(raw_inventory, function (all_records, records, cycle_id) { var cycle = _.find($scope.cycle_options, { value: parseInt(cycle_id) }); - _.forEach(records, function(record) { + _.forEach(records, function (record) { record.cycle_name = cycle.label; record.cycle_start = cycle.start; - all_records.push(record) - }) - return all_records + all_records.push(record); + }); + return all_records; }, []); }; @@ -100,16 +100,16 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.data = $scope.format_records(inventory); // Refreshes inventory by making API call - $scope.refresh_objects = function() { + $scope.refresh_objects = function () { spinner_utility.show(); var profile_id = _.has($scope.currentProfile, 'id') ? $scope.currentProfile.id : undefined; - if ($scope.inventory_type == "properties") { - inventory_service.properties_cycle(profile_id, $scope.included_cycle_ids).then(function(refreshed_inventory) { + if ($scope.inventory_type == 'properties') { + inventory_service.properties_cycle(profile_id, $scope.included_cycle_ids).then(function (refreshed_inventory) { $scope.data = $scope.format_records(refreshed_inventory); spinner_utility.hide(); }); } else { - inventory_service.taxlots_cycle(profile_id, $scope.included_cycle_ids).then(function(refreshed_inventory) { + inventory_service.taxlots_cycle(profile_id, $scope.included_cycle_ids).then(function (refreshed_inventory) { $scope.data = $scope.format_records(refreshed_inventory); spinner_utility.hide(); }); @@ -117,34 +117,34 @@ angular.module('BE.seed.controller.inventory_cycles', []) }; // On profile change, refreshes objects and rebuild columns - $scope.profile_change = function() { + $scope.profile_change = function () { inventory_service.save_last_profile($scope.currentProfile.id, $scope.inventory_type); $scope.refresh_objects(); // uiGrid doesn't recognize complete columnDefs swap unless it's removed and refreshed and notified for each $scope.gridOptions.columnDefs = []; - $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN) + $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN); $scope.build_columns(); $scope.gridOptions.columnDefs = $scope.columns; - $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN) + $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN); }; // Agg function returning last value of matching criteria field (all should be the same if they match) - $scope.matching_field_value = function(aggregation, fieldValue) { + $scope.matching_field_value = function (aggregation, fieldValue) { aggregation.value = fieldValue; }; // matching_criteria_columns identified here to pin left on table - if ($scope.inventory_type == "properties") { + if ($scope.inventory_type == 'properties') { $scope.matching_criteria_columns = matching_criteria_columns.PropertyState; } else { $scope.matching_criteria_columns = matching_criteria_columns.TaxLotState; } // Builds columns with profile, default, and grouping settings - $scope.build_columns = function() { + $scope.build_columns = function () { $scope.columns = []; // Profile Settings @@ -167,7 +167,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) var column_def_defaults = { headerCellFilter: 'translate', minWidth: default_min_width, - width: 150, + width: 150 }; _.map($scope.columns, function (col) { @@ -185,7 +185,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) // Help indicate matching columns are given preferred sort priority col.displayName = col.displayName + '*'; - options.headerCellClass = "matching-column-header"; + options.headerCellClass = 'matching-column-header'; options.customTreeAggregationFn = $scope.matching_field_value; options.width = autopin_width; @@ -204,7 +204,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) visible: false, suppressRemoveSort: true, // since grouping relies on sorting minWidth: default_min_width, - width: autopin_width, + width: autopin_width }, { name: 'inventory detail link icon', @@ -222,31 +222,31 @@ angular.module('BE.seed.controller.inventory_cycles', []) enableSorting: false, pinnedLeft: true, visible: true, - width: 30, + width: 30 }, { - name: "cycle_name", - displayName: "Cycle", + name: 'cycle_name', + displayName: 'Cycle', pinnedLeft: true, treeAggregationType: uiGridGroupingConstants.aggregation.COUNT, customTreeAggregationFinalizerFn: function (aggregation) { - aggregation.rendered = "total cycles: " + aggregation.value; + aggregation.rendered = 'total cycles: ' + aggregation.value; }, minWidth: default_min_width, - width: autopin_width, + width: autopin_width }, { - name: "cycle_start", - displayName: "Cycle Start", + name: 'cycle_start', + displayName: 'Cycle Start', cellFilter: 'date:\'yyyy-MM-dd\'', filter: inventory_service.dateFilter(), type: 'date', sort: { priority: 1, direction: 'asc' }, pinnedLeft: true, minWidth: default_min_width, - width: autopin_width, - }, - ) + width: autopin_width + } + ); }; $scope.build_columns(); @@ -257,7 +257,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) // Lastly, non-matching columns are given next priority so that users can sort within a grouped set. if (sortColumns.length > 1) { var matching_cols = _.filter(sortColumns, function (col) { - return col.colDef.is_matching_criteria; + return col.colDef.is_matching_criteria; }); var linking_id_col = _.find(sortColumns, ['name', 'id']); var remaining_cols = _.filter(sortColumns, function (col) { @@ -266,7 +266,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) sortColumns = matching_cols.concat(linking_id_col).concat(remaining_cols); _.forEach(sortColumns, function (col, index) { col.sort.priority = index; - }) + }); } }; @@ -283,8 +283,8 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.gridApi.core.on.filterChanged($scope, function () { // This is a workaround for losing the state of expanded rows during filtering. - _.delay($scope.gridApi.treeBase.expandAllRows, 500); - }) + _.delay($scope.gridApi.treeBase.expandAllRows, 500); + }); // Prioritized to maintain grouping. $scope.gridApi.core.on.sortChanged($scope, prioritize_sort); @@ -296,7 +296,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.$on('$destroy', function () { angular.element($window).off('resize', debouncedHeightUpdate); }); - }, + } }; $scope.updateHeight = function () { diff --git a/seed/static/seed/js/controllers/inventory_detail_controller.js b/seed/static/seed/js/controllers/inventory_detail_controller.js index d53c4d7e47..da3e906bb6 100644 --- a/seed/static/seed/js/controllers/inventory_detail_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_controller.js @@ -115,7 +115,7 @@ angular.module('BE.seed.controller.inventory_detail', []) } }; - function populated_columns_modal () { + function populated_columns_modal() { $uibModal.open({ backdrop: 'static', templateUrl: urls.static_url + 'seed/partials/show_populated_columns_modal.html', @@ -127,9 +127,7 @@ angular.module('BE.seed.controller.inventory_detail', []) currentProfile: function () { return $scope.currentProfile; }, - cycle: function () { - return null; - }, + cycle: _.constant(null), inventory_type: function () { return $stateParams.inventory_type; }, @@ -144,7 +142,7 @@ angular.module('BE.seed.controller.inventory_detail', []) }); // add "master" copy - item_copy = angular.copy($scope.item_state); + var item_copy = angular.copy($scope.item_state); _.defaults(item_copy, $scope.item_state.extra_data); return provided_inventory; @@ -500,16 +498,37 @@ angular.module('BE.seed.controller.inventory_detail', []) }; $scope.export_building_sync = function () { - var the_url = '/api/v2_1/properties/' + $stateParams.view_id + '/building_sync/'; - $http.get(the_url, {}) - .then(function (response) { - var blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'}); - var downloadLink = angular.element(''); - var filename = 'buildingsync_property_' + $stateParams.view_id + '.xml'; - downloadLink.attr('href', $window.URL.createObjectURL(blob)); - downloadLink.attr('download', filename); - downloadLink[0].click(); - }); + var modalInstance = $uibModal.open({ + templateUrl: urls.static_url + 'seed/partials/export_buildingsync_modal.html', + controller: 'export_buildingsync_modal_controller', + resolve: { + property_view_id: function () { + return $stateParams.view_id; + }, + column_mapping_presets: [ + 'column_mappings_service', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', + function ( + column_mappings_service, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ) { + var filter_preset_types = [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ]; + return column_mappings_service.get_column_mapping_presets_for_org( + $scope.organization.id, + filter_preset_types + ).then(function (response) { + return response.data; + }); + }] + } + }); + modalInstance.result.then(function () { + }); }; $scope.export_building_sync_xlsx = function () { diff --git a/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js b/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js index fbe850b55c..e45b479abc 100644 --- a/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js @@ -30,19 +30,19 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) ) { $scope.inventory_type = $stateParams.inventory_type; $scope.inventory = { - view_id: $stateParams.view_id, + view_id: $stateParams.view_id }; $scope.states = inventory_payload.data; $scope.base_state = _.find(inventory_payload.data, {view_id: $stateParams.view_id}); - $scope.cycles = _.reduce(cycles.cycles, function(cycles_by_id, cycle) { + $scope.cycles = _.reduce(cycles.cycles, function (cycles_by_id, cycle) { cycles_by_id[cycle.id] = cycle; return cycles_by_id; }, {}); // Flag columns whose values have changed between cycles. - var changes_check = function(column) { + var changes_check = function (column) { var uniq_column_values; if (column.is_extra_data) { @@ -53,7 +53,7 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) uniq_column_values = _.uniqBy($scope.states, column.column_name); } - column['changed'] = uniq_column_values.length > 1; + column.changed = uniq_column_values.length > 1; return column; }; @@ -69,7 +69,7 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) }); } else { // No profiles exist - $scope.columns = _.map(_.reject(columns, 'is_extra_data'), function(col) { + $scope.columns = _.map(_.reject(columns, 'is_extra_data'), function (col) { return changes_check(col); }); } @@ -89,7 +89,7 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) // Horizontal scroll for "2 tables" that scroll together for fixed header effect. var table_container = $('.table-xscroll-fixed-header-container'); - table_container.scroll(function() { + table_container.scroll(function () { $('.table-xscroll-fixed-header-container > .table-body-x-scroll').width( table_container.width() + table_container.scrollLeft() ); diff --git a/seed/static/seed/js/controllers/inventory_detail_meters_controller.js b/seed/static/seed/js/controllers/inventory_detail_meters_controller.js index ae086ffc7d..00ec61db36 100644 --- a/seed/static/seed/js/controllers/inventory_detail_meters_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_meters_controller.js @@ -32,30 +32,32 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) spinner_utility, urls, user_service, - $log, + $log ) { spinner_utility.show(); $scope.item_state = inventory_payload.state; $scope.inventory_type = $stateParams.inventory_type; $scope.organization = user_service.get_organization(); $scope.filler_cycle = cycles.cycles[0].id; - $scope.scenarios = _.uniqBy(_.map(meters, function(meter) { + $scope.scenarios = _.uniqBy(_.map(meters, function (meter) { return { id: meter.scenario_id, name: meter.scenario_name - } - }), 'id').filter(scenario => scenario.id !== undefined && scenario.id !== null) + }; + }), 'id').filter(function (scenario) { + return !_.isNil(scenario.id); + }); $scope.inventory = { view_id: $stateParams.view_id }; - const getMeterLabel = (meter) => { - return meter.type + ' - ' + meter.source + ' - ' + meter.source_id - } + var getMeterLabel = function (meter) { + return meter.type + ' - ' + meter.source + ' - ' + meter.source_id; + }; - const resetSelections = () => { - $scope.meter_selections = _.map(sorted_meters, function(meter) { + var resetSelections = function () { + $scope.meter_selections = _.map(sorted_meters, function (meter) { return { selected: true, label: getMeterLabel(meter), @@ -63,14 +65,14 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) }; }); - $scope.scenario_selections = _.map($scope.scenarios, function(scenario) { + $scope.scenario_selections = _.map($scope.scenarios, function (scenario) { return { selected: true, label: scenario.name, value: scenario.id - } + }; }); - } + }; // On page load, all meters and readings $scope.data = property_meter_usage.readings; @@ -85,9 +87,9 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) } }; - $scope.scenario_selection_toggled = (is_open) => { + $scope.scenario_selection_toggled = function (is_open) { if (!is_open) { - $scope.applyFilters() + $scope.applyFilters(); } }; @@ -150,70 +152,85 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) }; // remove option to filter by scenario if there are no scenarios if ($scope.scenarios.length === 0) { - $scope.filterMethod.options = ['meter'] + $scope.filterMethod.options = ['meter']; } // given a list of meter labels, it returns the filtered readings and column defs // This is used by the primary filterBy... functions - const filterByMeterLabels = (readings, columnDefs, meterLabels) => { - const timeColumns = ['start_time', 'end_time', 'month', 'year'] - const selectedColumns = meterLabels.concat(timeColumns) + var filterByMeterLabels = function filterByMeterLabels (readings, columnDefs, meterLabels) { + var timeColumns = ['start_time', 'end_time', 'month', 'year']; + var selectedColumns = meterLabels.concat(timeColumns); + var filteredReadings = readings.map(function (reading) { + return Object.entries(reading).reduce(function (newReading, _ref) { + var key = _ref[0], + value = _ref[1]; - const filteredReadings = readings.map(reading => Object.entries(reading).reduce((newReading, [key, value]) => { - if (selectedColumns.includes(key)) { - newReading[key] = value; - } - return newReading; - }, {})); + if (selectedColumns.includes(key)) { + newReading[key] = value; + } - const filteredColumnDefs = columnDefs.filter(columnDef => selectedColumns.includes(columnDef.field)) - return { readings: filteredReadings, columnDefs: filteredColumnDefs} - } + return newReading; + }, {}); + }); + var filteredColumnDefs = columnDefs.filter(function (columnDef) { + return selectedColumns.includes(columnDef.field); + }); + return { + readings: filteredReadings, + columnDefs: filteredColumnDefs + }; + }; // given the meter selections, it returns the filtered readings and column defs - const filterByMeterSelections = (readings, columnDefs, meterSelections) => { + var filterByMeterSelections = function (readings, columnDefs, meterSelections) { // filter according to meter selections - const selectedMeterLabels = meterSelections.filter(selection => selection.selected) - .map(selection => selection.label); + var selectedMeterLabels = meterSelections.filter(function (selection) { + return selection.selected; + }) + .map(function (selection) { + return selection.label; + }); - return filterByMeterLabels(readings, columnDefs, selectedMeterLabels) - } + return filterByMeterLabels(readings, columnDefs, selectedMeterLabels); + }; // given the scenario selections, it returns the filtered readings and column defs - const filterByScenarioSelections = (readings, columnDefs, meters, scenarioSelections) => { - const selectedScenarioIds = scenarioSelections.filter(selection => selection.selected).map(selection => selection.value); - const selectedMeterLabels = meters.filter(meter => selectedScenarioIds.includes(meter.scenario_id)) - .map(meter => getMeterLabel(meter)) + var filterByScenarioSelections = function (readings, columnDefs, meters, scenarioSelections) { + var selectedScenarioIds = scenarioSelections.filter(function (selection) { + return selection.selected; + }).map(function (selection) { + return selection.value; + }); + var selectedMeterLabels = meters.filter(function (meter) { + return selectedScenarioIds.includes(meter.scenario_id); + }).map(function (meter) { + return getMeterLabel(meter); + }); - return filterByMeterLabels(readings, columnDefs, selectedMeterLabels) - } + return filterByMeterLabels(readings, columnDefs, selectedMeterLabels); + }; // filters the meter readings by selected meters or scenarios and updates the table - $scope.applyFilters = () => { - let readings, columnDefs; + $scope.applyFilters = function () { + var results, readings, columnDefs; if ($scope.filterMethod.selected === 'meter') { - ({readings, columnDefs} = filterByMeterSelections( - property_meter_usage.readings, - property_meter_usage.column_defs, - $scope.meter_selections - )) + results = filterByMeterSelections(property_meter_usage.readings, property_meter_usage.column_defs, $scope.meter_selections); + readings = results.readings; + columnDefs = results.columnDefs; } else if ($scope.filterMethod.selected === 'scenario') { - ({readings, columnDefs} = filterByScenarioSelections( - property_meter_usage.readings, - property_meter_usage.column_defs, - sorted_meters, - $scope.scenario_selections - )) + results = filterByScenarioSelections(property_meter_usage.readings, property_meter_usage.column_defs, sorted_meters, $scope.scenario_selections); + readings = results.readings; + columnDefs = results.columnDefs; } else { - $log.error("Invalid filter method selected: ", $scope.filterMethod) - return + $log.error('Invalid filter method selected: ', $scope.filterMethod); + return; } $scope.data = readings; $scope.gridOptions.columnDefs = columnDefs; $scope.has_readings = $scope.data.length > 0; $scope.apply_column_settings(); - } + }; // refresh_readings make an API call to refresh the base readings data // according to the selected interval diff --git a/seed/static/seed/js/controllers/inventory_detail_notes_controller.js b/seed/static/seed/js/controllers/inventory_detail_notes_controller.js index ae3e9d204d..5211861534 100644 --- a/seed/static/seed/js/controllers/inventory_detail_notes_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_notes_controller.js @@ -70,7 +70,7 @@ angular.module('BE.seed.controller.inventory_detail_notes', []) return $scope.org_id; }, note: _.constant({text: ''}), - action: _.constant('new'), + action: _.constant('new') } }); @@ -97,7 +97,7 @@ angular.module('BE.seed.controller.inventory_detail_notes', []) note: function () { return note; }, - action: _.constant('update'), + action: _.constant('update') } }); @@ -124,7 +124,7 @@ angular.module('BE.seed.controller.inventory_detail_notes', []) note: function () { return note; }, - action: _.constant('delete'), + action: _.constant('delete') } }); diff --git a/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js b/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js index 4a1a549ee4..1fa720e4b8 100644 --- a/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js @@ -36,7 +36,7 @@ angular.module('BE.seed.controller.inventory_detail_notes_modal', []) var data = { name: 'Manually Created', note_type: 'Note', - text: note.text, + text: note.text }; note_service.create_note($scope.orgId, $scope.inventoryType, $scope.viewId, data).then(function () { $uibModalInstance.close(); @@ -47,7 +47,7 @@ angular.module('BE.seed.controller.inventory_detail_notes_modal', []) var data = { name: note.name, note_type: note.note_type, - text: note.text, + text: note.text }; note_service.update_note($scope.orgId, $scope.inventoryType, $scope.viewId, note.id, data).then(function () { $uibModalInstance.close(); diff --git a/seed/static/seed/js/controllers/inventory_list_controller.js b/seed/static/seed/js/controllers/inventory_list_controller.js index 3271a5c485..aefe8d6218 100644 --- a/seed/static/seed/js/controllers/inventory_list_controller.js +++ b/seed/static/seed/js/controllers/inventory_list_controller.js @@ -59,7 +59,7 @@ angular.module('BE.seed.controller.inventory_list', []) $scope.data = []; var lastCycleId = inventory_service.get_last_cycle(); $scope.cycle = { - selected_cycle: _.find(cycles.cycles, {id: lastCycleId}) || _.first(cycles.cycles), + selected_cycle: _.find(cycles.cycles, {id: lastCycleId}) || _.first(cycles.cycles), cycles: cycles.cycles }; @@ -155,7 +155,7 @@ angular.module('BE.seed.controller.inventory_list', []) } }; - function populated_columns_modal() { + function populated_columns_modal () { $uibModal.open({ backdrop: 'static', templateUrl: urls.static_url + 'seed/partials/show_populated_columns_modal.html', @@ -173,9 +173,7 @@ angular.module('BE.seed.controller.inventory_list', []) inventory_type: function () { return $stateParams.inventory_type; }, - provided_inventory: function () { - return null; - } + provided_inventory: _.constant(null) } }); } @@ -192,7 +190,7 @@ angular.module('BE.seed.controller.inventory_list', []) }); }; - function updateApplicableLabels(current_labels) { + function updateApplicableLabels (current_labels) { var inventoryIds; if ($scope.inventory_type === 'properties') { inventoryIds = _.map($scope.data, 'property_view_id').sort(); @@ -829,7 +827,7 @@ angular.module('BE.seed.controller.inventory_list', []) }); }; - function currentColumns() { + function currentColumns () { // Save all columns except first 3 var gridCols = _.filter($scope.gridApi.grid.columns, function (col) { return !_.includes(['treeBaseRowHeaderCol', 'selectionRowHeaderCol', 'notes_count', 'merged_indicator', 'id'], col.name) && col.visible; diff --git a/seed/static/seed/js/controllers/inventory_map_controller.js b/seed/static/seed/js/controllers/inventory_map_controller.js index 724e5f8ff5..8d13575d21 100644 --- a/seed/static/seed/js/controllers/inventory_map_controller.js +++ b/seed/static/seed/js/controllers/inventory_map_controller.js @@ -541,7 +541,7 @@ angular.module('BE.seed.controller.inventory_map', []) }); }; - function updateApplicableLabels() { + function updateApplicableLabels () { var inventoryIds; if ($scope.inventory_type === 'properties') { inventoryIds = _.map($scope.data, 'property_view_id').sort(); diff --git a/seed/static/seed/js/controllers/inventory_reports_controller.js b/seed/static/seed/js/controllers/inventory_reports_controller.js index 2ab0c5d0ac..44329d4093 100644 --- a/seed/static/seed/js/controllers/inventory_reports_controller.js +++ b/seed/static/seed/js/controllers/inventory_reports_controller.js @@ -318,7 +318,7 @@ angular.module('BE.seed.controller.inventory_reports', []) xVar: $scope.chartData.xAxisVarName, xLabel: $scope.chartData.xAxisTitle, yVar: $scope.chartData.yAxisVarName, - yLabel: $scope.chartData.yAxisTitle, + yLabel: $scope.chartData.yAxisTitle }; }, cycle_start: function () { @@ -326,7 +326,7 @@ angular.module('BE.seed.controller.inventory_reports', []) }, cycle_end: function () { return $scope.toCycle.selected_cycle.end; - }, + } } }); }; @@ -405,7 +405,7 @@ angular.module('BE.seed.controller.inventory_reports', []) $scope.toCycle.selected_cycle.end ).then(function (data) { data = data.aggregated_data; - console.log(data); + $log.log(data); $scope.aggPropertyCounts = data.property_counts; var propertyCounts = data.property_counts; var colorsArr = mapColors(propertyCounts); @@ -440,7 +440,7 @@ angular.module('BE.seed.controller.inventory_reports', []) localStorage.setItem(localStorageFromCycleKey, JSON.stringify($scope.fromCycle.selected_cycle)); localStorage.setItem(localStorageToCycleKey, JSON.stringify($scope.toCycle.selected_cycle)); - }; + } /* Generate an array of color objects to be used as part of chart configuration Each color object should have the following properties: diff --git a/seed/static/seed/js/controllers/mapping_controller.js b/seed/static/seed/js/controllers/mapping_controller.js index 0fdcfb7783..86dea1a800 100644 --- a/seed/static/seed/js/controllers/mapping_controller.js +++ b/seed/static/seed/js/controllers/mapping_controller.js @@ -30,6 +30,9 @@ angular.module('BE.seed.controller.mapping', []) 'i18nService', // from ui-grid 'simple_modal_service', 'Notification', + 'COLUMN_MAPPING_PRESET_TYPE_NORMAL', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', function ( $scope, $log, @@ -56,10 +59,13 @@ angular.module('BE.seed.controller.mapping', []) $translate, i18nService, simple_modal_service, - Notification + Notification, + COLUMN_MAPPING_PRESET_TYPE_NORMAL, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM ) { $scope.presets = [ - {id: 0, mappings: [], name: ""} + {id: 0, mappings: [], name: ''} ].concat(column_mapping_presets_payload); // $scope.selected_preset = $scope.applied_preset = $scope.mock_presets[0]; @@ -76,6 +82,16 @@ angular.module('BE.seed.controller.mapping', []) $scope.mappings_change_possible = true; }; + $scope.is_buildingsync_and_preset_not_ok = function () { + if (!$scope.mappingBuildingSync) { + return false; + } + // BuildingSync requires a saved preset to be applied + return $scope.current_preset.id === 0 + || $scope.preset_change_possible + || $scope.mappings_change_possible; + }; + var analyze_chosen_inventory_types = function () { var chosenTypes = _.uniq(_.map($scope.mappings, 'suggestion_table_name')); var all_cols_have_table_name = !_.find($scope.mappings, {suggestion_table_name: undefined}); @@ -113,13 +129,22 @@ angular.module('BE.seed.controller.mapping', []) var preset_mappings_from_working_mappings = function () { // for to_field, try DB col name, if not use col display name return _.reduce($scope.mappings, function (preset_mapping_data, mapping) { - preset_mapping_data.push({ + var this_mapping = { from_field: mapping.name, from_units: mapping.from_units, to_field: mapping.suggestion_column_name || mapping.suggestion || '', to_table_name: mapping.suggestion_table_name - }); - + }; + var isBuildingSyncPreset = $scope.current_preset.preset_type !== undefined + && [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ].includes($scope.current_preset.preset_type); + + if (isBuildingSyncPreset) { + this_mapping.from_field_value = mapping.from_field_value; + } + preset_mapping_data.push(this_mapping); return preset_mapping_data; }, []); }; @@ -127,12 +152,43 @@ angular.module('BE.seed.controller.mapping', []) $scope.new_preset = function () { var preset_mapping_data = preset_mappings_from_working_mappings(); + var presetType; + if (!$scope.mappingBuildingSync) { + presetType = COLUMN_MAPPING_PRESET_TYPE_NORMAL; + } else { + presetType = COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM; + + // make sure the new preset mapping data has the required data + var currentPresetForBuildingSync = + $scope.current_preset.preset_type !== undefined + && [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ].includes($scope.current_preset.preset_type); + + if (!currentPresetForBuildingSync) { + // we need to add mapping data, from_field_value, using the default mapping + var defaultPreset = $scope.presets.find(function (preset) { + return preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT; + }); + preset_mapping_data = preset_mapping_data.map(function (mapping) { + // find the corresponding mapping in the default preset + var defaultMapping = defaultPreset.mappings.find(function (defaultMapping) { + return defaultMapping.from_field === mapping.from_field; + }); + return _.merge({}, mapping, { + from_field_value: defaultMapping.from_field_value + }); + }); + } + } + var modalInstance = $uibModal.open({ templateUrl: urls.static_url + 'seed/partials/column_mapping_preset_modal.html', controller: 'column_mapping_preset_modal_controller', resolve: { action: _.constant('new'), - data: _.constant({mappings: preset_mapping_data}), + data: _.constant({mappings: preset_mapping_data, preset_type: presetType}), org_id: _.constant(user_service.get_organization().id) } }); @@ -197,6 +253,8 @@ angular.module('BE.seed.controller.mapping', []) $scope.isValidCycle = Boolean(_.find(cycles.cycles, {id: $scope.import_file.cycle})); + $scope.mappingBuildingSync = $scope.import_file.source_type === 'BuildingSync Raw'; + matching_criteria_columns_payload.PropertyState = _.map(matching_criteria_columns_payload.PropertyState, function (column_name) { var display_name = _.find($scope.mappable_property_columns, {column_name: column_name}).display_name; return { @@ -219,10 +277,13 @@ angular.module('BE.seed.controller.mapping', []) $scope.setAllFieldsOptions = [{ name: 'Property', value: 'PropertyState' - }, { - name: 'Tax Lot', - value: 'TaxLotState' }]; + if (!$scope.mappingBuildingSync) { + $scope.setAllFieldsOptions.push({ + name: 'Tax Lot', + value: 'TaxLotState' + }); + } var eui_columns = _.filter($scope.mappable_property_columns, {data_type: 'eui'}); $scope.is_eui_column = function (col) { @@ -331,29 +392,20 @@ angular.module('BE.seed.controller.mapping', []) $scope.duplicates_present = duplicates_present; }; - const get_col_from_suggestion = (name) => { - if ($scope.import_file.source_type === "BuildingSync Raw") { - const suggestion = $scope.suggested_mappings[name]; + var get_col_from_suggestion = function (name) { - return { - name: name, - suggestion_column_name: suggestion[1], - suggestion_table_name: suggestion[0], - raw_data: _.map(first_five_rows_payload.first_five_rows, name) - }; - } - - const suggestion = _.find($scope.current_preset.mappings, {from_field: name}) || {}; + var suggestion = _.find($scope.current_preset.mappings, {from_field: name}) || {}; return { from_units: suggestion.from_units, name: name, + from_field_value: suggestion.from_field_value, raw_data: _.map(first_five_rows_payload.first_five_rows, name), suggestion: suggestion.to_field, suggestion_column_name: suggestion.to_field, suggestion_table_name: suggestion.to_table_name }; - } + }; /** * initialize_mappings: prototypical inheritance for all the raw columns @@ -362,7 +414,7 @@ angular.module('BE.seed.controller.mapping', []) $scope.initialize_mappings = function () { $scope.mappings = []; _.forEach($scope.raw_columns, function (name) { - let col = get_col_from_suggestion(name); + var col = get_col_from_suggestion(name); var match; if (col.suggestion_table_name === 'PropertyState') { @@ -378,7 +430,7 @@ angular.module('BE.seed.controller.mapping', []) } if (match) { col.suggestion = match.display_name; - } else if ($scope.import_file.source_type === "BuildingSync Raw") { + } else if ($scope.mappingBuildingSync) { col.suggestion = $filter('titleCase')(col.suggestion_column_name); } @@ -759,7 +811,9 @@ angular.module('BE.seed.controller.mapping', []) var init = function () { $scope.initialize_mappings(); - if ($scope.import_file.matching_done) { display_cached_column_mappings(); } + if ($scope.import_file.matching_done) { + display_cached_column_mappings(); + } $scope.updateColDuplicateStatus(); $scope.updateInventoryTypeDropdown(); diff --git a/seed/static/seed/js/controllers/members_controller.js b/seed/static/seed/js/controllers/members_controller.js index 2a79bf20ce..63680fe0a5 100644 --- a/seed/static/seed/js/controllers/members_controller.js +++ b/seed/static/seed/js/controllers/members_controller.js @@ -63,7 +63,7 @@ angular.module('BE.seed.controller.members', []) } }; - function confirm_remove_user(user) { + function confirm_remove_user (user) { organization_service.remove_user(user.user_id, $scope.org.id).then(function () { organization_service.get_organization_users({org_id: $scope.org.id}).then(function (data) { $scope.users = data.users; diff --git a/seed/static/seed/js/controllers/menu_controller.js b/seed/static/seed/js/controllers/menu_controller.js index d3e668eeb6..9d1c7d4037 100644 --- a/seed/static/seed/js/controllers/menu_controller.js +++ b/seed/static/seed/js/controllers/menu_controller.js @@ -88,14 +88,14 @@ angular.module('BE.seed.controller.menu', []) }; //Sets initial expanded/collapse state of sidebar menu - function init_menu() { + function init_menu () { //Default to false but use cookie value if one has been set var isNavExpanded = $cookies.seed_nav_is_expanded === 'true'; $scope.expanded_controller = isNavExpanded; $scope.collapsed_controller = !isNavExpanded; $scope.narrow_controller = isNavExpanded; $scope.wide_controller = !isNavExpanded; - }; + } // returns true if menu toggle has never been clicked, i.e. first run, else returns false $scope.menu_toggle_has_never_been_clicked = function () { diff --git a/seed/static/seed/js/controllers/merge_modal_controller.js b/seed/static/seed/js/controllers/merge_modal_controller.js index eaf90f9fcf..e874c77123 100644 --- a/seed/static/seed/js/controllers/merge_modal_controller.js +++ b/seed/static/seed/js/controllers/merge_modal_controller.js @@ -111,15 +111,15 @@ angular.module('BE.seed.controller.merge_modal', []) var plural = ($scope.inventory_type === 'properties' ? ' properties' : ' tax lots'); // The term "subsequent" below implies not including itself var merged_count = Math.max(result.match_merged_count - 1, 0); - var link_count = result.match_link_count; + var link_count = result.match_link_count; Notification.info({ message: (merged_count + ' subsequent ' + (merged_count === 1 ? singular : plural) + ' merged'), - delay: 10000, + delay: 10000 }); Notification.info({ message: ('Resulting ' + singular + ' has ' + link_count + ' cross-cycle link' + (link_count === 1 ? '' : 's')), - delay: 10000, + delay: 10000 }); }; @@ -136,8 +136,8 @@ angular.module('BE.seed.controller.merge_modal', []) }, headers: function () { return { - properties: "The resulting property will be further merged & linked with any matching properties.", - taxlots: "The resulting tax lot will be further merged & linked with any matching tax lots.", + properties: 'The resulting property will be further merged & linked with any matching properties.', + taxlots: 'The resulting tax lot will be further merged & linked with any matching tax lots.' }; } } @@ -155,7 +155,7 @@ angular.module('BE.seed.controller.merge_modal', []) state_ids = _.map($scope.data, 'property_state_id').reverse(); return matching_service.mergeProperties(state_ids).then(function (data) { Notification.success('Successfully merged ' + state_ids.length + ' properties'); - notify_merges_and_links(data) + notify_merges_and_links(data); $scope.close(); }, function (err) { $log.error(err); @@ -167,7 +167,7 @@ angular.module('BE.seed.controller.merge_modal', []) state_ids = _.map($scope.data, 'taxlot_state_id').reverse(); return matching_service.mergeTaxlots(state_ids).then(function (data) { Notification.success('Successfully merged ' + state_ids.length + ' tax lots'); - notify_merges_and_links(data) + notify_merges_and_links(data); $scope.close(); }, function (err) { $log.error(err); diff --git a/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js b/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js index 455cb86119..acd72b28e1 100644 --- a/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js +++ b/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js @@ -33,33 +33,33 @@ angular.module('BE.seed.controller.record_match_merge_link_modal', []) $scope.helpBtnText = 'Expand Help'; - $scope.changeHelpBtnText = function(helpBtnText) { - if(helpBtnText === 'Collapse Help'){ - $scope.helpBtnText = 'Expand Help' ; + $scope.changeHelpBtnText = function (helpBtnText) { + if (helpBtnText === 'Collapse Help') { + $scope.helpBtnText = 'Expand Help'; } else { $scope.helpBtnText = 'Collapse Help'; } }; - promises = [ - organization_service.matching_criteria_columns(organization_id), + var promises = [ + organization_service.matching_criteria_columns(organization_id) ]; if (inventory_type === 'properties') { - promises.unshift(inventory_service.get_property_columns()) + promises.unshift(inventory_service.get_property_columns()); } else if (inventory_type === 'taxlots') { - promises.unshift(inventory_service.get_taxlot_columns()) + promises.unshift(inventory_service.get_taxlot_columns()); } - $scope.matching_criteria_columns = ["Loading..."]; + $scope.matching_criteria_columns = ['Loading...']; - $q.all(promises).then(function(results) { - var inventory_columns = _.filter(results[0], { table_name: $scope.table_name }); + $q.all(promises).then(function (results) { + var inventory_columns = _.filter(results[0], {table_name: $scope.table_name}); var raw_column_names = results[1][$scope.table_name]; // Use display names to identify matching criteria columns. - $scope.matching_criteria_columns = _.map(raw_column_names, function(col_name) { - return _.find(inventory_columns, { column_name: col_name }).displayName; + $scope.matching_criteria_columns = _.map(raw_column_names, function (col_name) { + return _.find(inventory_columns, {column_name: col_name}).displayName; }); }); diff --git a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js index 0198b76717..e6b87b7990 100644 --- a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js +++ b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js @@ -58,7 +58,7 @@ angular.module('BE.seed.controller.show_populated_columns_modal', []) var relatedCols = _.filter($scope.columns, 'related'); // console.log('relatedCols', relatedCols); - var col_key = provided_inventory ? "column_name" : "name"; + var col_key = provided_inventory ? 'column_name' : 'name'; _.forEach(inventory, function (record, index) { // console.log(cols.length + ' remaining cols to check'); diff --git a/seed/static/seed/js/seed.js b/seed/static/seed/js/seed.js index 4f41f331eb..78d5ffa60e 100644 --- a/seed/static/seed/js/seed.js +++ b/seed/static/seed/js/seed.js @@ -55,6 +55,7 @@ angular.module('BE.seed.controllers', [ 'BE.seed.controller.delete_modal', 'BE.seed.controller.delete_user_modal', 'BE.seed.controller.developer', + 'BE.seed.controller.export_buildingsync_modal', 'BE.seed.controller.export_report_modal', 'BE.seed.controller.export_inventory_modal', 'BE.seed.controller.geocode_modal', @@ -152,7 +153,8 @@ var SEED_app = angular.module('BE.seed', [ 'BE.seed.directives', 'BE.seed.services', 'BE.seed.controllers', - 'BE.seed.utilities' + 'BE.seed.utilities', + 'BE.seed.constants' ], ['$interpolateProvider', '$qProvider', function ($interpolateProvider, $qProvider) { $interpolateProvider.startSymbol('{$'); $interpolateProvider.endSymbol('$}'); @@ -239,7 +241,6 @@ SEED_app.run([ // Revert the url when the transition was triggered by a sidebar link (options.source === 'url') if (transition.options().source === 'url') { - var $state = transition.router.stateService; var $urlRouter = transition.router.urlRouter; $urlRouter.push($state.$current.navigable.url, $state.params, {replace: true}); @@ -529,12 +530,37 @@ SEED_app.config(['stateHelperProvider', '$urlRouterProvider', '$locationProvider templateUrl: static_url + 'seed/partials/mapping.html', controller: 'mapping_controller', resolve: { - column_mapping_presets_payload: ['column_mappings_service', 'user_service', function (column_mappings_service, user_service) { - var organization_id = user_service.get_organization().id; - return column_mappings_service.get_column_mapping_presets_for_org(organization_id).then(function (response) { - return response.data; - }); - }], + column_mapping_presets_payload: [ + 'column_mappings_service', + 'user_service', + 'COLUMN_MAPPING_PRESET_TYPE_NORMAL', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', + 'import_file_payload', + function ( + column_mappings_service, + user_service, + COLUMN_MAPPING_PRESET_TYPE_NORMAL, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + import_file_payload) { + var filter_preset_types; + if (import_file_payload.import_file.source_type === 'BuildingSync Raw') { + filter_preset_types = [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ]; + } else { + filter_preset_types = [COLUMN_MAPPING_PRESET_TYPE_NORMAL]; + } + var organization_id = user_service.get_organization().id; + return column_mappings_service.get_column_mapping_presets_for_org( + organization_id, + filter_preset_types + ).then(function (response) { + return response.data; + }); + }], import_file_payload: ['dataset_service', '$stateParams', function (dataset_service, $stateParams) { var importfile_id = $stateParams.importfile_id; return dataset_service.get_import_file(importfile_id); diff --git a/seed/static/seed/js/services/column_mappings_service.js b/seed/static/seed/js/services/column_mappings_service.js index f67bbd1890..d41ba178a7 100644 --- a/seed/static/seed/js/services/column_mappings_service.js +++ b/seed/static/seed/js/services/column_mappings_service.js @@ -28,11 +28,15 @@ angular.module('BE.seed.service.column_mappings', []).factory('column_mappings_s }); }; - column_mappings_factory.get_column_mapping_presets_for_org = function (org_id) { + column_mappings_factory.get_column_mapping_presets_for_org = function (org_id, filter_preset_types) { + var params = { + organization_id: org_id + }; + if (filter_preset_types != null) { + params.preset_type = filter_preset_types; + } return $http.get('/api/v2/column_mapping_presets/', { - params: { - organization_id: org_id - } + params: params }).then(function (response) { return response.data; }); @@ -54,10 +58,10 @@ angular.module('BE.seed.service.column_mappings', []).factory('column_mappings_s column_mappings_factory.get_header_suggestions_for_org = function (org_id, headers) { return $http.post('/api/v2/column_mapping_presets/suggestions/', { - headers: headers, + headers: headers }, { params: { - organization_id: org_id, + organization_id: org_id } }).then(function (response) { return response.data; diff --git a/seed/static/seed/js/services/inventory_reports_service.js b/seed/static/seed/js/services/inventory_reports_service.js index a82b3c65f6..f659ce091d 100644 --- a/seed/static/seed/js/services/inventory_reports_service.js +++ b/seed/static/seed/js/services/inventory_reports_service.js @@ -44,7 +44,7 @@ angular.module('BE.seed.service.inventory_reports', function get_report_data (xVar, yVar, start, end) { // Error checks - if (_.isNil(xVar) || _.isNil(yVar) || _.isNil(start) || _.isNil(end)) { + if (_.some([xVar, yVar, start, end], _.isNil)) { $log.error('#inventory_reports_service.get_report_data(): null parameter'); throw new Error('Invalid Parameter'); } @@ -97,7 +97,7 @@ angular.module('BE.seed.service.inventory_reports', function get_aggregated_report_data (xVar, yVar, start, end) { // Error checks - if (_.isNil(xVar) || _.isNil(yVar) || _.isNil(start) || _.isNil(end)) { + if (_.some([xVar, yVar, start, end], _.isNil)) { $log.error('#inventory_reports_service.get_aggregated_report_data(): null parameter'); throw new Error('Invalid Parameter'); } @@ -123,8 +123,8 @@ angular.module('BE.seed.service.inventory_reports', var xLabel = axes_data.xLabel; var yVar = axes_data.yVar; var yLabel = axes_data.yLabel; - // Error checks - if (_.isNil(xVar) || _.isNil(xLabel) || _.isNil(yVar) || _.isNil(yLabel) || _.isNil(start) || _.isNil(end)) { + // Error checks + if (_.some([xVar, xLabel, yVar, yLabel, start, end], _.isNil)) { $log.error('#inventory_reports_service.get_aggregated_report_data(): null parameter'); throw new Error('Invalid Parameter'); } @@ -137,11 +137,11 @@ angular.module('BE.seed.service.inventory_reports', y_var: yVar, y_label: yLabel, start: start, - end: end, + end: end }, responseType: 'arraybuffer' }).then(function (response) { - return response + return response; }); } @@ -158,7 +158,7 @@ angular.module('BE.seed.service.inventory_reports', //get_summary_data : get_summary_data, get_report_data: get_report_data, get_aggregated_report_data: get_aggregated_report_data, - export_reports_data: export_reports_data, + export_reports_data: export_reports_data }; diff --git a/seed/static/seed/js/services/inventory_service.js b/seed/static/seed/js/services/inventory_service.js index b03953cb45..165a77414d 100644 --- a/seed/static/seed/js/services/inventory_service.js +++ b/seed/static/seed/js/services/inventory_service.js @@ -54,7 +54,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ return $http.post('/api/v2/properties/cycles/', { organization_id: user_service.get_organization().id, profile_id: profile_id, - cycle_ids: cycle_ids, + cycle_ids: cycle_ids }).then(function (response) { return response.data; }); @@ -163,7 +163,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ spinner_utility.show(); return $http.post('/api/v2/properties/' + view_id + '/links/', { - organization_id: user_service.get_organization().id, + organization_id: user_service.get_organization().id }).then(function (response) { return response.data; }).finally(function () { @@ -288,7 +288,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ return $http.post('/api/v2/taxlots/cycles/', { organization_id: user_service.get_organization().id, profile_id: profile_id, - cycle_ids: cycle_ids, + cycle_ids: cycle_ids }).then(function (response) { return response.data; }); @@ -392,7 +392,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ spinner_utility.show(); return $http.post('/api/v2/taxlots/' + view_id + '/links/', { - organization_id: user_service.get_organization().id, + organization_id: user_service.get_organization().id }).then(function (response) { return response.data; }).finally(function () { diff --git a/seed/static/seed/js/services/label_service.js b/seed/static/seed/js/services/label_service.js index f05655b0d4..62cc8283e1 100644 --- a/seed/static/seed/js/services/label_service.js +++ b/seed/static/seed/js/services/label_service.js @@ -61,7 +61,7 @@ angular.module('BE.seed.service.label', []) return get_labels_for_org(user_service.get_organization().id, inventory_type, filter_ids); } - function get_labels_for_org (org_id, inventory_type, filter_ids) { + function get_labels_for_org(org_id, inventory_type, filter_ids) { var params = { organization_id: org_id }; @@ -78,7 +78,6 @@ angular.module('BE.seed.service.label', []) }).then(map_labels); } - /* Add a label to an organization's list of labels @param {object} label Label object to use for creating label on server. @@ -105,7 +104,6 @@ angular.module('BE.seed.service.label', []) }); } - /* Update an existing a label in an organization @param {object} label A label object with changed properties to update on server. diff --git a/seed/static/seed/js/services/main_service.js b/seed/static/seed/js/services/main_service.js index 241b6c5835..57ae150b4f 100644 --- a/seed/static/seed/js/services/main_service.js +++ b/seed/static/seed/js/services/main_service.js @@ -9,7 +9,7 @@ angular.module('BE.seed.service.main', []).factory('main_service', [ var main_factory = {}; main_factory.version = function () { - return $http.get('/api/v2/version').then(function (response) { + return $http.get('/api/v2/version/').then(function (response) { return response.data; }); }; diff --git a/seed/static/seed/js/services/organization_service.js b/seed/static/seed/js/services/organization_service.js index 30981cd76e..fe5315c24a 100644 --- a/seed/static/seed/js/services/organization_service.js +++ b/seed/static/seed/js/services/organization_service.js @@ -165,7 +165,7 @@ angular.module('BE.seed.service.organization', []).factory('organization_service organization_factory.match_merge_link = function (org_id, inventory_type) { return $http.post('/api/v2/organizations/' + org_id + '/match_merge_link/', { - inventory_type: inventory_type, + inventory_type: inventory_type }).then(function (response) { return response.data; }); @@ -181,7 +181,7 @@ angular.module('BE.seed.service.organization', []).factory('organization_service return $http.post('/api/v2/organizations/' + org_id + '/match_merge_link_preview/', { inventory_type: inventory_type, add: criteria_change_columns.add, - remove: criteria_change_columns.remove, + remove: criteria_change_columns.remove }).then(function (response) { return response.data; }); diff --git a/seed/static/seed/js/services/uploader_service.js b/seed/static/seed/js/services/uploader_service.js index 8df8013c1e..bc02e58d4e 100644 --- a/seed/static/seed/js/services/uploader_service.js +++ b/seed/static/seed/js/services/uploader_service.js @@ -39,6 +39,26 @@ angular.module('BE.seed.service.uploader', []).factory('uploader_service', [ }); }; + /** + * validate_use_cases + * This service call will simply call a view on the backend to validate + * BuildingSync files with use cases + * @param file_id: the pk of a ImportFile object we're going to save raw. + */ + uploader_factory.validate_use_cases = function (file_id) { + return $http.post('/api/v2/import_files/' + file_id + '/validate_use_cases/') + .then(response => { + return response.data + }) + .catch(err => { + if (err.data.status === 'error') { + return err.data + } + // something unexpected happened... throw it + throw err + }) + }; + /** * save_raw_data * This service call will simply call a view on the backend to save raw diff --git a/seed/static/seed/partials/column_mappings.html b/seed/static/seed/partials/column_mappings.html index 5cb6d4eee9..a9c8dc90b6 100644 --- a/seed/static/seed/partials/column_mappings.html +++ b/seed/static/seed/partials/column_mappings.html @@ -31,13 +31,13 @@

Column Mappings

{$:: 'Column Mapping Preset' | translate $}: - - - -
+
@@ -95,8 +95,10 @@

Column Mappings

+ {$:: 'Set all fields to' | translate $}: +

Mapped Fields

@@ -110,7 +112,7 @@

Column Mappings

Measurement Units Data File Header - + @@ -119,7 +121,7 @@

Column Mappings

@@ -130,14 +132,15 @@

Column Mappings

class="form-control input-sm tcm_field" ng-change="flag_change(col)" typeahead-on-select="seed_header_change(col)" - ng-attr-id="mapped-row-input-box-{$ $index $}"> + ng-attr-id="mapped-row-input-box-{$ $index $}" + ng-disabled="!preset_action_ok('change_to_field')"> - - @@ -146,13 +149,13 @@

Column Mappings

- + {$:: cell_value $} - @@ -160,7 +163,7 @@

Column Mappings

- diff --git a/seed/static/seed/partials/data_upload_modal.html b/seed/static/seed/partials/data_upload_modal.html index f0af0ca658..3964b8fbc0 100644 --- a/seed/static/seed/partials/data_upload_modal.html +++ b/seed/static/seed/partials/data_upload_modal.html @@ -12,7 +12,6 @@

{$:: 'UPLOAD_COMPLETE_TITLE' | trans

Confirm Save Mappings?

Error Processing Data

Portfolio Manager Import

-

Successful upload!

UPLOAD_PM_READINGS_MODAL_TITLE

PM_METER_IMPORT_RESULTS

@@ -100,6 +99,18 @@

PM_METER_IMPORT_RESULTS +
+
+

There were one or more warnings for the file(s) uploaded.

+
+ {$ file_issues.file $} +
+
Use Case Warnings
+
{$ warn $}
+
+
+
+
@@ -250,10 +261,26 @@

PM_METER_IMPORT_RESULTS
-

An error occurred while processing the file. Please ensure that your file meets the required specifications.

-

Error(s) occurred while processing the BuildingSync data bulk upload, potentially resulting in a partial import. Fix the errors and re-upload the corrected files.

-

An error occurred while processing the BuildingSync xml file. Please ensure that your file meets the required specifications.

- 
{$ step_12_error_message | json $}
+

An error occurred while processing the file. Please ensure that your file meets the required specifications.

+ 
{$ step_12_error_message | json $}
+ +

BuildingSync file(s) failed to validate. No files were imported. Please fix the errors and re-upload the corrected file(s).

+
+ {$ file_issues.file $} +
+
Schema Errors
+
{$ err.path $}: {$ err.message $}
+
+
+
Use Case Errors
+
{$ err $}
+
+
+
Use Case Warnings
+
{$ warn $}
+
+
+

@@ -270,7 +297,7 @@

PM_METER_IMPORT_RESULTS
- +
@@ -292,21 +319,6 @@

PM_METER_IMPORT_RESULTS

- -
-
-
-
-
-

Warnings:

- 
{$ step_14_message | json $}
-
-
- -
-
An error occurred while processing the file. Please ensure that your file meets the required specifications.
@@ -402,14 +414,6 @@

PM_METER_IMPORT_RESULTSCONFIRM_AND_START_MATCHING

-
-
- -
-
- -
-
diff --git a/seed/static/seed/partials/export_buildingsync_modal.html b/seed/static/seed/partials/export_buildingsync_modal.html new file mode 100644 index 0000000000..2f57777b97 --- /dev/null +++ b/seed/static/seed/partials/export_buildingsync_modal.html @@ -0,0 +1,27 @@ +
+

Select Preset for Export

+
+ +
+
Please select the preset used when importing the data, else the exported data may be incorrect.
+
+ Unexpected error downloading file + 
{$ download_error_message | json $}
+
+
+ +
+ +
+
+
+ +
+ + +
diff --git a/seed/static/seed/partials/home.html b/seed/static/seed/partials/home.html index f04ba7191b..ea1ef92091 100644 --- a/seed/static/seed/partials/home.html +++ b/seed/static/seed/partials/home.html @@ -14,7 +14,7 @@

Getting Started

The DOE developed the Standard Energy Efficiency Data (SEED) Platform™ as a free software tool that provides a standardized format for collecting, storing and analyzing building energy performance information about large portfolios. Upload your buildings list to get started.

-

  {$:: 'Upload your buildings list' | translate $}   {$:: 'Getting Started Guide' | translate $}

+

  {$:: 'Upload your buildings list' | translate $}   {$:: 'Getting Started Guide' | translate $}

diff --git a/seed/static/seed/partials/mapping.html b/seed/static/seed/partials/mapping.html index 6a42dd5fa9..37ef1a8c59 100644 --- a/seed/static/seed/partials/mapping.html +++ b/seed/static/seed/partials/mapping.html @@ -83,23 +83,23 @@
- +
- +
-
+
Current Column Mapping Preset: -
Please review SEED column names or the column names of the file being imported. Duplicate values are not allowed in either case.
Please review SEED headers. Empty values are not allowed.
+
Please apply a column mapping preset or save your current mapping as a preset.
@@ -126,8 +127,8 @@ @@ -150,13 +151,13 @@
- {$:: 'Set all fields to' | translate $}: - + {$:: 'Set all fields to' | translate $}: +

Mapped Fields
- - + - + +

{$ error_message | translate $}

- +

{$ error_message | translate $}

- +

{$ error_message | translate $}

diff --git a/seed/templates/seed/_scripts.html b/seed/templates/seed/_scripts.html index 5322664c0a..3b56eaf3c2 100644 --- a/seed/templates/seed/_scripts.html +++ b/seed/templates/seed/_scripts.html @@ -2,6 +2,8 @@ {% compress js %} + + @@ -43,6 +45,7 @@ + diff --git a/seed/tests/test_api.py b/seed/tests/test_api.py index 4d9d0754cc..7f3a154d70 100644 --- a/seed/tests/test_api.py +++ b/seed/tests/test_api.py @@ -50,7 +50,10 @@ def test_get_api_endpoints_utils(self): '/api/v2/gbr_properties', '/api/v2/notes', '/api/v2.1/properties', - '/api/v2.1/scenarios',)): + '/api/v2.1/scenarios', + '/api/v3/labels', + '/api/v3/columns', + '/api/v3/cycles')): self.assertTrue( url.endswith('/'), "Endpoint %s does not end with / as expected" % url diff --git a/seed/tests/test_column_mapping_preset_views.py b/seed/tests/test_column_mapping_preset_views.py index 4c2b0964a8..99dbe64537 100644 --- a/seed/tests/test_column_mapping_preset_views.py +++ b/seed/tests/test_column_mapping_preset_views.py @@ -11,6 +11,7 @@ Column, ColumnMappingPreset, ) +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings class ColumnMappingPresetViewsCore(DataMappingBaseTestCase): @@ -44,8 +45,54 @@ def test_list_preset_endpoint(self): data = loads(response.content)['data'] names = [d['name'] for d in data] + self.assertCountEqual(['Portfolio Manager Defaults', 'BuildingSync v2.0 Defaults', 'test_preset_1'], names) + + def test_list_preset_endpoint_by_type(self): + preset_info = { + "name": 'test_preset_1', + "mappings": [ + {"from_field": "Property Id", "from_units": None, "to_field": "PM Property ID", "to_table_name": "PropertyState"}, + {"from_field": "Property Name", "from_units": None, "to_field": "Property Name", "to_table_name": "PropertyState"}, + ], + } + + self.org.columnmappingpreset_set.create(**preset_info) + + url = reverse('api:v2:column_mapping_presets-list') + '?organization_id=' + str(self.org.id) + '&preset_type=Normal' + + response = self.client.get(url) + self.assertEqual(200, response.status_code, response.content) + + data = loads(response.content)['data'] + names = [d['name'] for d in data] + self.assertCountEqual(['Portfolio Manager Defaults', 'test_preset_1'], names) + def test_list_preset_endpoint_by_multiple_types(self): + preset_info = { + "name": 'test_preset_1', + "mappings": [ + {"from_field": "Property Id", "from_units": None, "to_field": "PM Property ID", "to_table_name": "PropertyState"}, + {"from_field": "Property Name", "from_units": None, "to_field": "Property Name", "to_table_name": "PropertyState"}, + ], + "preset_type": ColumnMappingPreset.BUILDINGSYNC_CUSTOM + } + + self.org.columnmappingpreset_set.create(**preset_info) + + url = (reverse('api:v2:column_mapping_presets-list') + + '?organization_id=' + str(self.org.id) + + '&preset_type=BuildingSync Default' + + '&preset_type=BuildingSync Custom') + + response = self.client.get(url) + self.assertEqual(200, response.status_code, response.content) + + data = loads(response.content)['data'] + names = [d['name'] for d in data] + + self.assertCountEqual(['BuildingSync v2.0 Defaults', 'test_preset_1'], names) + def test_update_preset_endpoint(self): preset_info = { "name": 'test_preset_1', @@ -73,15 +120,6 @@ def test_update_preset_endpoint(self): self.assertEqual('changed_preset_name', datum['name']) self.assertEqual(1, ColumnMappingPreset.objects.filter(name='changed_preset_name').count()) - # Spot check error case - url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) - post_params = dumps({ - "some_wrong_field_name": 'changed_preset_name', - }) - - response = self.client.put(url, post_params, content_type='application/json') - self.assertEqual(400, response.status_code) - def test_create_preset_endpoint(self): url = reverse('api:v2:column_mapping_presets-list') + '?organization_id=' + str(self.org.id) @@ -102,13 +140,13 @@ def test_create_preset_endpoint(self): self.assertEqual(1, ColumnMappingPreset.objects.filter(name='test_preset_1').count()) def test_delete_preset_endpoint(self): - preset = self.org.columnmappingpreset_set.get() + preset = self.org.columnmappingpreset_set.get(preset_type=ColumnMappingPreset.NORMAL) url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) response = self.client.delete(url) self.assertEqual(200, response.status_code) - self.assertFalse(ColumnMappingPreset.objects.exists()) + self.assertFalse(ColumnMappingPreset.objects.filter(preset_type=ColumnMappingPreset.NORMAL).exists()) class ColumnMappingPresetViewsNonCrud(DataMappingBaseTestCase): @@ -157,3 +195,214 @@ def test_get_suggestion_given_raw_column_headers(self): for header in results: self.assertEqual(expected[header], results[header]) + + +class ColumnMappingPresetViewsBuildingSync(DataMappingBaseTestCase): + def setUp(self): + selfvars = self.set_up(ASSESSED_RAW) + self.user, self.org, _import_file, _import_record, _cycle = selfvars + + user_details = { + 'username': 'test_user@demo.com', + 'password': 'test_pass', + 'email': 'test_user@demo.com' + } + self.client.login(**user_details) + + def test_update_default_bsync_preset_fails(self): + preset = self.org.columnmappingpreset_set.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + + url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) + update_vals = { + "name": 'changed_preset_name', + "mappings": [ + {"from_field": "Updated Property Name", "from_units": None, "to_field": "Property Name", "to_table_name": "PropertyState"}, + ], + } + + response = self.client.put(url, dumps(update_vals), content_type='application/json') + self.assertEqual(400, response.status_code) + + preset_after = self.org.columnmappingpreset_set.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + self.assertNotEqual(preset.name, update_vals['name']) + updated_mapping = [m for m in preset_after.mappings if m['from_field'] == 'Updated Property Name'] + self.assertEqual([], updated_mapping) + + def test_delete_default_bsync_preset_fails(self): + preset = self.org.columnmappingpreset_set.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + + url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) + response = self.client.delete(url) + + self.assertEqual(400, response.status_code) + self.assertTrue(ColumnMappingPreset.objects.filter(id=preset.id).exists()) + + def test_delete_custom_bsync_preset_succeeds(self): + preset = self.org.columnmappingpreset_set.create( + name='Custom BSync Preset', + mappings=[], + preset_type=ColumnMappingPreset.BUILDINGSYNC_CUSTOM + ) + + url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) + response = self.client.delete(url) + + self.assertEqual(200, response.status_code) + self.assertFalse(ColumnMappingPreset.objects.filter(id=preset.id).exists()) + + def test_update_custom_bsync_preset_successfully_changes_to_fields(self): + # -- Setup + # create the custom preset + preset_mappings = default_buildingsync_preset_mappings() + preset_name = 'Custom BSync Preset' + preset = self.org.columnmappingpreset_set.create( + name=preset_name, + mappings=preset_mappings, + preset_type=ColumnMappingPreset.BUILDINGSYNC_CUSTOM) + + # -- Act + # change one of the mapping's to_field + url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) + updated_mappings = [mapping.copy() for mapping in preset_mappings] + updated_mappings[0]['to_field'] = 'my_new_to_field' + update_vals = { + 'name': 'New Preset Name', + 'mappings': updated_mappings, + } + + response = self.client.put(url, dumps(update_vals), content_type='application/json') + + # -- Assert + self.assertEqual(200, response.status_code) + + # get the updated preset using the updated name + updated_preset = self.org.columnmappingpreset_set.filter(name='New Preset Name') + self.assertTrue(updated_preset.exists()) + updated_preset = updated_preset[0] + # look for the mapping that was changed + changed_mapping = [m for m in updated_preset.mappings if m['to_field'] == 'my_new_to_field'] + self.assertNotEqual([], changed_mapping) + + def test_update_custom_bsync_preset_successfully_removes_mappings(self): + # -- Setup + # create the custom preset + preset_mappings = default_buildingsync_preset_mappings() + preset_name = 'Custom BSync Preset' + preset = self.org.columnmappingpreset_set.create( + name=preset_name, + mappings=preset_mappings, + preset_type=ColumnMappingPreset.BUILDINGSYNC_CUSTOM) + + # -- Act + # remove one of the mappings and update it + url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) + updated_mappings = [mapping.copy() for mapping in preset_mappings] + removed_mapping = updated_mappings.pop() + update_vals = { + 'name': 'New Preset Name', + 'mappings': updated_mappings, + } + + response = self.client.put(url, dumps(update_vals), content_type='application/json') + + # -- Assert + self.assertEqual(200, response.status_code) + + # get the updated preset using the updated name + updated_preset = self.org.columnmappingpreset_set.filter(name='New Preset Name') + self.assertTrue(updated_preset.exists()) + updated_preset = updated_preset[0] + # look for the mapping that was supposed to be removed (should not be found) + filtered_mappings = [m for m in updated_preset.mappings if m['from_field'] == removed_mapping['from_field']] + self.assertFalse(filtered_mappings) + + def test_update_custom_bsync_preset_does_NOT_change_from_fields(self): + # -- Setup + # create the custom preset + preset_mappings = default_buildingsync_preset_mappings() + preset_name = 'Custom BSync Preset' + preset = self.org.columnmappingpreset_set.create( + name=preset_name, + mappings=preset_mappings, + preset_type=ColumnMappingPreset.BUILDINGSYNC_CUSTOM) + + # -- Act + # change one of the mappings in an acceptable way + url = reverse('api:v2:column_mapping_presets-detail', args=[preset.id]) + '?organization_id=' + str(self.org.id) + updated_mappings = [mapping.copy() for mapping in preset_mappings] + updated_mappings[0]['from_field'] = 'my_new_from_field' + update_vals = { + 'name': 'New Preset Name', + 'mappings': updated_mappings, + } + + response = self.client.put(url, dumps(update_vals), content_type='application/json') + + # -- Assert + self.assertEqual(200, response.status_code) + + updated_preset = self.org.columnmappingpreset_set.filter(name='New Preset Name') + self.assertTrue(updated_preset.exists()) + updated_preset = updated_preset[0] + # try to find a mapping with the new from_field (it should not exist) + changed_mapping = [m for m in updated_preset.mappings if m['from_field'] == 'my_new_from_field'] + self.assertFalse(changed_mapping) + + def test_create_custom_bsync_preset_succeeds(self): + url = reverse('api:v2:column_mapping_presets-list') + '?organization_id=' + str(self.org.id) + + preset_info = dumps({ + "name": 'BSync Preset', + "mappings": [ + { + "from_field": "Property Id", + "from_field_value": "text", + "from_units": None, + "to_field": "PM Property ID", + "to_table_name": "PropertyState" + }, { + "from_field": "Property Name", + "from_field_value": "@ID", + "from_units": None, + "to_field": "Property Name", + "to_table_name": "PropertyState" + } + ], + "preset_type": "BuildingSync Custom" + }) + + response = self.client.post(url, preset_info, content_type='application/json') + self.assertEqual(200, response.status_code, response.content) + + datum = loads(response.content)['data'] + + self.assertEqual('BSync Preset', datum.get('name')) + self.assertEqual(1, ColumnMappingPreset.objects.filter(name='BSync Preset', preset_type=ColumnMappingPreset.BUILDINGSYNC_CUSTOM).count()) + + def test_create_custom_bsync_preset_fails_when_missing_from_field_value(self): + url = reverse('api:v2:column_mapping_presets-list') + '?organization_id=' + str(self.org.id) + + preset_info = dumps({ + "name": 'BSync Preset', + "mappings": [ + { + "from_field": "Property Id", + # NOTE: no from_field_value + "from_units": None, + "to_field": "PM Property ID", + "to_table_name": "PropertyState" + }, { + "from_field": "Property Name", + # NOTE: no from_field_value + "from_units": None, + "to_field": "Property Name", + "to_table_name": "PropertyState" + } + ], + "preset_type": "BuildingSync Custom" + }) + + response = self.client.post(url, preset_info, content_type='application/json') + self.assertEqual(400, response.status_code, response.content) + + self.assertFalse(ColumnMappingPreset.objects.filter(name='BSync Preset').exists()) diff --git a/seed/tests/test_data_quality_rules.py b/seed/tests/test_data_quality_rules.py new file mode 100644 index 0000000000..36e9a1d9cf --- /dev/null +++ b/seed/tests/test_data_quality_rules.py @@ -0,0 +1,263 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import json + +from copy import deepcopy + +from django.urls import reverse + +from seed.models.data_quality import ( + DataQualityCheck, + Rule, +) +from seed.models.models import ASSESSED_RAW + +from seed.tests.util import DataMappingBaseTestCase + + +class RuleViewTests(DataMappingBaseTestCase): + def setUp(self): + selfvars = self.set_up(ASSESSED_RAW) + + self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars + + self.client.login( + username='test_user@demo.com', + password='test_pass', + email='test_user@demo.com' + ) + + def test_valid_data_rule_without_label_does_not_actually_update_or_delete_any_rules(self): + # Start with 3 Rules + dq = DataQualityCheck.retrieve(self.org.id) + dq.remove_all_rules() + base_rule_info = { + 'field': 'address_line_1', + 'table_name': 'PropertyState', + 'enabled': True, + 'data_type': Rule.TYPE_STRING, + 'rule_type': Rule.RULE_TYPE_DEFAULT, + 'condition': Rule.RULE_INCLUDE, + 'required': False, + 'not_null': False, + 'min': None, + 'max': None, + 'text_match': 'Test Rule 1', + 'severity': Rule.SEVERITY_ERROR, + 'units': "", + 'status_label_id': None + } + dq.add_rule(base_rule_info) + + rule_2_info = deepcopy(base_rule_info) + rule_2_info['text_match'] = 'Test Rule 2' + dq.add_rule(rule_2_info) + + rule_3_info = deepcopy(base_rule_info) + rule_3_info['text_match'] = 'Test Rule 3' + dq.add_rule(rule_3_info) + + self.assertEqual(dq.rules.count(), 3) + + property_rules = [base_rule_info, rule_2_info, rule_3_info] + + # Make some adjustments to mimic how data is expected in API endpoint + rule_3_info['severity'] = dict(Rule.SEVERITY).get(Rule.SEVERITY_ERROR) + for rule in property_rules: + rule['data_type'] = dict(Rule.DATA_TYPES).get(rule['data_type']) + rule['label'] = None + + # Make 2 rules trigger the "valid without label" failure + base_rule_info['severity'] = dict(Rule.SEVERITY).get(Rule.SEVERITY_VALID) + rule_2_info['severity'] = dict(Rule.SEVERITY).get(Rule.SEVERITY_VALID) + + url = reverse('api:v2:data_quality_checks-save-data-quality-rules') + '?organization_id=' + str(self.org.pk) + post_data = { + "data_quality_rules": { + "properties": property_rules, + "taxlots": [], + }, + } + res = self.client.post(url, content_type='application/json', data=json.dumps(post_data)) + + self.assertEqual(res.status_code, 400) + self.assertEqual(json.loads(res.content)['message'], 'Label must be assigned when using Valid Data Severity.') + + # Count 3 total rules. None of them were updated + self.assertEqual(dq.rules.count(), 3) + self.assertEqual(dq.rules.filter(severity=Rule.SEVERITY_VALID).count(), 0) + + def test_include_exclude_without_text_match_does_not_actually_update_or_delete_any_rules(self): + # Start with 3 Rules + dq = DataQualityCheck.retrieve(self.org.id) + dq.remove_all_rules() + base_rule_info = { + 'field': 'address_line_1', + 'table_name': 'PropertyState', + 'enabled': True, + 'data_type': Rule.TYPE_STRING, + 'rule_type': Rule.RULE_TYPE_DEFAULT, + 'condition': Rule.RULE_INCLUDE, + 'required': False, + 'not_null': False, + 'min': None, + 'max': None, + 'text_match': 'Test Rule 1', + 'severity': Rule.SEVERITY_ERROR, + 'units': "", + 'status_label_id': None + } + dq.add_rule(base_rule_info) + + rule_2_info = deepcopy(base_rule_info) + rule_2_info['text_match'] = 'Test Rule 2' + dq.add_rule(rule_2_info) + + rule_3_info = deepcopy(base_rule_info) + rule_3_info['text_match'] = 'Test Rule 3' + dq.add_rule(rule_3_info) + + self.assertEqual(dq.rules.count(), 3) + + property_rules = [base_rule_info, rule_2_info, rule_3_info] + + # Make some adjustments to mimic how data is expected in API endpoint + rule_3_info['severity'] = dict(Rule.SEVERITY).get(Rule.SEVERITY_ERROR) + for rule in property_rules: + rule['data_type'] = dict(Rule.DATA_TYPES).get(rule['data_type']) + rule['label'] = None + + # Make 2 rules trigger the include or exclude without text_match failure + base_rule_info['text_match'] = '' + + rule_2_info['condition'] = Rule.RULE_EXCLUDE + rule_2_info['text_match'] = '' + + url = reverse('api:v2:data_quality_checks-save-data-quality-rules') + '?organization_id=' + str(self.org.pk) + post_data = { + "data_quality_rules": { + "properties": property_rules, + "taxlots": [], + }, + } + res = self.client.post(url, content_type='application/json', data=json.dumps(post_data)) + + self.assertEqual(res.status_code, 400) + self.assertEqual(json.loads(res.content)['message'], 'Rule must not include or exclude an empty string.') + + # Count 3 total rules. None of them were updated + self.assertEqual(dq.rules.count(), 3) + self.assertEqual(dq.rules.filter(condition=Rule.RULE_EXCLUDE).count(), 0) + self.assertEqual(dq.rules.filter(text_match='').count(), 0) + + def test_failed_rule_creation_doesnt_prevent_other_rules_from_being_created(self): + # Start with 0 Rules + dq = DataQualityCheck.retrieve(self.org.id) + dq.remove_all_rules() + + # Post 3 rules - one of which will fail + base_rule_post_data = { + 'field': 'address_line_1', + 'table_name': 'PropertyState', + 'enabled': True, + 'data_type': dict(Rule.DATA_TYPES).get(Rule.TYPE_STRING), + 'rule_type': Rule.RULE_TYPE_DEFAULT, + 'condition': Rule.RULE_INCLUDE, + 'required': False, + 'not_null': False, + 'min': None, + 'max': None, + 'text_match': 'Test Rule 1', + 'severity': dict(Rule.SEVERITY).get(Rule.SEVERITY_ERROR), + 'units': "", + 'label': None + } + + rule_2_post_data = deepcopy(base_rule_post_data) + rule_2_post_data['text_match'] = 'Test Rule 2' + rule_2_post_data['rule_type'] = 'some invalid rule type' + + rule_3_post_data = deepcopy(base_rule_post_data) + rule_3_post_data['text_match'] = 'Test Rule 3' + rule_3_post_data['rule_type'] = Rule.RULE_TYPE_DEFAULT + + property_rules = [base_rule_post_data, rule_2_post_data, rule_3_post_data] + + url = reverse('api:v2:data_quality_checks-save-data-quality-rules') + '?organization_id=' + str(self.org.pk) + post_data = { + "data_quality_rules": { + "properties": property_rules, + "taxlots": [], + }, + } + res = self.client.post(url, content_type='application/json', data=json.dumps(post_data)) + + self.assertEqual(res.status_code, 400) + self.assertEqual(json.loads(res.content)['message'], "Rule could not be recreated: invalid literal for int() with base 10: 'some invalid rule type'") + + # Count 2 total rules - the first and second rules + self.assertEqual(dq.rules.count(), 2) + self.assertEqual(dq.rules.filter(text_match__in=['Test Rule 1', 'Test Rule 3']).count(), 2) + + def test_multiple_unique_errors_get_reported(self): + # Start with 3 Rules + dq = DataQualityCheck.retrieve(self.org.id) + dq.remove_all_rules() + base_rule_info = { + 'field': 'address_line_1', + 'table_name': 'PropertyState', + 'enabled': True, + 'data_type': Rule.TYPE_STRING, + 'rule_type': Rule.RULE_TYPE_DEFAULT, + 'condition': Rule.RULE_INCLUDE, + 'required': False, + 'not_null': False, + 'min': None, + 'max': None, + 'text_match': 'Test Rule 1', + 'severity': Rule.SEVERITY_ERROR, + 'units': "", + 'status_label_id': None + } + dq.add_rule(base_rule_info) + + rule_2_info = deepcopy(base_rule_info) + rule_2_info['text_match'] = 'Test Rule 2' + dq.add_rule(rule_2_info) + + rule_3_info = deepcopy(base_rule_info) + rule_3_info['text_match'] = 'Test Rule 3' + dq.add_rule(rule_3_info) + + self.assertEqual(dq.rules.count(), 3) + + property_rules = [base_rule_info, rule_2_info, rule_3_info] + + # Make some adjustments to mimic how data is expected in API endpoint + rule_3_info['severity'] = dict(Rule.SEVERITY).get(Rule.SEVERITY_ERROR) + for rule in property_rules: + rule['data_type'] = dict(Rule.DATA_TYPES).get(rule['data_type']) + rule['label'] = None + + # Make 1 rule trigger the include without text_match failure + base_rule_info['text_match'] = '' + + # Make 1 rule trigger the "valid without label" failure + rule_2_info['severity'] = dict(Rule.SEVERITY).get(Rule.SEVERITY_VALID) + + url = reverse('api:v2:data_quality_checks-save-data-quality-rules') + '?organization_id=' + str(self.org.pk) + post_data = { + "data_quality_rules": { + "properties": property_rules, + "taxlots": [], + }, + } + res = self.client.post(url, content_type='application/json', data=json.dumps(post_data)) + + self.assertEqual(res.status_code, 400) + self.assertTrue('Rule must not include or exclude an empty string.' in json.loads(res.content)['message']) + self.assertTrue('Label must be assigned when using Valid Data Severity.' in json.loads(res.content)['message']) diff --git a/seed/tests/test_property_views.py b/seed/tests/test_property_views.py index c8843cfd56..f06e4794db 100644 --- a/seed/tests/test_property_views.py +++ b/seed/tests/test_property_views.py @@ -27,6 +27,7 @@ ImportRecord, ) from seed.data_importer.tasks import match_buildings +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings from seed.models import ( DATA_STATE_MAPPING, @@ -40,7 +41,8 @@ TaxLotProperty, Column, BuildingFile, - Scenario + Scenario, + ColumnMappingPreset, ) from seed.test_helpers.fake import ( FakeCycleFactory, @@ -1141,3 +1143,90 @@ def test_unmerge_results_in_the_persistence_of_old_canonical_state_if_related_to self.assertTrue(Property.objects.filter(pk=view.property_id).exists()) self.assertEqual(Property.objects.count(), 3) + + +class PropertyViewExportTests(DataMappingBaseTestCase): + def setUp(self): + user_details = { + 'username': 'test_user@demo.com', + 'password': 'test_pass', + 'email': 'test_user@demo.com' + } + self.user = User.objects.create_superuser(**user_details) + self.org, self.org_user, _ = create_organization(self.user) + self.column_factory = FakeColumnFactory(organization=self.org) + self.cycle_factory = FakeCycleFactory(organization=self.org, user=self.user) + self.property_factory = FakePropertyFactory(organization=self.org) + self.property_state_factory = FakePropertyStateFactory(organization=self.org) + self.property_view_factory = FakePropertyViewFactory(organization=self.org) + self.cycle = self.cycle_factory.get_cycle( + start=datetime(2010, 10, 10, tzinfo=get_current_timezone())) + self.column_list_factory = FakeColumnListSettingsFactory(organization=self.org) + self.client.login(**user_details) + + def test_export_bsync_works_with_default_preset(self): + # -- Setup + state = self.property_state_factory.get_property_state() + prprty = self.property_factory.get_property() + view = PropertyView.objects.create( + property=prprty, cycle=self.cycle, state=state + ) + preset = ColumnMappingPreset.objects.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + + # -- Act + url = reverse('api:v2.1:properties-building-sync', args=[view.id]) + response = self.client.get(url, {'preset_id': preset.id}) + + # -- Assert + self.assertEqual(200, response.status_code, response.content) + + def test_export_bsync_works_with_custom_preset(self): + """Tests that using a different column mapping preset from the default + results in a different xml output + """ + # -- Setup + # manually set the lat and long + state = self.property_state_factory.get_property_state() + orig_lat = 5555 + orig_long = 4444 + state.latitude = orig_lat + state.longitude = orig_long + state.save() + + prprty = self.property_factory.get_property() + view = PropertyView.objects.create( + property=prprty, cycle=self.cycle, state=state + ) + + # create a preset mapping where longitude and latitude are swapped + preset_mappings = default_buildingsync_preset_mappings() + for mapping in preset_mappings: + if mapping['to_field'] == 'longitude': + mapping['to_field'] = 'latitude' + elif mapping['to_field'] == 'latitude': + mapping['to_field'] = 'longitude' + + custom_preset_name = 'BSync Custom Preset' + self.org.columnmappingpreset_set.create(name=custom_preset_name, mappings=preset_mappings, preset_type=ColumnMappingPreset.BUILDINGSYNC_CUSTOM) + custom_preset = self.org.columnmappingpreset_set.get(name=custom_preset_name) + + # grab the default preset to export with for comparison + default_preset = self.org.columnmappingpreset_set.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + + # -- Act + url = reverse('api:v2.1:properties-building-sync', args=[view.id]) + default_export_response = self.client.get(url, {'preset_id': default_preset.id}) + url = reverse('api:v2.1:properties-building-sync', args=[view.id]) + custom_export_response = self.client.get(url, {'preset_id': custom_preset.id}) + + # -- Assert + self.assertEqual(200, default_export_response.status_code, default_export_response.content) + self.assertEqual(200, custom_export_response.status_code, custom_export_response.content) + + # check that longitude and latitude were swapped by finding different lines in the results + default_lines = default_export_response.content.decode().split('\n') + custom_lines = custom_export_response.content.decode().split('\n') + diffs = [line.strip() for line in set(default_lines).symmetric_difference(custom_lines)] + expected_diffs = ['5555.0', '4444.0', + '4444.0', '5555.0'] + self.assertCountEqual(expected_diffs, diffs) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 50035dc6d1..309f8ff041 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -2,20 +2,22 @@ # encoding: utf-8 from drf_yasg.inspectors import SwaggerAutoSchema from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema class AutoSchemaHelper(SwaggerAutoSchema): + # overrides the serialization of existing endpoints + overwrite_params = [] # Used to easily build out example values displayed on Swagger page. - body_parameter_formats = { - 'interger_list': openapi.Schema( - type=openapi.TYPE_ARRAY, - items=openapi.Schema(type=openapi.TYPE_INTEGER) - ), - 'string': openapi.Schema(type=openapi.TYPE_STRING) + openapi_types = { + 'string': openapi.TYPE_STRING, + 'boolean': openapi.TYPE_BOOLEAN, + 'integer': openapi.TYPE_INTEGER, } - def base_field(self, name, location_attr, description, required, type): + @staticmethod + def base_field(name, location_attr, description, required, type): """ Created to avoid needing to directly access openapi within ViewSets. Ideally, the cases below will be used instead of this one. @@ -28,16 +30,18 @@ def base_field(self, name, location_attr, description, required, type): type=type ) - def org_id_field(self): + @staticmethod + def query_org_id_field(required=True, description="Organization ID"): return openapi.Parameter( 'organization_id', openapi.IN_QUERY, - description='Organization ID', - required=True, + description=description, + required=required, type=openapi.TYPE_INTEGER ) - def query_integer_field(self, name, required, description): + @staticmethod + def query_integer_field(name, required, description): return openapi.Parameter( name, openapi.IN_QUERY, @@ -46,7 +50,28 @@ def query_integer_field(self, name, required, description): type=openapi.TYPE_INTEGER ) - def path_id_field(self, description): + @staticmethod + def query_string_field(name, required, description): + return openapi.Parameter( + name, + openapi.IN_QUERY, + description=description, + required=required, + type=openapi.TYPE_STRING + ) + + @staticmethod + def query_boolean_field(name, required, description): + return openapi.Parameter( + name, + openapi.IN_QUERY, + description=description, + required=required, + type=openapi.TYPE_BOOLEAN + ) + + @staticmethod + def path_id_field(description): return openapi.Parameter( 'id', openapi.IN_PATH, @@ -55,27 +80,74 @@ def path_id_field(self, description): type=openapi.TYPE_INTEGER ) - def body_field(self, required, description, name='body', params_to_formats={}): + @classmethod + def body_field(cls, required, description, name='body', params_to_formats={}): return openapi.Parameter( name, openapi.IN_BODY, description=description, required=required, - schema=self._build_body_schema(params_to_formats) + schema=cls.schema_factory(params_to_formats) ) - def _build_body_schema(self, params_to_formats): - return openapi.Schema( - type=openapi.TYPE_OBJECT, - properties={ - k: self.body_parameter_formats.get(format_name, "") - for k, format_name - in params_to_formats.items() + @classmethod + def schema_factory(cls, obj, **kwargs): + """Translates an object into an openapi Schema + + This can handle nested objects (lists, dicts), and "types" defined as strings + For example: + { + 'hello': ['string'], + 'world': 'integer', + 'fruits': { + 'orange': 'boolean', + 'apple': 'boolean' } - ) + } + + :param obj: str, list, dict[str, obj] + :return: drf_yasg.openapi.Schema + """ + if type(obj) is str: + if obj not in cls.openapi_types: + raise Exception(f'Invalid type "{obj}"; expected one of {cls.openapi_types.keys()}') + return openapi.Schema( + type=cls.openapi_types[obj], + **kwargs + ) + + if type(obj) is list: + if len(obj) != 1: + raise Exception('List types must have exactly one element to specify the schema of `items`') + return openapi.Schema( + type=openapi.TYPE_ARRAY, + items=cls.schema_factory(obj[0]), + **kwargs + ) + + if type(obj) is dict: + return openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + k: cls.schema_factory(sub_obj) + for k, sub_obj + in obj.items() + }, + **kwargs + ) + + raise Exception(f'Unhandled type "{type(obj)}" for {obj}') def add_manual_parameters(self, parameters): manual_params = self.manual_fields.get((self.method, self.view.action), []) + if (self.method, self.view.action) in self.overwrite_params: + return manual_params # I think this should add to existing parameters, but haven't been able to confirm. return parameters + manual_params + + +# this is a commonly used swagger decorator so moved here for DRYness +swagger_auto_schema_org_query_param = swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()] +) diff --git a/seed/utils/labels.py b/seed/utils/labels.py new file mode 100644 index 0000000000..bbe3bdcd09 --- /dev/null +++ b/seed/utils/labels.py @@ -0,0 +1,60 @@ +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, +through Lawrence Berkeley National Laboratory (subject to receipt of any +required approvals from the U.S. Department of Energy) and contributors. +All rights reserved. # NOQA +:author +""" +import json + +from seed import search +from seed.serializers.labels import ( + LabelSerializer, +) +from rest_framework import ( + response, + status, +) + + +def filter_labels_for_inv_type(request, inventory_type=None): + """ + Method used to filter labels by inventory type and return is_applied inventory id's + Method was initially built as a class defined function above to handle more parameters. + """ + params = request.query_params.dict() + # Since this is being passed in as a query string, the object ends up + # coming through as a string. + params['filter_params'] = json.loads(params.get('filter_params', '{}')) + params = search.process_search_params( + params=params, + user=request.user, + is_api_request=True, + ) + queryset = search.inventory_search_filter_sort( + inventory_type, + params=params, + user=request.user, + ) + if 'selected' in request.data: + # Return labels limited to the 'selected' list. Otherwise, if selected is empty, return all + if request.data['selected']: + return queryset.filter( + id__in=request.data['selected'], + ) + return queryset + + +def get_labels(request, qs, super_organization, inv_type): + inventory = filter_labels_for_inv_type( + request=request, inventory_type=inv_type + ) + results = [ + LabelSerializer( + q, + super_organization=super_organization, + inventory=inventory + ).data for q in qs + ] + status_code = status.HTTP_200_OK + return response.Response(results, status=status_code) diff --git a/seed/utils/nosniff.py b/seed/utils/nosniff.py deleted file mode 100644 index 4762b48d62..0000000000 --- a/seed/utils/nosniff.py +++ /dev/null @@ -1,17 +0,0 @@ -# !/usr/bin/env python -# encoding: utf-8 -""" -:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA -:author -""" - - -class DisableMIMESniffingMiddleware(object): - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - response = self.get_response(request) - - response['X-Content-Type-Options'] = 'nosniff' - return response diff --git a/seed/utils/organizations.py b/seed/utils/organizations.py index a91a8ad6b9..80d4631531 100644 --- a/seed/utils/organizations.py +++ b/seed/utils/organizations.py @@ -13,7 +13,8 @@ OrganizationUser, ROLE_MEMBER ) -from seed.models import Column +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings +from seed.models import Column, ColumnMappingPreset from seed.models.data_quality import DataQualityCheck @@ -137,12 +138,19 @@ def create_organization(user=None, org_name='', *args, **kwargs): # the default columns _create_default_columns(organization.id) - # ... and the default column mapping preset + # ... and the default column mapping preset for Portfolio Manager organization.columnmappingpreset_set.create( name='Portfolio Manager Defaults', mappings=default_pm_mappings() ) + # ... and the default column mapping preset for BuildingSync + organization.columnmappingpreset_set.create( + name='BuildingSync v2.0 Defaults', + mappings=default_buildingsync_preset_mappings(), + preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT + ) + # create the default rules for this organization DataQualityCheck.retrieve(organization.id) diff --git a/seed/utils/viewsets.py b/seed/utils/viewsets.py index 2248a2eeae..0b0a8ea969 100644 --- a/seed/utils/viewsets.py +++ b/seed/utils/viewsets.py @@ -18,6 +18,11 @@ from rest_framework.parsers import FormParser, JSONParser, MultiPartParser from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from oauth2_provider.ext.rest_framework import OAuth2Authentication +from rest_framework.mixins import ( + CreateModelMixin, + DestroyModelMixin, + UpdateModelMixin, +) # Local Imports from seed.authentication import SEEDAuthentication @@ -41,6 +46,16 @@ PERMISSIONS_CLASSES = (SEEDOrgPermissions,) +class UpdateWithoutPatchModelMixin(object): + # Taken from: https://github.com/encode/django-rest-framework/pull/3081#issuecomment-518396378 + # Rebuilds the UpdateModelMixin without the patch action + def update(self, request, *args, **kwargs): + return UpdateModelMixin.update(self, request, *args, **kwargs) + + def perform_update(self, serializer): + return UpdateModelMixin.perform_update(self, serializer) + + class SEEDOrgModelViewSet(DecoratorMixin(drf_api_endpoint), OrgQuerySetMixin, ModelViewSet): """Viewset class customized with SEED standard attributes. @@ -85,3 +100,13 @@ class SEEDOrgCreateUpdateModelViewSet(OrgCreateUpdateMixin, SEEDOrgModelViewSet) and/or perform_update overrides appropriate to the model's needs. """ pass + + +class SEEDOrgNoPatchOrOrgCreateModelViewSet(SEEDOrgReadOnlyModelViewSet, + CreateModelMixin, + DestroyModelMixin, + UpdateWithoutPatchModelMixin): + """Extends SEEDOrgReadOnlyModelViewSet to include update (without patch), + create, and destroy actions. + """ + pass diff --git a/seed/views/column_mapping_presets.py b/seed/views/column_mapping_presets.py index 864332c2fc..270ee27424 100644 --- a/seed/views/column_mapping_presets.py +++ b/seed/views/column_mapping_presets.py @@ -1,7 +1,6 @@ # !/usr/bin/env python # encoding: utf-8 -from django.core.exceptions import FieldDoesNotExist from django.http import JsonResponse from rest_framework.decorators import action @@ -16,7 +15,7 @@ from seed.utils.api import api_endpoint_class from rest_framework.viewsets import ViewSet -from rest_framework.status import HTTP_400_BAD_REQUEST +from rest_framework.status import HTTP_400_BAD_REQUEST, HTTP_200_OK class ColumnMappingPresetViewSet(ViewSet): @@ -32,8 +31,12 @@ def list(self, request): paramType: query """ try: - org_id = request.query_params.get('organization_id', None) - presets = ColumnMappingPreset.objects.filter(organizations__pk=org_id) + preset_types = request.GET.getlist('preset_type') + preset_types = [ColumnMappingPreset.get_preset_type(pt) for pt in preset_types] + filter_params = {'organizations__pk': request.query_params.get('organization_id', None)} + if preset_types: + filter_params['preset_type__in'] = preset_types + presets = ColumnMappingPreset.objects.filter(**filter_params) data = [ColumnMappingPresetSerializer(p).data for p in presets] return JsonResponse({ @@ -67,24 +70,51 @@ def update(self, request, pk=None): paramType: body """ try: - updated = ColumnMappingPreset.objects.filter(pk=pk).update(**request.data) - except FieldDoesNotExist as e: + preset = ColumnMappingPreset.objects.get(pk=pk) + except ColumnMappingPreset.DoesNotExist: return JsonResponse({ 'status': 'error', - 'message': str(e), + 'data': 'No preset with given id' }, status=HTTP_400_BAD_REQUEST) - if updated: - preset = ColumnMappingPreset.objects.get(pk=pk) - status = 'success' - data = ColumnMappingPresetSerializer(preset).data - else: - status = 'error' - data = "Column Mapping Preset not updated." + if preset.preset_type == ColumnMappingPreset.BUILDINGSYNC_DEFAULT: + return JsonResponse({ + 'status': 'error', + 'data': 'Default BuildingSync presets are not editable' + }, status=HTTP_400_BAD_REQUEST) + updated_name, updated_mappings = request.data.get('name'), request.data.get('mappings') + + # update the name + if updated_name is not None: + preset.name = updated_name + + # update the mappings according to the preset type + if updated_mappings is not None: + if preset.preset_type == ColumnMappingPreset.BUILDINGSYNC_CUSTOM: + # only allow these updates to the mappings + # - changing the to_field or from_units + # - removing mappings + original_mappings_dict = {m['from_field']: m.copy() for m in preset.mappings} + final_mappings = [] + for updated_mapping in updated_mappings: + from_field = updated_mapping['from_field'] + original_mapping = original_mappings_dict.get(from_field) + if original_mapping is not None: + original_mapping['to_field'] = updated_mapping['to_field'] + original_mapping['from_units'] = updated_mapping['from_units'] + final_mappings.append(original_mapping) + del original_mappings_dict[from_field] + + preset.mappings = final_mappings + elif updated_mappings: + # indiscriminantly update the mappings + preset.mappings = updated_mappings + + preset.save() return JsonResponse({ - 'status': status, - 'data': data, + 'status': 'success', + 'data': ColumnMappingPresetSerializer(preset).data, }) @api_endpoint_class @@ -109,13 +139,26 @@ def create(self, request, pk=None): """ org_id = request.query_params.get('organization_id', None) try: - org = Organization.objects.get(pk=org_id) - preset = org.columnmappingpreset_set.create(**request.data) + # verify the org exists then validate and create the preset + Organization.objects.get(pk=org_id) + + preset_data = request.data + preset_data['organizations'] = [org_id] + ser_preset = ColumnMappingPresetSerializer(data=preset_data) + if ser_preset.is_valid(): + preset = ser_preset.save() + response_status = 'success' + response_data = ColumnMappingPresetSerializer(preset).data + response_code = HTTP_200_OK + else: + response_status = 'error' + response_data = ser_preset.errors + response_code = HTTP_400_BAD_REQUEST return JsonResponse({ - 'status': 'success', - 'data': ColumnMappingPresetSerializer(preset).data, - }) + 'status': response_status, + 'data': response_data, + }, status=response_code) except Exception as e: return JsonResponse({ 'status': 'error', @@ -134,18 +177,25 @@ def delete(self, request, pk=None): paramType: path """ try: - ColumnMappingPreset.objects.get(pk=pk).delete() - - return JsonResponse({ - 'status': 'success', - 'data': "Successfully deleted", - }) + preset = ColumnMappingPreset.objects.get(pk=pk) except Exception as e: return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST) + if preset.preset_type == ColumnMappingPreset.BUILDINGSYNC_DEFAULT: + return JsonResponse({ + 'status': 'error', + 'data': 'Not allowed to edit default BuildingSync presets' + }, status=HTTP_400_BAD_REQUEST) + else: + preset.delete() + return JsonResponse({ + 'status': 'success', + 'data': "Successfully deleted", + }) + @api_endpoint_class @has_perm_class('requires_member') @action(detail=False, methods=['POST']) diff --git a/seed/views/columns.py b/seed/views/columns.py index c4b27c6e44..20315c273f 100644 --- a/seed/views/columns.py +++ b/seed/views/columns.py @@ -4,7 +4,6 @@ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author """ - import logging import coreapi @@ -16,7 +15,6 @@ from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer from rest_framework.response import Response - from seed.decorators import ajax_request_class from seed.lib.superperms.orgs.decorators import has_perm_class from seed.lib.superperms.orgs.models import Organization diff --git a/seed/views/data_quality.py b/seed/views/data_quality.py index 8da6657385..23f605eba3 100644 --- a/seed/views/data_quality.py +++ b/seed/views/data_quality.py @@ -8,6 +8,7 @@ import csv from celery.utils.log import get_task_logger +from django.db import transaction from django.http import JsonResponse, HttpResponse from rest_framework import viewsets, serializers, status from rest_framework.decorators import action @@ -365,7 +366,16 @@ def save_data_quality_rules(self, request, pk=None): posted_rules = body['data_quality_rules'] updated_rules = [] + valid_rules = True + validation_messages = set() for rule in posted_rules['properties']: + if _get_severity_from_js(rule['severity']) == Rule.SEVERITY_VALID and rule['label'] is None: + valid_rules = False + validation_messages.add('Label must be assigned when using Valid Data Severity.') + if rule['condition'] == Rule.RULE_INCLUDE or rule['condition'] == Rule.RULE_EXCLUDE: + if rule['text_match'] is None or rule['text_match'] == '': + valid_rules = False + validation_messages.add('Rule must not include or exclude an empty string.') updated_rules.append( { 'field': rule['field'], @@ -386,6 +396,13 @@ def save_data_quality_rules(self, request, pk=None): ) for rule in posted_rules['taxlots']: + if _get_severity_from_js(rule['severity']) == Rule.SEVERITY_VALID and rule['label'] is None: + valid_rules = False + validation_messages.add('Label must be assigned when using Valid Data Severity.') + if rule['condition'] == Rule.RULE_INCLUDE or rule['condition'] == Rule.RULE_EXCLUDE: + if rule['text_match'] is None or rule['text_match'] == '': + valid_rules = False + validation_messages.add('Rule must not include or exclude an empty string.') updated_rules.append( { 'field': rule['field'], @@ -405,29 +422,33 @@ def save_data_quality_rules(self, request, pk=None): } ) + if valid_rules is False: + return JsonResponse({ + 'status': 'error', + 'message': '\n'.join(validation_messages), + }, status=status.HTTP_400_BAD_REQUEST) + + # This pattern of deleting and recreating Rules is slated to be deprecated + bad_rule_creation = False + error_messages = set() dq = DataQualityCheck.retrieve(organization.id) dq.remove_all_rules() for rule in updated_rules: - if rule['severity'] == Rule.SEVERITY_VALID and rule['status_label_id'] is None: - return JsonResponse({ - 'status': 'error', - 'message': 'Label must be assigned when using Valid Data Severity.', - }, status=status.HTTP_400_BAD_REQUEST) - if rule['condition'] == Rule.RULE_INCLUDE or rule['condition'] == Rule.RULE_EXCLUDE: - if rule['text_match'] is None or rule['text_match'] == '': - return JsonResponse({ - 'status': 'error', - 'message': 'Rule must not include or exclude an empty string.', - }, status=status.HTTP_400_BAD_REQUEST) - try: - dq.add_rule(rule) - except TypeError as e: - return JsonResponse({ - 'status': 'error', - 'message': e, - }, status=status.HTTP_400_BAD_REQUEST) - - return self.data_quality_rules(request) + with transaction.atomic(): + try: + dq.add_rule(rule) + except Exception as e: + error_messages.add('Rule could not be recreated: ' + str(e)) + bad_rule_creation = True + continue + + if bad_rule_creation: + return JsonResponse({ + 'status': 'error', + 'message': '\n'.join(error_messages), + }, status=status.HTTP_400_BAD_REQUEST) + else: + return self.data_quality_rules(request) @api_endpoint_class @ajax_request_class diff --git a/seed/views/labels.py b/seed/views/labels.py index 9b6870fcb7..70151a3ee0 100644 --- a/seed/views/labels.py +++ b/seed/views/labels.py @@ -118,6 +118,21 @@ def list(self, request): class UpdateInventoryLabelsAPIView(APIView): + """API endpoint for viewing and creating labels. + + Returns:: + [ + { + 'id': Label's primary key + 'name': Name given to label + 'color': Color of label, + 'organization_id': Id of organization label belongs to, + 'is_applied': Will be empty array if not applied to property/taxlots + } + ] + + --- + """ renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} diff --git a/seed/views/properties.py b/seed/views/properties.py index e4d6597fd6..281253f3aa 100644 --- a/seed/views/properties.py +++ b/seed/views/properties.py @@ -199,7 +199,7 @@ class PropertyViewViewSet(SEEDOrgModelViewSet): filter_class = PropertyViewFilterSet orgfilter = 'property__organization_id' data_name = "property_views" - queryset = PropertyView.objects.all().select_related('state') + queryset = PropertyView.objects.all() class PropertyViewSet(GenericViewSet, ProfileIdMixin): diff --git a/seed/views/v3/columns.py b/seed/views/v3/columns.py new file mode 100644 index 0000000000..d41f9e22f7 --- /dev/null +++ b/seed/views/v3/columns.py @@ -0,0 +1,161 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import logging +from django.http import JsonResponse +from django.utils.decorators import method_decorator +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework.parsers import JSONParser, FormParser +from rest_framework.renderers import JSONRenderer +from seed.decorators import ajax_request_class +from seed.lib.superperms.orgs.decorators import has_perm_class +from rest_framework.decorators import action +from seed.models.columns import Column +from seed.serializers.columns import ColumnSerializer +from seed.utils.api import OrgValidateMixin, OrgCreateUpdateMixin +from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param + +_log = logging.getLogger(__name__) + + +@method_decorator( + name='create', + decorator=swagger_auto_schema_org_query_param +) +@method_decorator( + name='update', + decorator=swagger_auto_schema_org_query_param +) +@method_decorator( + name='destroy', + decorator=swagger_auto_schema_org_query_param +) +class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet, OrgCreateUpdateMixin): + """ + create: + Create a new Column within a specified org or user's currently activated org. + update: + Update a column and modify which dataset it belongs to. + delete: + Deletes a single column. + """ + raise_exception = True + serializer_class = ColumnSerializer + renderer_classes = (JSONRenderer,) + model = Column + pagination_class = None + parser_classes = (JSONParser, FormParser) + + def get_queryset(self): + # check if the request is properties or taxlots + org_id = self.get_organization(self.request) + return Column.objects.filter(organization_id=org_id) + + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_org_id_field(required=False), + AutoSchemaHelper.query_string_field( + name='inventory_type', + required=False, + description='Which inventory type is being matched (for related fields and naming)' + '\nDefault: "property"' + ), + AutoSchemaHelper.query_boolean_field( + name='used_only', + required=False, + description='Determine whether or not to show only the used fields ' + '(i.e. only columns that have been mapped)' + '\nDefault: "false"' + ), + ], + ) + @ajax_request_class + def list(self, request): + """ + Retrieves all columns for the user's organization including the raw database columns. Will + return all the columns across both the Property and Tax Lot tables. The related field will + be true if the column came from the other table that is not the 'inventory_type' (which + defaults to Property) + """ + organization_id = self.get_organization(self.request) + inventory_type = request.query_params.get('inventory_type', 'property') + only_used = request.query_params.get('only_used', False) + columns = Column.retrieve_all(organization_id, inventory_type, only_used) + return JsonResponse({ + 'status': 'success', + 'columns': columns, + }) + + @swagger_auto_schema_org_query_param + @ajax_request_class + def retrieve(self, request, pk=None): + """ + This API endpoint retrieves a Column + """ + organization_id = self.get_organization(self.request) + # check if column exists for the organization + try: + c = Column.objects.get(pk=pk) + except Column.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'column with id {} does not exist'.format(pk) + }, status=status.HTTP_404_NOT_FOUND) + + if c.organization.id != organization_id: + return JsonResponse({ + 'status': 'error', + 'message': 'Organization ID mismatch between column and organization' + }, status=status.HTTP_400_BAD_REQUEST) + + return JsonResponse({ + 'status': 'success', + 'column': ColumnSerializer(c).data + }) + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'new_column_name': 'string', + 'overwrite': 'boolean' + }) + ) + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def rename(self, request, pk=None): + """ + This API endpoint renames a Column + """ + org_id = self.get_organization(request) + try: + column = Column.objects.get(id=pk, organization_id=org_id) + except Column.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': 'Cannot find column in org=%s with pk=%s' % (org_id, pk) + }, status=status.HTTP_404_NOT_FOUND) + + new_column_name = request.data.get('new_column_name', None) + overwrite = request.data.get('overwrite', False) + if not new_column_name: + return JsonResponse({ + 'success': False, + 'message': 'You must specify the name of the new column as "new_column_name"' + }, status=status.HTTP_400_BAD_REQUEST) + + result = column.rename_column(new_column_name, overwrite) + if not result[0]: + return JsonResponse({ + 'success': False, + 'message': 'Unable to rename column with message: "%s"' % result[1] + }, status=status.HTTP_400_BAD_REQUEST) + else: + return JsonResponse({ + 'success': True, + 'message': result[1] + }) diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py new file mode 100644 index 0000000000..c25f965c7a --- /dev/null +++ b/seed/views/v3/cycles.py @@ -0,0 +1,86 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, +through Lawrence Berkeley National Laboratory (subject to receipt of any +required approvals from the U.S. Department of Energy) and contributors. +All rights reserved. # NOQA +:authors Paul Munday Fable Turas +""" +from django.utils.decorators import method_decorator + +from seed.models import Cycle + +from seed.serializers.cycles import CycleSerializer +from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet +from seed.utils.api_schema import swagger_auto_schema_org_query_param + + +@method_decorator( + name='list', + decorator=swagger_auto_schema_org_query_param) +@method_decorator( + name='create', + decorator=swagger_auto_schema_org_query_param) +@method_decorator( + name='retrieve', + decorator=swagger_auto_schema_org_query_param) +@method_decorator( + name='update', + decorator=swagger_auto_schema_org_query_param) +@method_decorator( + name='destroy', + decorator=swagger_auto_schema_org_query_param) +class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): + """API endpoint for viewing and creating cycles (time periods). + + Returns:: + { + 'status': 'success', + 'cycles': [ + { + 'id': Cycle`s primary key, + 'name': Name given to cycle, + 'start': Start date of cycle, + 'end': End date of cycle, + 'created': Created date of cycle, + 'properties_count': Count of properties in cycle, + 'taxlots_count': Count of tax lots in cycle, + 'organization': Id of organization cycle belongs to, + 'user': Id of user who created cycle + } + ] + } + + + retrieve: + Return a cycle instance by pk if it is within user`s specified org. + + list: + + Return all cycles available to user through user`s specified org. + + create: + Create a new cycle within user`s specified org. + + delete: + Remove an existing cycle. + + update: + Update a cycle record. + + """ + serializer_class = CycleSerializer + pagination_class = None + model = Cycle + data_name = 'cycles' + + def get_queryset(self): + org_id = self.get_organization(self.request) + # Order cycles by name because if the user hasn't specified then the front end WILL default to the first + return Cycle.objects.filter(organization_id=org_id).order_by('name') + + def perform_create(self, serializer): + org_id = self.get_organization(self.request) + user = self.request.user + serializer.save(organization_id=org_id, user=user) diff --git a/seed/views/v3/data_quality.py b/seed/views/v3/data_quality.py index 02a45a39c0..7912215943 100644 --- a/seed/views/v3/data_quality.py +++ b/seed/views/v3/data_quality.py @@ -9,6 +9,7 @@ from celery.utils.log import get_task_logger from django.http import JsonResponse, HttpResponse +from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets, serializers, status from rest_framework.decorators import action from unidecode import unidecode @@ -30,6 +31,7 @@ logger = get_task_logger(__name__) +# TODO: Consider moving these serializers into seed/serializers (and maybe actually use them...) class RulesSubSerializer(serializers.Serializer): field = serializers.CharField(max_length=100) severity = serializers.CharField(max_length=100) @@ -55,6 +57,29 @@ class RulesSerializer(serializers.Serializer): data_quality_rules = RulesIntermediateSerializer() +class RulePayloadSerializer(serializers.ModelSerializer): + # currently our requests put this in a different field + label = serializers.IntegerField(source='status_label') + + class Meta: + model = Rule + exclude = ['id', 'status_label'] + + +class DataQualityRulesSerializer(serializers.Serializer): + properties = serializers.ListField(child=RulePayloadSerializer()) + taxlots = serializers.ListField(child=RulePayloadSerializer()) + + +class SaveDataQualityRulesPayloadSerializer(serializers.Serializer): + data_quality_rules = DataQualityRulesSerializer() + + +class DataQualityRulesResponseSerializer(serializers.Serializer): + status = serializers.CharField() + rules = DataQualityRulesSerializer() + + def _get_js_rule_type(data_type): """return the JS friendly data type name for the data data_quality rule @@ -93,79 +118,40 @@ def _get_severity_from_js(severity): return d.get(severity) -class DataQualitySchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('POST', 'create'): [ - self.org_id_field(), - self.body_field( - name='data_quality_ids', - required=True, - description="An object containing IDs of the records to perform data quality checks on. Should contain two keys- property_state_ids and taxlot_state_ids, each of which is an array of appropriate IDs.", - params_to_formats={ - 'property_state_ids': 'interger_list', - 'taxlot_state_ids': 'interger_list' - } - ), - ], - ('GET', 'data_quality_rules'): [self.org_id_field()], - ('PUT', 'reset_all_data_quality_rules'): [self.org_id_field()], - ('PUT', 'reset_default_data_quality_rules'): [self.org_id_field()], - ('POST', 'save_data_quality_rules'): [ - self.org_id_field(), - self.body_field( - name='data_quality_rules', - required=True, - description="Rules information" - ) - ], - ('GET', 'results'): [ - self.org_id_field(), - self.query_integer_field( - name='data_quality_id', - required=True, - description="Task ID created when DataQuality task is created." - ), - ], - ('GET', 'csv'): [ - # This will replace the auto-generated field - adds description. - self.path_id_field(description="Import file ID or cache key") - ], - } - - class DataQualityViews(viewsets.ViewSet): """ Handles Data Quality API operations within Inventory backend. (1) Post, wait, get… (2) Respond with what changed """ - swagger_schema = DataQualitySchema + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_org_id_field(), + ], + request_body=AutoSchemaHelper.schema_factory( + { + 'property_state_ids': ['integer'], + 'taxlot_state_ids': ['integer'], + }, + description='An object containing IDs of the records to perform' + ' data quality checks on. Should contain two keys- ' + 'property_state_ids and taxlot_state_ids, each of ' + 'which is an array of appropriate IDs.', + ), + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'num_properties': 'integer', + 'num_taxlots': 'integer', + 'progress_key': 'string', + 'progress': {}, + }) + } + ) def create(self, request): """ This API endpoint will create a new cleansing operation process in the background, on potentially a subset of properties/taxlots, and return back a query key - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - - name: data_quality_ids - description: An object containing IDs of the records to perform data quality checks on. - Should contain two keys- property_state_ids and taxlot_state_ids, each of which is an array - of appropriate IDs. - required: true - paramType: body - type: - status: - type: string - description: success or error - required: true """ # step 0: retrieving the data body = request.data @@ -187,6 +173,11 @@ def create(self, request): 'progress': return_value, }) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.path_id_field(description="Import file ID or cache key") + ] + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -194,13 +185,6 @@ def create(self, request): def csv(self, request, pk): """ Download a csv of the data quality checks by the pk which is the cache_key - --- - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID or cache key - required: true - paramType: path """ data_quality_results = get_cache_raw(DataQualityCheck.cache_key(pk)) response = HttpResponse(content_type='text/csv') @@ -233,6 +217,12 @@ def csv(self, request, pk): return response + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -240,22 +230,6 @@ def csv(self, request, pk): def data_quality_rules(self, request): """ Returns the data_quality rules for an org. - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - type: - status: - type: string - required: true - description: success or error - rules: - type: object - required: true - description: An object containing 'properties' and 'taxlots' arrays of rules """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -288,6 +262,12 @@ def data_quality_rules(self, request): return JsonResponse(result) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -295,30 +275,6 @@ def data_quality_rules(self, request): def reset_all_data_quality_rules(self, request): """ Resets an organization's data data_quality rules - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - type: - status: - type: string - description: success or error - required: true - in_range_checking: - type: array[string] - required: true - description: An array of in-range error rules - missing_matching_field: - type: array[string] - required: true - description: An array of fields to verify existence - missing_values: - type: array[string] - required: true - description: An array of fields to ignore missing values """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -326,6 +282,12 @@ def reset_all_data_quality_rules(self, request): dq.reset_all_rules() return self.data_quality_rules(request) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -333,30 +295,6 @@ def reset_all_data_quality_rules(self, request): def reset_default_data_quality_rules(self, request): """ Resets an organization's data data_quality rules - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - type: - status: - type: string - description: success or error - required: true - in_range_checking: - type: array[string] - required: true - description: An array of in-range error rules - missing_matching_field: - type: array[string] - required: true - description: An array of fields to verify existence - missing_values: - type: array[string] - required: true - description: An array of fields to ignore missing values """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -364,6 +302,13 @@ def reset_default_data_quality_rules(self, request): dq.reset_default_rules() return self.data_quality_rules(request) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=SaveDataQualityRulesPayloadSerializer, + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -373,28 +318,6 @@ def save_data_quality_rules(self, request, pk=None): Saves an organization's settings: name, query threshold, shared fields. The method passes in all the fields again, so it is okay to remove all the rules in the db, and just recreate them (albeit inefficient) - --- - parameter_strategy: replace - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - - name: body - description: JSON body containing organization rules information - paramType: body - pytype: RulesSerializer - required: true - type: - status: - type: string - description: success or error - required: true - message: - type: string - description: error message, if any - required: true """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -464,6 +387,16 @@ def save_data_quality_rules(self, request, pk=None): return self.data_quality_rules(request) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_org_id_field(), + AutoSchemaHelper.query_integer_field( + name='data_quality_id', + required=True, + description="Task ID created when DataQuality task is created." + ), + ] + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') diff --git a/seed/views/v3/datasets.py b/seed/views/v3/datasets.py index eb4cfd9780..306250c285 100644 --- a/seed/views/v3/datasets.py +++ b/seed/views/v3/datasets.py @@ -8,6 +8,7 @@ from django.http import JsonResponse from django.utils import timezone +from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework import viewsets from rest_framework.decorators import action @@ -20,75 +21,21 @@ from seed.utils.api import api_endpoint_class from seed.utils.time import convert_to_js_timestamp -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param _log = logging.getLogger(__name__) -class DatasetSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('GET', 'list'): [self.org_id_field()], - ('POST', 'create'): [ - self.org_id_field(), - self.body_field( - name='name', - required=True, - description="The name of the dataset to be created", - params_to_formats={ - 'name': 'string' - } - ), - ], - ('GET', 'count'): [self.org_id_field()], - ('GET', 'retrieve'): [ - self.org_id_field()], - ('PUT', 'update'): [ - self.org_id_field(), - self.body_field( - name='dataset', - required=True, - description="The new name for this dataset", - params_to_formats={ - 'dataset': 'string' - } - ) - ], - ('DELETE', 'destroy'): [self.org_id_field()] - - } - - class DatasetViewSet(viewsets.ViewSet): raise_exception = True - swagger_schema = DatasetSchema - + @swagger_auto_schema_org_query_param @require_organization_id_class @api_endpoint_class @ajax_request_class def list(self, request): """ Retrieves all datasets for the user's organization. - --- - type: - status: - required: true - type: string - description: Either success or error - datasets: - required: true - type: array[dataset] - description: Returns an array where each item is a full dataset structure, including - keys ''name'', ''number_of_buildings'', ''id'', ''updated_at'', - ''last_modified_by'', ''importfiles'', ... - parameters: - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query """ org_id = request.query_params.get('organization_id', None) @@ -108,35 +55,19 @@ def list(self, request): 'datasets': datasets, }) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + {'dataset': 'string'}, + description='The new name for this dataset' + ) + ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def update(self, request, pk=None): """ Updates the name of a dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: either success or error - message: - type: string - description: error message, if any - parameter_strategy: replace - parameters: - - name: organization_id - description: "The organization_id" - required: true - paramType: query - - name: dataset - description: "The new name for this dataset" - required: true - paramType: string - - name: pk - description: "Primary Key" - required: true - paramType: path """ organization_id = request.query_params.get('organization_id', None) @@ -167,33 +98,12 @@ def update(self, request, pk=None): 'status': 'success', }) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): """ Retrieves a dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: Either success or error - dataset: - required: true - type: dictionary - description: A dictionary of a full dataset structure, including - keys ''name'', ''id'', ''updated_at'', - ''last_modified_by'', ''importfiles'', ... - parameter_strategy: replace - parameters: - - name: pk - description: The ID of the dataset to retrieve - required: true - paramType: path - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query """ organization_id = request.query_params.get('organization_id', None) @@ -251,31 +161,13 @@ def retrieve(self, request, pk=None): 'dataset': dataset, }) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def destroy(self, request, pk=None): """ Deletes a dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: either success or error - message: - type: string - description: error message, if any - parameter_strategy: replace - parameters: - - name: organization_id - description: The organization_id - required: true - paramType: query - - name: pk - description: "Primary Key" - required: true - paramType: path """ organization_id = int(request.query_params.get('organization_id', None)) dataset_id = pk @@ -292,6 +184,19 @@ def destroy(self, request, pk=None): d.delete() return JsonResponse({'status': 'success'}) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + {'name': 'string'}, + description='Name of the dataset to be created' + ), + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'id': 'integer', + 'name': 'string', + }) + } + ) @require_organization_id_class @api_endpoint_class @ajax_request_class @@ -299,29 +204,6 @@ def destroy(self, request, pk=None): def create(self, request): """ Creates a new empty dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: either success or error - id: - required: true - type: integer - description: primary key for the newly created dataset - name: - required: true - type: string - description: name of the newly created dataset - parameters: - - name: name - description: The name of this dataset - required: true - paramType: string - - name: organization_id - description: The organization_id - required: true - paramType: query """ body = request.data @@ -344,6 +226,14 @@ def create(self, request): return JsonResponse({'status': 'success', 'id': record.pk, 'name': record.name}) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'datasets_count': 'integer', + }) + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @@ -352,21 +242,6 @@ def create(self, request): def count(self, request): """ Retrieves the number of datasets for an org. - --- - parameters: - - name: organization_id - description: The organization_id - required: true - paramType: query - type: - status: - description: success or error - type: string - required: true - datasets_count: - description: Number of datasets belonging to this org - type: integer - required: true """ org_id = int(request.query_params.get('organization_id', None)) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py new file mode 100644 index 0000000000..6a08948ac4 --- /dev/null +++ b/seed/views/v3/import_files.py @@ -0,0 +1,808 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import logging + +from django.core.exceptions import ObjectDoesNotExist +from django.http import JsonResponse +from drf_yasg.utils import swagger_auto_schema +from rest_framework import serializers, status, viewsets +from rest_framework.decorators import ( + action, + permission_classes, +) + +from seed.data_importer.models import ( + ImportFile, + ImportRecord +) +from seed.data_importer.models import ROW_DELIMITER +from seed.data_importer.tasks import do_checks +from seed.data_importer.tasks import ( + map_data, + geocode_buildings_task as task_geocode_buildings, + map_additional_models as task_map_additional_models, + match_buildings as task_match_buildings, + save_raw_data as task_save_raw, + validate_use_cases as task_validate_use_cases, +) +from seed.decorators import ajax_request_class +from seed.lib.mappings import mapper as simple_mapper +from seed.lib.mcm import mapper +from seed.lib.xml_mapping import mapper as xml_mapper +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import ( + Organization, +) +from seed.lib.superperms.orgs.models import OrganizationUser +from seed.lib.superperms.orgs.permissions import SEEDOrgPermissions +from seed.models import ( + get_column_mapping, +) +from seed.models import ( + obj_to_dict, + PropertyState, + TaxLotState, + DATA_STATE_MAPPING, + DATA_STATE_MATCHING, + MERGE_STATE_UNKNOWN, + MERGE_STATE_NEW, + MERGE_STATE_MERGED, + Cycle, + Column, + PropertyAuditLog, + TaxLotAuditLog, + AUDIT_USER_EDIT, + TaxLotProperty, +) +from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param +from seed.utils.geocode import MapQuestAPIKeyError + +_log = logging.getLogger(__name__) + + +class MappingResultsPropertySerializer(serializers.Serializer): + pm_property_id = serializers.CharField(max_length=100) + address_line_1 = serializers.CharField(max_length=100) + property_name = serializers.CharField(max_length=100) + + +class MappingResultsTaxLotSerializer(serializers.Serializer): + pm_property_id = serializers.CharField(max_length=100) + address_line_1 = serializers.CharField(max_length=100) + jurisdiction_tax_lot_id = serializers.CharField(max_length=100) + + +class MappingResultsResponseSerializer(serializers.Serializer): + status = serializers.CharField(max_length=100) + properties = MappingResultsPropertySerializer(many=True) + tax_lots = MappingResultsTaxLotSerializer(many=True) + + +def convert_first_five_rows_to_list(header, first_five_rows): + """ + Return the first five rows. This is a complicated method because it handles converting the + persisted format of the first five rows into a list of dictionaries. It handles some basic + logic if there are crlf in the fields. Note that this method does not cover all the use cases + and cannot due to the custom delimeter. See the tests in + test_views.py:test_get_first_five_rows_newline_should_work to see the limitation + + :param header: list, ordered list of headers as strings + :param first_five_rows: string, long string with |#*#| delimeter. + :return: list + """ + row_data = [] + rows = [] + number_of_columns = len(header) + split_cells = first_five_rows.split(ROW_DELIMITER) + number_cells = len(split_cells) + # catch the case where there is only one column, therefore no ROW_DELIMITERs + if number_of_columns == 1: + # Note that this does not support having a single column with carriage returns! + rows = first_five_rows.splitlines() + else: + for idx, l in enumerate(split_cells): + crlf_count = l.count('\n') + + if crlf_count == 0: + row_data.append(l) + elif crlf_count >= 1: + # if add this element to row_data equals number_of_columns, then it is a new row + if len(row_data) == number_of_columns - 1: + # check if this is the last columns, if so, then just store the value and move on + if idx == number_cells - 1: + row_data.append(l) + rows.append(row_data) + continue + else: + # split the cell_data. The last cell becomes the beginning of the new + # row, and the other cells stay joined with \n. + cell_data = l.splitlines() + row_data.append('\n'.join(cell_data[:crlf_count])) + rows.append(row_data) + + # initialize the next row_data with the remainder + row_data = [cell_data[-1]] + continue + else: + # this is not the end, so it must be a carriage return in the cell, just save data + row_data.append(l) + + if len(row_data) == number_of_columns: + rows.append(row_data) + + return [dict(zip(header, row)) for row in rows] + + +class ImportFileViewSet(viewsets.ViewSet): + raise_exception = True + queryset = ImportFile.objects.all() + + @api_endpoint_class + @ajax_request_class + def retrieve(self, request, pk=None): + """ + Retrieves details about an ImportFile. + """ + import_file_id = pk + orgs = request.user.orgs.all() + try: + import_file = ImportFile.objects.get( + pk=import_file_id + ) + d = ImportRecord.objects.filter( + super_organization__in=orgs, pk=import_file.import_record_id + ) + except ObjectDoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'Could not access an import file with this ID' + }, status=status.HTTP_403_FORBIDDEN) + # check if user has access to the import file + if not d.exists(): + return JsonResponse({ + 'status': 'error', + 'message': 'Could not locate import file with this ID', + 'import_file': {}, + }, status=status.HTTP_400_BAD_REQUEST) + + f = obj_to_dict(import_file) + f['name'] = import_file.filename_only + if not import_file.uploaded_filename: + f['uploaded_filename'] = import_file.filename + f['dataset'] = obj_to_dict(import_file.import_record) + + return JsonResponse({ + 'status': 'success', + 'import_file': f, + }) + + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['GET']) + def first_five_rows(self, request, pk=None): + """ + Retrieves the first five rows of an ImportFile. + """ + import_file = ImportFile.objects.get(pk=pk) + if import_file is None: + return JsonResponse( + {'status': 'error', 'message': 'Could not find import file with pk=' + str( + pk)}, status=status.HTTP_400_BAD_REQUEST) + if import_file.cached_second_to_fifth_row is None: + return JsonResponse({'status': 'error', + 'message': 'Internal problem occurred, import file first five rows not cached'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + ''' + import_file.cached_second_to_fifth_row is a field that contains the first + 5 lines of data from the file, split on newlines, delimited by + ROW_DELIMITER. This becomes an issue when fields have newlines in them, + so the following is to handle newlines in the fields. + In the case of only one data column there will be no ROW_DELIMITER. + ''' + header = import_file.cached_first_row.split(ROW_DELIMITER) + data = import_file.cached_second_to_fifth_row + return JsonResponse({ + 'status': 'success', + 'first_five_rows': convert_first_five_rows_to_list(header, data) + }) + + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['GET']) + def raw_column_names(self, request, pk=None): + """ + Retrieves a list of all column names from an ImportFile. + """ + import_file = ImportFile.objects.get(pk=pk) + return JsonResponse({ + 'status': 'success', + 'raw_columns': import_file.first_row_columns + }) + + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_org_id_field(), + ], + responses={ + 200: MappingResultsResponseSerializer + } + ) + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['POST'], url_path='mapping_results') + def mapping_results(self, request, pk=None): + """ + Retrieves a paginated list of Properties and Tax Lots for an import file after mapping. + """ + import_file_id = pk + org_id = request.query_params.get('organization_id', False) + + # get the field names that were in the mapping + import_file = ImportFile.objects.get(id=import_file_id) + + # List of the only fields to show + field_names = import_file.get_cached_mapped_columns + + # set of fields + fields = { + 'PropertyState': ['id', 'extra_data', 'lot_number'], + 'TaxLotState': ['id', 'extra_data'] + } + columns_from_db = Column.retrieve_all(org_id) + property_column_name_mapping = {} + taxlot_column_name_mapping = {} + for field_name in field_names: + # find the field from the columns in the database + for column in columns_from_db: + if column['table_name'] == 'PropertyState' and \ + field_name[0] == 'PropertyState' and \ + field_name[1] == column['column_name']: + property_column_name_mapping[column['column_name']] = column['name'] + if not column['is_extra_data']: + fields['PropertyState'].append(field_name[1]) # save to the raw db fields + continue + elif column['table_name'] == 'TaxLotState' and \ + field_name[0] == 'TaxLotState' and \ + field_name[1] == column['column_name']: + taxlot_column_name_mapping[column['column_name']] = column['name'] + if not column['is_extra_data']: + fields['TaxLotState'].append(field_name[1]) # save to the raw db fields + continue + + inventory_type = request.data.get('inventory_type', 'all') + + result = { + 'status': 'success' + } + + if inventory_type == 'properties' or inventory_type == 'all': + properties = PropertyState.objects.filter( + import_file_id=import_file_id, + data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], + merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW] + ).only(*fields['PropertyState']).order_by('id') + + property_results = [] + for prop in properties: + prop_dict = TaxLotProperty.model_to_dict_with_mapping( + prop, + property_column_name_mapping, + fields=fields['PropertyState'], + exclude=['extra_data'] + ) + + prop_dict.update( + TaxLotProperty.extra_data_to_dict_with_mapping( + prop.extra_data, + property_column_name_mapping, + fields=prop.extra_data.keys(), + ).items() + ) + property_results.append(prop_dict) + + result['properties'] = property_results + + if inventory_type == 'taxlots' or inventory_type == 'all': + tax_lots = TaxLotState.objects.filter( + import_file_id=import_file_id, + data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], + merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW] + ).only(*fields['TaxLotState']).order_by('id') + + tax_lot_results = [] + for tax_lot in tax_lots: + tax_lot_dict = TaxLotProperty.model_to_dict_with_mapping( + tax_lot, + taxlot_column_name_mapping, + fields=fields['TaxLotState'], + exclude=['extra_data'] + ) + tax_lot_dict.update( + TaxLotProperty.extra_data_to_dict_with_mapping( + tax_lot.extra_data, + taxlot_column_name_mapping, + fields=tax_lot.extra_data.keys(), + ).items() + ) + tax_lot_results.append(tax_lot_dict) + + result['tax_lots'] = tax_lot_results + + return result + + @staticmethod + def has_coparent(state_id, inventory_type, fields=None): + """ + Return the coparent of the current state id based on the inventory type. If fields + are given (as a list), then it will only return the fields specified of the state model + object as a dictionary. + + :param state_id: int, ID of PropertyState or TaxLotState + :param inventory_type: string, either properties | taxlots + :param fields: list, either None or list of fields to return + :return: dict or state object, If fields is not None then will return state_object + """ + state_model = PropertyState if inventory_type == 'properties' else TaxLotState + + # TODO: convert coparent to instance method, not class method + audit_entry, audit_count = state_model.coparent(state_id) + + if audit_count == 0: + return False + + if audit_count > 1: + return JsonResponse( + { + 'status': 'error', + 'message': 'Internal problem occurred, more than one merge record found' + }, + status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + return audit_entry[0] + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'remap': 'boolean', + 'mark_as_done': 'boolean', + }) + ) + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def map(self, request, pk=None): + """ + Starts a background task to convert imported raw data into + PropertyState and TaxLotState, using user's column mappings. + """ + + body = request.data + + remap = body.get('remap', False) + mark_as_done = body.get('mark_as_done', True) + if not ImportFile.objects.filter(pk=pk).exists(): + return { + 'status': 'error', + 'message': 'ImportFile {} does not exist'.format(pk) + } + + # return remap_data(import_file_id) + return JsonResponse(map_data(pk, remap, mark_as_done)) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def start_system_matching_and_geocoding(self, request, pk=None): + """ + Starts a background task to attempt automatic matching between buildings + in an ImportFile with other existing buildings within the same org. + """ + try: + task_geocode_buildings(pk) + except MapQuestAPIKeyError: + result = JsonResponse({ + 'status': 'error', + 'message': 'MapQuest API key may be invalid or at its limit.' + }, status=status.HTTP_403_FORBIDDEN) + return result + + try: + import_file = ImportFile.objects.get(pk=pk) + except ImportFile.DoesNotExist: + return { + 'status': 'error', + 'message': 'ImportFile {} does not exist'.format(pk) + } + + # if the file is BuildingSync, don't do the merging, but instead finish + # creating it's associated models (scenarios, meters, etc) + if import_file.from_buildingsync: + return task_map_additional_models(pk) + + return task_match_buildings(pk) + + @swagger_auto_schema_org_query_param + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def start_data_quality_checks(self, request, pk=None): + """ + Starts a background task to attempt automatic matching between buildings + in an ImportFile with other existing buildings within the same org. + """ + organization = Organization.objects.get(pk=request.query_params['organization_id']) + + return_value = do_checks(organization.id, None, None, pk) + # step 5: create a new model instance + return JsonResponse({ + 'progress_key': return_value['progress_key'], + 'progress': return_value, + }) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def validate_use_cases(self, request, pk=None): + """ + Starts a background task to call BuildingSync's use case validation + tool. + """ + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must include pk of import_file to validate' + }, status=status.HTTP_400_BAD_REQUEST) + + return task_validate_use_cases(import_file_id) + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({'cycle_id': 'string'}) + ) + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def start_save_data(self, request, pk=None): + """ + Starts a background task to import raw data from an ImportFile + into PropertyState objects as extra_data. If the cycle_id is set to + year_ending then the cycle ID will be set to the year_ending column for each + record in the uploaded file. Note that the year_ending flag is not yet enabled. + """ + body = request.data + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must pass file_id of file to save' + }, status=status.HTTP_400_BAD_REQUEST) + + cycle_id = body.get('cycle_id') + if not cycle_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must pass cycle_id of the cycle to save the data' + }, status=status.HTTP_400_BAD_REQUEST) + elif cycle_id == 'year_ending': + _log.error("NOT CONFIGURED FOR YEAR ENDING OPTION AT THE MOMENT") + return JsonResponse({ + 'status': 'error', + 'message': 'SEED is unable to parse year_ending at the moment' + }, status=status.HTTP_400_BAD_REQUEST) + else: + # find the cycle + cycle = Cycle.objects.get(id=cycle_id) + if cycle: + # assign the cycle id to the import file object + import_file = ImportFile.objects.get(id=import_file_id) + import_file.cycle = cycle + import_file.save() + else: + return JsonResponse({ + 'status': 'error', + 'message': 'cycle_id was invalid' + }, status=status.HTTP_400_BAD_REQUEST) + + return JsonResponse(task_save_raw(import_file_id)) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def mapping_done(self, request, pk=None): + """ + Tell the backend that the mapping is complete. + """ + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must pass import_file_id' + }, status=status.HTTP_400_BAD_REQUEST) + + try: + import_file = ImportFile.objects.get(pk=import_file_id) + except ImportFile.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'no import file with given id' + }, status=status.HTTP_404_NOT_FOUND) + + import_file.mapping_done = True + import_file.save() + + return JsonResponse( + { + 'status': 'success', + 'message': '' + } + ) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @action(detail=True, methods=['GET']) + def matching_and_geocoding_results(self, request, pk=None): + """ + Retrieves the number of matched and unmatched properties & tax lots for + a given ImportFile record. Specifically for new imports + """ + import_file = ImportFile.objects.get(pk=pk) + + # property views associated with this imported file (including merges) + properties_new = [] + properties_matched = list(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_MERGED, + ).values_list('id', flat=True)) + + # Check audit log in case PropertyStates are listed as "new" but were merged into a different property + properties = list(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_NEW, + )) + + for state in properties: + audit_creation_id = PropertyAuditLog.objects.only('id').exclude( + import_filename=None).get( + state_id=state.id, + name='Import Creation' + ) + if PropertyAuditLog.objects.exclude(record_type=AUDIT_USER_EDIT).filter( + parent1_id=audit_creation_id + ).exists(): + properties_matched.append(state.id) + else: + properties_new.append(state.id) + + tax_lots_new = [] + tax_lots_matched = list(TaxLotState.objects.only('id').filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_MERGED, + ).values_list('id', flat=True)) + + # Check audit log in case TaxLotStates are listed as "new" but were merged into a different tax lot + taxlots = list(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_NEW, + )) + + for state in taxlots: + audit_creation_id = TaxLotAuditLog.objects.only('id').exclude(import_filename=None).get( + state_id=state.id, + name='Import Creation' + ) + if TaxLotAuditLog.objects.exclude(record_type=AUDIT_USER_EDIT).filter( + parent1_id=audit_creation_id + ).exists(): + tax_lots_matched.append(state.id) + else: + tax_lots_new.append(state.id) + + # Construct Geocode Results + property_geocode_results = { + 'high_confidence': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='High' + )), + 'low_confidence': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='Low' + )), + 'manual': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Manually geocoded (N/A)' + )), + 'missing_address_components': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Missing address components (N/A)' + )), + } + + tax_lot_geocode_results = { + 'high_confidence': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='High' + )), + 'low_confidence': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='Low' + )), + 'manual': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Manually geocoded (N/A)' + )), + 'missing_address_components': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Missing address components (N/A)' + )), + } + + # merge in any of the matching results from the JSON field + return { + 'status': 'success', + 'import_file_records': import_file.matching_results_data.get('import_file_records', None), + 'properties': { + 'initial_incoming': import_file.matching_results_data.get('property_initial_incoming', None), + 'duplicates_against_existing': import_file.matching_results_data.get('property_duplicates_against_existing', None), + 'duplicates_within_file': import_file.matching_results_data.get('property_duplicates_within_file', None), + 'merges_against_existing': import_file.matching_results_data.get('property_merges_against_existing', None), + 'merges_between_existing': import_file.matching_results_data.get('property_merges_between_existing', None), + 'merges_within_file': import_file.matching_results_data.get('property_merges_within_file', None), + 'new': import_file.matching_results_data.get('property_new', None), + 'geocoded_high_confidence': property_geocode_results.get('high_confidence'), + 'geocoded_low_confidence': property_geocode_results.get('low_confidence'), + 'geocoded_manually': property_geocode_results.get('manual'), + 'geocode_not_possible': property_geocode_results.get('missing_address_components'), + }, + 'tax_lots': { + 'initial_incoming': import_file.matching_results_data.get('tax_lot_initial_incoming', None), + 'duplicates_against_existing': import_file.matching_results_data.get('tax_lot_duplicates_against_existing', None), + 'duplicates_within_file': import_file.matching_results_data.get('tax_lot_duplicates_within_file', None), + 'merges_against_existing': import_file.matching_results_data.get('tax_lot_merges_against_existing', None), + 'merges_between_existing': import_file.matching_results_data.get('tax_lot_merges_between_existing', None), + 'merges_within_file': import_file.matching_results_data.get('tax_lot_merges_within_file', None), + 'new': import_file.matching_results_data.get('tax_lot_new', None), + 'geocoded_high_confidence': tax_lot_geocode_results.get('high_confidence'), + 'geocoded_low_confidence': tax_lot_geocode_results.get('low_confidence'), + 'geocoded_manually': tax_lot_geocode_results.get('manual'), + 'geocode_not_possible': tax_lot_geocode_results.get('missing_address_components'), + } + } + + @swagger_auto_schema_org_query_param + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @permission_classes((SEEDOrgPermissions,)) + @action(detail=True, methods=['GET']) + def mapping_suggestions(self, request, pk): + """ + Returns suggested mappings from an uploaded file's headers to known + data fields. + """ + organization_id = request.query_params.get('organization_id', None) + + result = {'status': 'success'} + + membership = OrganizationUser.objects.select_related('organization') \ + .get(organization_id=organization_id, user=request.user) + organization = membership.organization + + # For now, each organization holds their own mappings. This is non-ideal, but it is the + # way it is for now. In order to move to parent_org holding, then we need to be able to + # dynamically match columns based on the names and not the db id (or support many-to-many). + # parent_org = organization.get_parent() + + import_file = ImportFile.objects.get( + pk=pk, + import_record__super_organization_id=organization.pk + ) + + # Get a list of the database fields in a list, these are the db columns and the extra_data columns + property_columns = Column.retrieve_mapping_columns(organization.pk, 'property') + taxlot_columns = Column.retrieve_mapping_columns(organization.pk, 'taxlot') + + # If this is a portfolio manager file, then load in the PM mappings and if the column_mappings + # are not in the original mappings, default to PM + if import_file.from_portfolio_manager: + pm_mappings = simple_mapper.get_pm_mapping(import_file.first_row_columns, + resolve_duplicates=True) + suggested_mappings = mapper.build_column_mapping( + import_file.first_row_columns, + Column.retrieve_all_by_tuple(organization_id), + previous_mapping=get_column_mapping, + map_args=[organization], + default_mappings=pm_mappings, + thresh=80 + ) + elif import_file.from_buildingsync: + bsync_mappings = xml_mapper.build_column_mapping() + suggested_mappings = mapper.build_column_mapping( + import_file.first_row_columns, + Column.retrieve_all_by_tuple(organization_id), + previous_mapping=get_column_mapping, + map_args=[organization], + default_mappings=bsync_mappings, + thresh=80 + ) + else: + # All other input types + suggested_mappings = mapper.build_column_mapping( + import_file.first_row_columns, + Column.retrieve_all_by_tuple(organization.pk), + previous_mapping=get_column_mapping, + map_args=[organization], + thresh=80 # percentage match that we require. 80% is random value for now. + ) + # replace None with empty string for column names and PropertyState for tables + # TODO #239: Move this fix to build_column_mapping + for m in suggested_mappings: + table, destination_field, _confidence = suggested_mappings[m] + if destination_field is None: + suggested_mappings[m][1] = '' + + # Fix the table name, eventually move this to the build_column_mapping + for m in suggested_mappings: + table, _destination_field, _confidence = suggested_mappings[m] + # Do not return the campus, created, updated fields... that is force them to be in the property state + if not table or table == 'Property': + suggested_mappings[m][0] = 'PropertyState' + elif table == 'TaxLot': + suggested_mappings[m][0] = 'TaxLotState' + + result['suggested_column_mappings'] = suggested_mappings + result['property_columns'] = property_columns + result['taxlot_columns'] = taxlot_columns + + return JsonResponse(result) + + @swagger_auto_schema_org_query_param + @api_endpoint_class + @ajax_request_class + @permission_classes((SEEDOrgPermissions,)) + @has_perm_class('requires_member') + def destroy(self, request, pk): + """ + Deletes an import file + """ + organization_id = int(request.query_params.get('organization_id', None)) + import_file = ImportFile.objects.get(pk=pk) + + # check if the import record exists for the file and organization + d = ImportRecord.objects.filter( + super_organization_id=organization_id, + pk=import_file.import_record.pk + ) + + if not d.exists(): + return JsonResponse({ + 'status': 'error', + 'message': 'user does not have permission to delete file', + }, status=status.HTTP_403_FORBIDDEN) + + # This does not actually delete the object because it is a NonDeletableModel + import_file.delete() + return JsonResponse({'status': 'success'}) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py new file mode 100644 index 0000000000..a3637187ce --- /dev/null +++ b/seed/views/v3/labels.py @@ -0,0 +1,121 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author 'Piper Merriam ' +""" +from drf_yasg.utils import swagger_auto_schema +from rest_framework.parsers import JSONParser, FormParser +from rest_framework.renderers import JSONRenderer + +from seed.decorators import DecoratorMixin +from seed.filters import ( + LabelFilterBackend, +) +from seed.models import ( + StatusLabel as Label, +) +from seed.serializers.labels import ( + LabelSerializer, +) +from seed.utils.api import drf_api_endpoint +from django.utils.decorators import method_decorator +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import filter_labels_for_inv_type +from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet + + +@method_decorator( + name='retrieve', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")] + ), +) +@method_decorator( + name='list', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")] + ), +) +@method_decorator( + name='create', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")], + request_body=AutoSchemaHelper.schema_factory( + { + 'name': 'string', + 'color': 'string', + }, + required=['name'], + description='An object containing meta data for a new label' + ) + ), +) +@method_decorator( + name='destroy', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")] + ), +) +@method_decorator( + name='update', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")], + request_body=AutoSchemaHelper.schema_factory( + { + 'name': 'string', + 'color': 'string', + }, + required=['name'], + description='An object containing meta data for updating a label' + ) + ), +) +class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateModelViewSet): + """ + retrieve: + Return a label instance by pk if it is within user`s specified org. + + list: + Return all labels available to user through user`s specified org. + + create: + Create a new label within user`s specified org. + + delete: + Remove an existing label. + + update: + Update a label record. + """ + serializer_class = LabelSerializer + renderer_classes = (JSONRenderer,) + parser_classes = (JSONParser, FormParser) + queryset = Label.objects.none() + filter_backends = (LabelFilterBackend,) + pagination_class = None + _organization = None + + def get_queryset(self): + labels = Label.objects.filter( + super_organization=self.get_parent_org(self.request) + ).order_by("name").distinct() + return labels + + def get_serializer(self, *args, **kwargs): + kwargs['super_organization'] = self.get_organization(self.request) + inventory = filter_labels_for_inv_type( + request=self.request + ) + kwargs['inventory'] = inventory + return super().get_serializer(*args, **kwargs) diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py new file mode 100644 index 0000000000..b5cf9292c3 --- /dev/null +++ b/seed/views/v3/organizations.py @@ -0,0 +1,131 @@ +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" + +import logging + +from django.http import JsonResponse +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework import viewsets +from rest_framework.decorators import action + +from seed.decorators import ajax_request_class +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import Organization +from seed.models.columns import Column +from seed.models import ImportFile +from seed.serializers.column_mappings import SaveColumnMappingsRequestPayloadSerializer +from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper + +_log = logging.getLogger(__name__) + + +class OrganizationViewSet(viewsets.ViewSet): + + model = Column + + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['DELETE']) + def columns(self, request, pk=None): + """ + Delete all columns for an organization. This method is typically not recommended if there + are data in the inventory as it will invalidate all extra_data fields. This also removes + all the column mappings that existed. + + --- + parameters: + - name: pk + description: The organization_id + required: true + paramType: path + type: + status: + description: success or error + type: string + required: true + column_mappings_deleted_count: + description: Number of column_mappings that were deleted + type: integer + required: true + columns_deleted_count: + description: Number of columns that were deleted + type: integer + required: true + """ + try: + org = Organization.objects.get(pk=pk) + c_count, cm_count = Column.delete_all(org) + return JsonResponse( + { + 'status': 'success', + 'column_mappings_deleted_count': cm_count, + 'columns_deleted_count': c_count, + } + ) + except Organization.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'organization with with id {} does not exist'.format(pk) + }, status=status.HTTP_404_NOT_FOUND) + + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_integer_field( + 'import_file_id', required=True, description='Import file id'), + openapi.Parameter( + 'id', openapi.IN_PATH, type=openapi.TYPE_INTEGER, description='Organization id'), + ], + request_body=SaveColumnMappingsRequestPayloadSerializer, + responses={ + 200: 'success response' + } + ) + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @action(detail=True, methods=['POST']) + def column_mappings(self, request, pk=None): + """ + Saves the mappings between the raw headers of an ImportFile and the + destination fields in the `to_table_name` model which should be either + PropertyState or TaxLotState + + Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` + """ + import_file_id = request.query_params.get('import_file_id') + if import_file_id is None: + return JsonResponse({ + 'status': 'error', + 'message': 'Query param `import_file_id` is required' + }, status=status.HTTP_400_BAD_REQUEST) + try: + _ = ImportFile.objects.get(pk=import_file_id) + organization = Organization.objects.get(pk=pk) + except ImportFile.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'No import file found' + }, status=status.HTTP_404_NOT_FOUND) + except Organization.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'No organization found' + }, status=status.HTTP_404_NOT_FOUND) + + result = Column.create_mappings( + request.data.get('mappings', []), + organization, + request.user, + import_file_id + ) + + if result: + return JsonResponse({'status': 'success'}) + else: + return JsonResponse({'status': 'error'}) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py new file mode 100644 index 0000000000..072078805f --- /dev/null +++ b/seed/views/v3/properties.py @@ -0,0 +1,47 @@ +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +from drf_yasg.utils import swagger_auto_schema +from rest_framework import viewsets +from collections import namedtuple + +from rest_framework.decorators import action +from rest_framework.parsers import JSONParser +from rest_framework.renderers import JSONRenderer +from seed.models import ( + StatusLabel as Label, +) +from seed.utils.api import OrgMixin +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import get_labels + +ErrorState = namedtuple('ErrorState', ['status_code', 'message']) + + +class PropertyViewSet(viewsets.ViewSet, OrgMixin): + renderer_classes = (JSONRenderer,) + parser_classes = (JSONParser,) + _organization = None + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)], + request_body=AutoSchemaHelper.schema_factory( + { + 'selected': ['integer'], + }, + description='IDs for properties to be checked for which labels are applied.' + ) + ) + @action(detail=False, methods=['POST']) + def labels(self, request): + """ + Returns a list of all labels where the is_applied field + in the response pertains to the labels applied to property_view + """ + labels = Label.objects.filter( + super_organization=self.get_parent_org(self.request) + ).order_by("name").distinct() + super_organization = self.get_organization(request) + # TODO: refactor to avoid passing request here + return get_labels(request, labels, super_organization, 'property_view') diff --git a/seed/views/v3/taxlots.py b/seed/views/v3/taxlots.py new file mode 100644 index 0000000000..baa5615f4e --- /dev/null +++ b/seed/views/v3/taxlots.py @@ -0,0 +1,45 @@ +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +from drf_yasg.utils import swagger_auto_schema +from rest_framework import viewsets +from collections import namedtuple + +from rest_framework.decorators import action +from rest_framework.parsers import JSONParser +from rest_framework.renderers import JSONRenderer +from seed.models import StatusLabel as Label +from seed.utils.api import OrgMixin +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import get_labels + +ErrorState = namedtuple('ErrorState', ['status_code', 'message']) + + +class TaxlotViewSet(viewsets.ViewSet, OrgMixin): + renderer_classes = (JSONRenderer,) + parser_classes = (JSONParser,) + _organization = None + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)], + request_body=AutoSchemaHelper.schema_factory( + { + 'selected': ['integer'], + }, + description='IDs for taxlots to be checked for which labels are applied.' + ) + ) + @action(detail=False, methods=['POST']) + def labels(self, request): + """ + Returns a list of all labels where the is_applied field + in the response pertains to the labels applied to taxlot_view + """ + labels = Label.objects.filter( + super_organization=self.get_parent_org(self.request) + ).order_by("name").distinct() + super_organization = self.get_organization(request) + # TODO: refactor to avoid passing request here + return get_labels(request, labels, super_organization, 'taxlot_view') diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py new file mode 100644 index 0000000000..106a02eefa --- /dev/null +++ b/seed/views/v3/users.py @@ -0,0 +1,646 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import logging + +from django.contrib.auth.password_validation import validate_password +from django.contrib.auth.tokens import default_token_generator +from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ValidationError +from django.http import JsonResponse +from drf_yasg.utils import swagger_auto_schema +from rest_framework import viewsets, status, serializers +from rest_framework.decorators import action + +from seed.decorators import ajax_request_class +from seed.landing.models import SEEDUser as User, SEEDUser +from seed.lib.superperms.orgs.decorators import PERMS +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import ( + ROLE_OWNER, + ROLE_MEMBER, + ROLE_VIEWER, + Organization, + OrganizationUser, +) +from seed.models.data_quality import Rule +from seed.tasks import ( + invite_to_seed, +) +from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param +from seed.utils.organizations import create_organization +from rest_framework.status import HTTP_400_BAD_REQUEST + +_log = logging.getLogger(__name__) + + +def _get_js_role(role): + """return the JS friendly role name for user + :param role: role as defined in superperms.models + :returns: (string) JS role name + """ + roles = { + ROLE_OWNER: 'owner', + ROLE_VIEWER: 'viewer', + ROLE_MEMBER: 'member', + } + return roles.get(role, 'viewer') + + +def _get_role_from_js(role): + """return the OrganizationUser role_level from the JS friendly role name + + :param role: 'member', 'owner', or 'viewer' + :returns: int role as defined in superperms.models + """ + roles = { + 'owner': ROLE_OWNER, + 'viewer': ROLE_VIEWER, + 'member': ROLE_MEMBER, + } + return roles[role] + + +def _get_js_rule_type(data_type): + """return the JS friendly data type name for the data data_quality rule + + :param data_type: data data_quality rule data type as defined in data_quality.models + :returns: (string) JS data type name + """ + return dict(Rule.DATA_TYPES).get(data_type) + + +def _get_rule_type_from_js(data_type): + """return the Rules TYPE from the JS friendly data type + + :param data_type: 'string', 'number', 'date', or 'year' + :returns: int data type as defined in data_quality.models + """ + d = {v: k for k, v in dict(Rule.DATA_TYPES).items()} + return d.get(data_type) + + +def _get_js_rule_severity(severity): + """return the JS friendly severity name for the data data_quality rule + + :param severity: data data_quality rule severity as defined in data_quality.models + :returns: (string) JS severity name + """ + return dict(Rule.SEVERITY).get(severity) + + +def _get_severity_from_js(severity): + """return the Rules SEVERITY from the JS friendly severity + + :param severity: 'error', or 'warning' + :returns: int severity as defined in data_quality.models + """ + d = {v: k for k, v in dict(Rule.SEVERITY).items()} + return d.get(severity) + + +class EmailAndIDSerializer(serializers.Serializer): + email = serializers.CharField(max_length=100) + user_id = serializers.IntegerField() + + +class ListUsersResponseSerializer(serializers.Serializer): + users = EmailAndIDSerializer(many=True) + + +# this is used for swagger docs for some views below +user_response_schema = AutoSchemaHelper.schema_factory({ + 'first_name': 'string', + 'last_name': 'string', + 'email': 'string', + 'api_key': 'string', +}) + + +class UserViewSet(viewsets.ViewSet): + raise_exception = True + + def validate_request_user(self, pk, request): + try: + user = User.objects.get(pk=pk) + except ObjectDoesNotExist: + return False, JsonResponse( + {'status': 'error', 'message': "Could not find user with pk = " + str(pk)}, + status=status.HTTP_404_NOT_FOUND) + if not user == request.user: + return False, JsonResponse( + {'status': 'error', 'message': "Cannot access user with pk = " + str(pk)}, + status=status.HTTP_403_FORBIDDEN) + return True, user + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'org_name': 'string', + 'first_name': 'string', + 'last_name': 'string', + 'role': 'string', + 'email': 'string', + }, + required=['first_name', 'last_name', 'role', 'email'], + description='An object containing meta data for a new user:\n' + '-org_name: New organization name if creating a new organization for this user\n' + '-first_name: First name of new user\n' + '-last_name: Last name of new user\n' + '-role: one of owner, member, or viewer\n' + '-email: Email address of the new user' + ), + responses={ + 200: AutoSchemaHelper.schema_factory( + { + 'status': 'string', + 'message': 'string', + 'org': 'string', + 'org_created': 'boolean', + 'username': 'string', + 'user_id': 'string', + }, + description='Properties:\n' + '-org: name of new org (or existing org)\n' + '-org_created: true if new org created\n' + '-username: username of new user\n' + '-user_id: user id (pk) of new user' + ) + } + ) + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_owner') + def create(self, request): + """ + Creates a new SEED user. One of 'organization_id' or 'org_name' is needed. + Sends invitation email to the new user. + """ + body = request.data + org_name = body.get('org_name') + org_id = request.query_params.get('organization_id', None) + if (org_name and org_id) or (not org_name and not org_id): + return JsonResponse({ + 'status': 'error', + 'message': 'Choose either an existing org or provide a new one' + }, status=status.HTTP_409_CONFLICT) + + first_name = body['first_name'] + last_name = body['last_name'] + email = body['email'] + username = body['email'] + user, created = User.objects.get_or_create(username=username.lower()) + + if org_id: + org = Organization.objects.get(pk=org_id) + org_created = False + else: + org, _, _ = create_organization(user, org_name) + org_created = True + + # Add the user to the org. If this is the org's first user, + # the user becomes the owner/admin automatically. + # see Organization.add_member() + if not org.is_member(user): + org.add_member(user) + + if body.get('role'): + # check if this is a dict, if so, grab the value out of 'value' + role = body['role'] + try: + _get_role_from_js(role) + except Exception: + return JsonResponse({'status': 'error', 'message': 'valid arguments for role are [viewer, member, ' + 'owner]'}, + status=status.HTTP_400_BAD_REQUEST) + + OrganizationUser.objects.filter( + organization_id=org.pk, + user_id=user.pk + ).update(role_level=_get_role_from_js(role)) + + if created: + user.set_unusable_password() + user.email = email + user.first_name = first_name + user.last_name = last_name + user.save() + + try: + domain = request.get_host() + except Exception: + domain = 'seed-platform.org' + invite_to_seed( + domain, user.email, default_token_generator.make_token(user), user.pk, first_name + ) + + return JsonResponse({ + 'status': 'success', + 'message': user.email, + 'org': org.name, + 'org_created': org_created, + 'username': user.username, + 'user_id': user.id + }) + + @swagger_auto_schema( + responses={ + 200: ListUsersResponseSerializer, + } + ) + @ajax_request_class + @has_perm_class('requires_superuser') + def list(self, request): + """ + Retrieves all users' email addresses and IDs. + Only usable by superusers. + """ + users = [] + for user in User.objects.only('id', 'email'): + users.append({'email': user.email, 'user_id': user.id}) + return JsonResponse({'users': users}) + + @ajax_request_class + @api_endpoint_class + @action(detail=False, methods=['GET']) + def current(self, request): + """ + Returns the id (primary key) for the current user to allow it + to be passed to other user related endpoints + --- + type: + pk: + description: Primary key for the current user + required: true + type: string + """ + return JsonResponse({'pk': request.user.id}) + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'role': 'string', + }, + description='new role for user', + ), + ) + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_owner') + @action(detail=True, methods=['PUT']) + def role(self, request, pk=None): + """ + Updates a user's role within an organization. + """ + body = request.data + role = _get_role_from_js(body['role']) + + user_id = pk + organization_id = request.query_params.get('organization_id', None) + + is_last_member = not OrganizationUser.objects.filter( + organization_id=organization_id, + ).exclude(user_id=user_id).exists() + + if is_last_member: + return JsonResponse({ + 'status': 'error', + 'message': 'an organization must have at least one member' + }, status=status.HTTP_409_CONFLICT) + + is_last_owner = not OrganizationUser.objects.filter( + organization_id=organization_id, + role_level=ROLE_OWNER, + ).exclude(user_id=user_id).exists() + + if is_last_owner: + return JsonResponse({ + 'status': 'error', + 'message': 'an organization must have at least one owner level member' + }, status=status.HTTP_409_CONFLICT) + + OrganizationUser.objects.filter( + user_id=user_id, + organization_id=organization_id + ).update(role_level=role) + + return JsonResponse({'status': 'success'}) + + @swagger_auto_schema( + responses={ + 200: user_response_schema, + } + ) + @api_endpoint_class + @ajax_request_class + def retrieve(self, request, pk=None): + """ + Retrieves the a user's first_name, last_name, email + and api key if exists by user ID (pk). + """ + + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + return JsonResponse({ + 'status': 'success', + 'first_name': user.first_name, + 'last_name': user.last_name, + 'email': user.email, + 'api_key': user.api_key, + }) + + @ajax_request_class + @action(detail=True, methods=['POST']) + def generate_api_key(self, request, pk=None): + """ + Generates a new API key + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID / primary key + type: integer + required: true + paramType: path + type: + status: + description: success or error + type: string + required: true + api_key: + description: the new API key for this user + type: string + required: true + """ + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + user.generate_key() + return { + 'status': 'success', + 'api_key': User.objects.get(pk=pk).api_key + } + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'first_name': 'string', + 'last_name': 'string', + 'email': 'string' + }), + description='An object containing meta data for a updated user: \n' + '- Required - first_name, last_name, email', + responses={ + 200: user_response_schema, + } + ) + @api_endpoint_class + @ajax_request_class + def update(self, request, pk=None): + """ + Updates the user's first name, last name, and email + """ + body = request.data + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + json_user = body + user.first_name = json_user.get('first_name') + user.last_name = json_user.get('last_name') + user.email = json_user.get('email') + user.username = json_user.get('email') + user.save() + return JsonResponse({ + 'status': 'success', + 'first_name': user.first_name, + 'last_name': user.last_name, + 'email': user.email, + 'api_key': user.api_key, + }) + + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory( + { + 'current_password': 'string', + 'password_1': 'string', + 'password_2': 'string', + }, + description='Fill in the current and new matching passwords'), + ) + @ajax_request_class + @action(detail=True, methods=['PUT']) + def set_password(self, request, pk=None): + """ + sets/updates a user's password, follows the min requirement of + django password validation settings in config/settings/common.py + """ + body = request.data + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + current_password = body.get('current_password') + p1 = body.get('password_1') + p2 = body.get('password_2') + if not user.check_password(current_password): + return JsonResponse({'status': 'error', 'message': 'current password is not valid'}, + status=status.HTTP_400_BAD_REQUEST) + if p1 is None or p1 != p2: + return JsonResponse({'status': 'error', 'message': 'entered password do not match'}, + status=status.HTTP_400_BAD_REQUEST) + try: + validate_password(p2) + except ValidationError as e: + return JsonResponse({'status': 'error', 'message': e.messages[0]}, + status=status.HTTP_400_BAD_REQUEST) + user.set_password(p1) + user.save() + return JsonResponse({'status': 'success'}) + + @ajax_request_class + def get_actions(self, request): + """returns all actions""" + return { + 'status': 'success', + 'actions': list(PERMS.keys()), + } + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'actions': ['string'], + }, + description='A list of actions to check: examples include (requires_parent_org_owner, ' + 'requires_owner, requires_member, requires_viewer, ' + 'requires_superuser, can_create_sub_org, can_remove_org)' + ), + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'auth': { + 'action_name': 'boolean' + } + }) + } + ) + @ajax_request_class + @action(detail=True, methods=['POST']) + def is_authorized(self, request, pk=None): + """ + Checks the auth for a given action, if user is the owner of the parent + org then True is returned for each action + """ + actions, org, error, message = self._parse_is_authenticated_params(request) + if error: + return JsonResponse({ + 'status': 'error', + 'message': message + }, status=status.HTTP_400_BAD_REQUEST) + + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + + # If the only action requested is 'requires_superuser' no need to check an org affiliation + if len(actions) == 1 and actions[0] == 'requires_superuser': + return JsonResponse( + {'status': 'success', 'auth': {'requires_superuser': user.is_superuser}}) + + auth = self._try_parent_org_auth(user, org, actions) + if auth: + return JsonResponse({'status': 'success', 'auth': auth}) + + try: + ou = OrganizationUser.objects.get(user=user, organization=org) + except OrganizationUser.DoesNotExist: + return JsonResponse({'status': 'error', 'message': 'user does not exist'}) + + auth = {action: PERMS[action](ou) for action in actions} + return JsonResponse({'status': 'success', 'auth': auth}) + + def _parse_is_authenticated_params(self, request): + """checks if the org exists and if the actions are present + + :param request: the request + :returns: tuple (actions, org, error, message) + """ + error = False + message = "" + body = request.data + if not body.get('actions'): + message = 'no actions to check' + error = True + + org_id = request.query_params.get('organization_id', None) + if org_id == '': + message = 'organization id is undefined' + error = True + org = None + else: + try: + org = Organization.objects.get(pk=org_id) + except Organization.DoesNotExist: + message = 'organization does not exist' + error = True + org = None + + return body.get('actions'), org, error, message + + def _try_parent_org_auth(self, user, organization, actions): + """checks the parent org for permissions, if the user is not the owner of + the parent org, then None is returned. + + :param user: the request user + :param organization: org to check its parent + :param actions: list of str actions to check + :returns: a dict of action permission resolutions or None + """ + try: + ou = OrganizationUser.objects.get( + user=user, + organization=organization.parent_org, + role_level=ROLE_OWNER + ) + except OrganizationUser.DoesNotExist: + return None + + return { + action: PERMS['requires_owner'](ou) for action in actions + } + + @swagger_auto_schema( + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'show_shared_buildings': 'boolean' + }) + } + ) + @ajax_request_class + @action(detail=True, methods=['GET']) + def shared_buildings(self, request, pk=None): + """ + Get the request user's ``show_shared_buildings`` attr + """ + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + + return JsonResponse({ + 'status': 'success', + 'show_shared_buildings': user.show_shared_buildings, + }) + + @swagger_auto_schema_org_query_param + @ajax_request_class + @action(detail=True, methods=['PUT']) + def default_organization(self, request, pk=None): + """ + Sets the user's default organization + """ + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + user.default_organization_id = request.query_params.get('organization_id', None) + user.save() + return {'status': 'success'} + + @ajax_request_class + @action(detail=True, methods=['PUT']) + def deactivate(self, request, pk=None): + """ + Deactivates a user + """ + try: + user_id = pk + user = SEEDUser.objects.get( + id=user_id + ) + user.deactivate_user() + return JsonResponse({ + 'status': 'successfully deactivated', + 'data': user.email + }) + except Exception as e: + return JsonResponse({ + 'status': 'error', + 'data': str(e), + }, status=HTTP_400_BAD_REQUEST)