From fc097c58495810e96938858a99a675960b99dc98 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Wed, 27 May 2020 17:35:47 -0400 Subject: [PATCH 1/5] wip: use case validation for bsync --- .../data/buildingsync_v2_0_bricr_workflow.xml | 15 ++ .../tests/test_validation_client.py | 226 ++++++++++++++++++ seed/building_sync/validation_client.py | 131 ++++++++++ seed/data_importer/tasks.py | 42 ++++ seed/data_importer/views.py | 39 +++ .../data_upload_modal_controller.js | 78 ++++-- .../seed/js/services/uploader_service.js | 20 ++ .../seed/partials/data_upload_modal.html | 48 ++-- 8 files changed, 553 insertions(+), 46 deletions(-) create mode 100644 seed/building_sync/tests/test_validation_client.py create mode 100644 seed/building_sync/validation_client.py diff --git a/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml b/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml index bf07ba1827..6364088d14 100644 --- a/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml +++ b/seed/building_sync/tests/data/buildingsync_v2_0_bricr_workflow.xml @@ -1915,6 +1915,7 @@ 0.0 + 0.0 0 @@ -1936,6 +1937,7 @@ kBtu 2235075.9859244428 2235.076 + 110.75706 Natural gas @@ -2166,6 +2168,9 @@ All end uses 3902.655949232491 + 52.8533358512101 + 2051986.1527934822 + 132.24988549512202 @@ -2183,6 +2188,7 @@ 758.5569977291548 + 1772.1336698497769 15183 @@ -2204,6 +2210,7 @@ kBtu 3206417.9271639367 2064.179 + 110.75706 Natural gas @@ -2434,6 +2441,9 @@ All end uses 3144.098951503336 + 52.8533358512101 + 2051986.1527934822 + 132.24988549512202 @@ -2456,6 +2466,7 @@ 284.11765998512055 + 1772.1336698497769 5194 @@ -2477,6 +2488,7 @@ kBtu 2913836.260294419 2913.836 + 110.75706 Natural gas @@ -2707,6 +2719,9 @@ All end uses 3618.5382892473704 + 52.8533358512101 + 2051986.1527934822 + 132.24988549512202 diff --git a/seed/building_sync/tests/test_validation_client.py b/seed/building_sync/tests/test_validation_client.py new file mode 100644 index 0000000000..17118f8b18 --- /dev/null +++ b/seed/building_sync/tests/test_validation_client.py @@ -0,0 +1,226 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2019, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import os +from unittest.mock import patch + +from django.test import TestCase +from requests.models import Response +import json + +from config.settings.common import BASE_DIR +from seed.building_sync.validation_client import validate_use_case, DEFAULT_USE_CASE + + +def responseFactory(status_code, body_dict): + the_response = Response() + the_response.status_code = status_code + the_response._content = json.dumps(body_dict).encode() + return the_response + + +class TestValidationClient(TestCase): + def setUp(self): + # NOTE: the contents of these files are not actually used, it's just convenient + # to use these files so we don't have to create tmp ones and clean them up + self.single_file = open(os.path.join(BASE_DIR, 'seed', 'building_sync', 'tests', 'data', 'buildingsync_v2_0_bricr_workflow.xml')) + self.zip_file = open(os.path.join(BASE_DIR, 'seed', 'building_sync', 'tests', 'data', 'ex_1_and_buildingsync_ex01_measures.zip')) + + def test_validation_single_file_ok(self): + good_body = { + 'success': True, + 'validation_results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, good_body)): + all_files_valid, file_summaries = validate_use_case(self.single_file) + + self.assertTrue(all_files_valid) + self.assertEqual([], file_summaries) + + def test_validation_zip_file_ok(self): + good_body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + }, { + 'file': 'file2.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + } + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, good_body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertTrue(all_files_valid) + self.assertEqual([], file_summaries) + + def test_validation_fails_when_one_file_has_bad_schema(self): + bad_file_result = { + 'file': 'bad.xml', + 'results': { + 'schema': { + # Set the schema as NOT valid + 'valid': False, + 'errors': ['schema was bad'] + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + } + + body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + }, + bad_file_result, + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertFalse(all_files_valid) + bad_file_names = [f['file'] for f in file_summaries] + self.assertEqual([bad_file_result['file']], bad_file_names) + + def test_validation_fails_when_one_file_fails_use_case(self): + bad_file_result = { + 'file': 'bad.xml', + 'results': { + 'schema': { + 'valid': True, + }, + 'use_cases': { + DEFAULT_USE_CASE: { + # Include a use case error + 'errors': ['something was wrong'], + 'warnings': [], + } + } + } + } + + body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + 'warnings': [], + } + } + } + }, + bad_file_result, + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertFalse(all_files_valid) + bad_file_names = [f['file'] for f in file_summaries] + self.assertEqual([bad_file_result['file']], bad_file_names) + + def test_validation_zip_file_ok_when_warnings(self): + good_body = { + 'success': True, + 'validation_results': [ + { + 'file': 'file1.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + # Include a warning + 'warnings': ['This is a warning!'], + } + } + } + }, { + 'file': 'file2.xml', + 'results': { + 'schema': { + 'valid': True + }, + 'use_cases': { + DEFAULT_USE_CASE: { + 'errors': [], + # Include a warning + 'warnings': ['This is another warning!'], + } + } + } + } + ] + } + + with patch('seed.building_sync.validation_client._validation_api_post', return_value=responseFactory(200, good_body)): + all_files_valid, file_summaries = validate_use_case(self.zip_file) + + self.assertTrue(all_files_valid) + file_names = [f['file'] for f in file_summaries] + self.assertEqual(['file1.xml', 'file2.xml'], file_names) diff --git a/seed/building_sync/validation_client.py b/seed/building_sync/validation_client.py new file mode 100644 index 0000000000..3f5632dcbb --- /dev/null +++ b/seed/building_sync/validation_client.py @@ -0,0 +1,131 @@ +import os + +import requests + + +VALIDATION_API_URL = "https://selectiontool.buildingsync.net/api/validate" +DEFAULT_SCHEMA_VERSION = '2.0.0' +DEFAULT_USE_CASE = 'L000 OpenStudio Simulation' + + +class ValidationClientException(Exception): + pass + + +def _validation_api_post(file_, schema_version, use_case_name): + payload = {'schema_version': schema_version} + files = [ + ('file', file_) + ] + + return requests.request( + "POST", + VALIDATION_API_URL, + data=payload, + files=files, + timeout=60 * 2, # timeout after two minutes (it can take a long time for zips) + ) + + +def validate_use_case(file_, schema_version=DEFAULT_SCHEMA_VERSION, use_case_name=DEFAULT_USE_CASE): + """calls Selection Tool's validation API + + :param file_: File, the file to validate; can be single xml or zip + :param schema_version: string + :param use_case_name: string + :return: tuple, (bool, list), bool indicates if the file passes validation, + the list is a collection of files and their errors, warnings, and infos + """ + + try: + response = _validation_api_post(file_, schema_version, use_case_name) + except requests.exceptions.Timeout: + raise ValidationClientException( + f"Request to Selection Tool timed out. SEED may need to increase the timeout", + ) + except Exception as e: + raise ValidationClientException( + f"Failed to make request to selection tool: {e}", + ) + + if response.status_code != 200: + raise ValidationClientException( + f"Received bad response from Selection Tool: {response.text}", + ) + + try: + response_body = response.json() + except ValueError: + raise ValidationClientException( + f"Expected JSON response from Selection Tool: {response.text}", + ) + + if response_body.get('success', False) is not True: + ValidationClientException( + f"Selection Tool request was not successful: {response.text}", + ) + + response_schema_version = response_body.get('schema_version') + if response_schema_version != schema_version: + ValidationClientException( + f"Expected schema_version to be '{schema_version}' but it was '{response_schema_version}'", + ) + + _, file_extension = os.path.splitext(file_.name) + validation_results = response_body.get('validation_results') + # check the returned type and make validation_results a list if it's not already + if file_extension == '.zip': + if type(validation_results) is not list: + raise ValidationClientException( + f"Expected response validation_results to be list for zip file: {response.text}", + ) + else: + if type(validation_results) is not dict: + raise ValidationClientException( + f"Expected response validation_results to be dict for single xml file: {response.text}", + ) + # turn the single file result into the same structure as zip file result + validation_results = [{ + 'file': os.path.basename(file_.name), + 'results': validation_results, + }] + + # check that the schema and use case is valid for every file + file_summaries = [] + all_files_valid = True + for validation_result in validation_results: + results = validation_result['results'] + filename = validation_result['file'] + + # it's possible there's no use_cases key - occurs when schema validation fails + if results['schema']['valid'] is False: + use_case_result = {} + else: + if use_case_name not in results.get('use_cases', []): + raise ValidationClientException( + f'Expected use case "{use_case_name}" to exist in result\'s uses cases.', + ) + use_case_result = results['use_cases'][use_case_name] + + file_summary = { + 'file': filename, + 'schema_errors': results['schema'].get('errors', []), + 'use_case_errors': use_case_result.get('errors', []), + 'use_case_warnings': use_case_result.get('warnings', []), + } + + file_has_errors = ( + len(file_summary['schema_errors']) > 0 + or len(file_summary['use_case_errors']) > 0 + ) + if file_has_errors: + all_files_valid = False + + file_has_errors_or_warnings = ( + file_has_errors + or len(file_summary['use_case_warnings']) > 0 + ) + if file_has_errors_or_warnings: + file_summaries.append(file_summary) + + return all_files_valid, file_summaries diff --git a/seed/data_importer/tasks.py b/seed/data_importer/tasks.py index bfe849b8d0..c52b4d97ce 100644 --- a/seed/data_importer/tasks.py +++ b/seed/data_importer/tasks.py @@ -33,6 +33,7 @@ from past.builtins import basestring from unidecode import unidecode +from seed.building_sync import validation_client from seed.data_importer.equivalence_partitioner import EquivalencePartitioner from seed.data_importer.match import ( match_and_link_incoming_properties_and_taxlots, @@ -1527,3 +1528,44 @@ def pair_new_states(merged_property_views, merged_taxlot_views): m2m_join.save() return + + +@shared_task +def _validate_use_cases(file_pk, progress_key): + import_file = ImportFile.objects.get(pk=file_pk) + progress_data = ProgressData.from_key(progress_key) + + try: + all_files_valid, file_summaries = validation_client.validate_use_case(import_file.file) + if all_files_valid is False: + import_file.delete() + progress_data.finish_with_success( + message=json.dumps({ + 'valid': all_files_valid, + 'issues': file_summaries, + }), + ) + except validation_client.ValidationClientException as e: + _log.debug(f'ValidationClientException while validating import_file `{file_pk}`: {e}') + progress_data.finish_with_error(message=str(e)) + progress_data.save() + import_file.delete() + except Exception as e: + _log.debug(f'Unexpected Exception while validating import_file `{file_pk}`: {e}') + progress_data.finish_with_error(message=str(e)) + progress_data.save() + import_file.delete() + + +def validate_use_cases(file_pk): + """ + Kicks off task for validating BuildingSync files for use cases + + :param file_pk: ImportFile Primary Key + :return: + """ + progress_data = ProgressData(func_name='validate_use_cases', unique_id=file_pk) + + _validate_use_cases.s(file_pk, progress_data.key).apply_async() + _log.debug(progress_data.result()) + return progress_data.result() diff --git a/seed/data_importer/views.py b/seed/data_importer/views.py index bba10c134f..03a9d440b5 100644 --- a/seed/data_importer/views.py +++ b/seed/data_importer/views.py @@ -34,6 +34,7 @@ map_additional_models as task_map_additional_models, match_buildings as task_match_buildings, save_raw_data as task_save_raw, + validate_use_cases as task_validate_use_cases, ) from seed.decorators import ajax_request, ajax_request_class from seed.decorators import get_prog_key @@ -905,6 +906,44 @@ def data_quality_progress(self, request, pk=None): @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) + @detail_route(methods=['POST']) + def validate_use_cases(self, request, pk=None): + """ + Starts a background task to call BuildingSync's use case validation + tool. + --- + type: + status: + required: true + type: string + description: either success or error + message: + required: false + type: string + description: error message, if any + progress_key: + type: integer + description: ID of background job, for retrieving job progress + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must include pk of import_file to validate' + }, status=status.HTTP_400_BAD_REQUEST) + + return task_validate_use_cases(import_file_id) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @detail_route(methods=['POST']) def save_raw_data(self, request, pk=None): """ Starts a background task to import raw data from an ImportFile diff --git a/seed/static/seed/js/controllers/data_upload_modal_controller.js b/seed/static/seed/js/controllers/data_upload_modal_controller.js index aa8bd5440d..b4ed4a1a86 100644 --- a/seed/static/seed/js/controllers/data_upload_modal_controller.js +++ b/seed/static/seed/js/controllers/data_upload_modal_controller.js @@ -21,7 +21,6 @@ * ng-switch-when="11" == Confirm Save Mappings? * ng-switch-when="12" == Error Processing Data * ng-switch-when="13" == Portfolio Manager Import - * ng-switch-when="14" == Successful upload! [BuildingSync] */ angular.module('BE.seed.controller.data_upload_modal', []) .controller('data_upload_modal_controller', [ @@ -318,34 +317,20 @@ angular.module('BE.seed.controller.data_upload_modal', []) case 'upload_error': $scope.step_12_error_message = file.error; $scope.step.number = 12; - // add variables to identify buildingsync bulk uploads - $scope.building_sync_files = (file.source_type === 'BuildingSync'); - $scope.bulk_upload = (_.last(file.filename.split('.')) === 'zip'); break; case 'upload_in_progress': $scope.uploader.in_progress = true; - if (file.source_type === 'BuildingSync') { - $scope.uploader.progress = 100 * progress.loaded / progress.total; - } else { - $scope.uploader.progress = 25 * progress.loaded / progress.total; - } + $scope.uploader.progress = 25 * progress.loaded / progress.total; break; case 'upload_complete': var current_step = $scope.step.number; $scope.uploader.status_message = 'upload complete'; $scope.dataset.filename = file.filename; - $scope.step_14_message = null; $scope.source_type = file.source_type; - if (file.source_type === 'BuildingSync') { - $scope.uploader.complete = true; - $scope.uploader.in_progress = false; - $scope.uploader.progress = 100; - $scope.step.number = 14; - $scope.step_14_message = (_.size(file.message.warnings) > 0) ? file.message.warnings : null; - } else if (file.source_type === 'PM Meter Usage') { + if (file.source_type === 'PM Meter Usage') { $scope.cycle_id = file.cycle_id; $scope.file_id = file.file_id; @@ -361,7 +346,12 @@ angular.module('BE.seed.controller.data_upload_modal', []) // Assessed Data; upload is step 2; PM import is currently treated as such, and is step 13 if (current_step === 2 || current_step === 13) { - save_raw_assessed_data(file.file_id, file.cycle_id, false); + // if importing BuildingSync, validate then save, otherwise just save + if (file.source_type === "BuildingSync Raw") { + validate_use_cases_then_save(file.file_id, file.cycle_id) + } else { + save_raw_assessed_data(file.file_id, file.cycle_id, false) + } } // Portfolio Data if (current_step === 4) { @@ -496,6 +486,58 @@ angular.module('BE.seed.controller.data_upload_modal', []) }; }; + /** + * validate_use_cases_then_save: validates BuildingSync files for use cases + * before saving the data + * + * @param {string} file_id: the id of the import file + * @param cycle_id + */ + var validate_use_cases_then_save = function (file_id, cycle_id) { + $scope.uploader.status_message = 'validating data'; + $scope.uploader.progress = 0; + + const successHandler = (progress_data) => { + $scope.uploader.complete = false + $scope.uploader.in_progress = true + $scope.uploader.status_message = 'validation complete; starting to save data'; + $scope.uploader.progress = 100; + + const result = JSON.parse(progress_data.message) + $scope.buildingsync_valid = result.valid + $scope.buildingsync_issues = result.issues + + // if validation failed, end the import flow here; otherwise continue + if ($scope.buildingsync_valid !== true) { + $scope.step_12_error_message = 'Failed to validate uploaded BuildingSync file(s)' + $scope.step_12_buildingsync_validation_error = true + $scope.step.number = 12 + } else { + // successfully passed validation, save the data + save_raw_assessed_data(file_id, cycle_id, false) + } + } + + const errorHandler = (data) => { + $log.error(data.message); + if (data.hasOwnProperty('stacktrace')) $log.error(data.stacktrace); + $scope.step_12_error_message = data.data ? data.data.message : data.message; + $scope.step.number = 12; + } + + uploader_service.validate_use_cases(file_id) + .then(data => { + const progress = _.clamp(data.progress, 0, 100); + uploader_service.check_progress_loop( + data.progress_key, + progress, 1 - (progress / 100), + successHandler, + errorHandler, + $scope.uploader, + ) + }) + }; + /** * save_raw_assessed_data: saves Assessed data * diff --git a/seed/static/seed/js/services/uploader_service.js b/seed/static/seed/js/services/uploader_service.js index 8df8013c1e..bc02e58d4e 100644 --- a/seed/static/seed/js/services/uploader_service.js +++ b/seed/static/seed/js/services/uploader_service.js @@ -39,6 +39,26 @@ angular.module('BE.seed.service.uploader', []).factory('uploader_service', [ }); }; + /** + * validate_use_cases + * This service call will simply call a view on the backend to validate + * BuildingSync files with use cases + * @param file_id: the pk of a ImportFile object we're going to save raw. + */ + uploader_factory.validate_use_cases = function (file_id) { + return $http.post('/api/v2/import_files/' + file_id + '/validate_use_cases/') + .then(response => { + return response.data + }) + .catch(err => { + if (err.data.status === 'error') { + return err.data + } + // something unexpected happened... throw it + throw err + }) + }; + /** * save_raw_data * This service call will simply call a view on the backend to save raw diff --git a/seed/static/seed/partials/data_upload_modal.html b/seed/static/seed/partials/data_upload_modal.html index f0af0ca658..d83cc10785 100644 --- a/seed/static/seed/partials/data_upload_modal.html +++ b/seed/static/seed/partials/data_upload_modal.html @@ -12,7 +12,6 @@

{$:: 'UPLOAD_COMPLETE_TITLE' | trans

Confirm Save Mappings?

Error Processing Data

Portfolio Manager Import

-

Successful upload!

UPLOAD_PM_READINGS_MODAL_TITLE

PM_METER_IMPORT_RESULTS

@@ -250,10 +249,26 @@

PM_METER_IMPORT_RESULTS
-

An error occurred while processing the file. Please ensure that your file meets the required specifications.

-

Error(s) occurred while processing the BuildingSync data bulk upload, potentially resulting in a partial import. Fix the errors and re-upload the corrected files.

-

An error occurred while processing the BuildingSync xml file. Please ensure that your file meets the required specifications.

- 
{$ step_12_error_message | json $}
+

An error occurred while processing the file. Please ensure that your file meets the required specifications.

+ 
{$ step_12_error_message | json $}
+ +

BuildingSync file(s) failed to validate. No files were imported. Please fix the errors and re-upload the corrected file(s).

+
+ {$ file_issues.file $} +
+
Schema Errors
+
{$ err.path $}: {$ err.message $}
+
+
+
Use Case Errors
+
{$ err $}
+
+
+
Use Case Warnings
+
{$ warn $}
+
+
+
@@ -292,21 +307,6 @@

PM_METER_IMPORT_RESULTS - -
-
-
-
-
-

Warnings:

- 
{$ step_14_message | json $}
-
-
- -
-
An error occurred while processing the file. Please ensure that your file meets the required specifications.
@@ -402,14 +402,6 @@

PM_METER_IMPORT_RESULTSCONFIRM_AND_START_MATCHING

-
-
- -
-
- -
-
From e43a5d74de021328044197316ccf28db19378fd7 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 28 May 2020 14:03:33 -0400 Subject: [PATCH 2/5] chore(data_upload_modal.html): display BuildingSync warnings if any on successful upload --- seed/static/seed/partials/data_upload_modal.html | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/seed/static/seed/partials/data_upload_modal.html b/seed/static/seed/partials/data_upload_modal.html index d83cc10785..fbbbfa42c3 100644 --- a/seed/static/seed/partials/data_upload_modal.html +++ b/seed/static/seed/partials/data_upload_modal.html @@ -99,6 +99,18 @@

PM_METER_IMPORT_RESULTS

+
+
+

There were one or more warnings for the file(s) uploaded.

+
+ {$ file_issues.file $} +
+
Use Case Warnings
+
{$ warn $}
+
+
+
+
From 6d60ae45ea23af3db186000661dd3430f165de1b Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 29 May 2020 17:08:06 -0400 Subject: [PATCH 3/5] refactor: validation_client accepts filename + styling --- seed/building_sync/validation_client.py | 11 +++++++---- seed/data_importer/tasks.py | 11 ++++++++++- seed/static/seed/partials/data_upload_modal.html | 10 +++++----- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/seed/building_sync/validation_client.py b/seed/building_sync/validation_client.py index 3f5632dcbb..94e0d6821c 100644 --- a/seed/building_sync/validation_client.py +++ b/seed/building_sync/validation_client.py @@ -5,7 +5,7 @@ VALIDATION_API_URL = "https://selectiontool.buildingsync.net/api/validate" DEFAULT_SCHEMA_VERSION = '2.0.0' -DEFAULT_USE_CASE = 'L000 OpenStudio Simulation' +DEFAULT_USE_CASE = 'BRICR_SEED' class ValidationClientException(Exception): @@ -27,10 +27,11 @@ def _validation_api_post(file_, schema_version, use_case_name): ) -def validate_use_case(file_, schema_version=DEFAULT_SCHEMA_VERSION, use_case_name=DEFAULT_USE_CASE): +def validate_use_case(file_, filename=None, schema_version=DEFAULT_SCHEMA_VERSION, use_case_name=DEFAULT_USE_CASE): """calls Selection Tool's validation API :param file_: File, the file to validate; can be single xml or zip + :param filename: string, (optional) name of the file, useful if file_.name is not user friendly (e.g. a Django SimpleUploadedFile). Not used if file_ is a zip :param schema_version: string :param use_case_name: string :return: tuple, (bool, list), bool indicates if the file passes validation, @@ -41,7 +42,7 @@ def validate_use_case(file_, schema_version=DEFAULT_SCHEMA_VERSION, use_case_nam response = _validation_api_post(file_, schema_version, use_case_name) except requests.exceptions.Timeout: raise ValidationClientException( - f"Request to Selection Tool timed out. SEED may need to increase the timeout", + "Request to Selection Tool timed out. SEED may need to increase the timeout", ) except Exception as e: raise ValidationClientException( @@ -85,8 +86,10 @@ def validate_use_case(file_, schema_version=DEFAULT_SCHEMA_VERSION, use_case_nam f"Expected response validation_results to be dict for single xml file: {response.text}", ) # turn the single file result into the same structure as zip file result + if filename is None: + filename = os.path.basename(file_.name) validation_results = [{ - 'file': os.path.basename(file_.name), + 'file': filename, 'results': validation_results, }] diff --git a/seed/data_importer/tasks.py b/seed/data_importer/tasks.py index c52b4d97ce..1b7d116835 100644 --- a/seed/data_importer/tasks.py +++ b/seed/data_importer/tasks.py @@ -1535,8 +1535,12 @@ def _validate_use_cases(file_pk, progress_key): import_file = ImportFile.objects.get(pk=file_pk) progress_data = ProgressData.from_key(progress_key) + progress_data.step('validating data with Selection Tool') try: - all_files_valid, file_summaries = validation_client.validate_use_case(import_file.file) + all_files_valid, file_summaries = validation_client.validate_use_case( + import_file.file, + filename=import_file.uploaded_filename + ) if all_files_valid is False: import_file.delete() progress_data.finish_with_success( @@ -1565,6 +1569,11 @@ def validate_use_cases(file_pk): :return: """ progress_data = ProgressData(func_name='validate_use_cases', unique_id=file_pk) + # break progress into two steps: + # 1. started job + # 2. finished request + progress_data.total = 2 + progress_data.save() _validate_use_cases.s(file_pk, progress_data.key).apply_async() _log.debug(progress_data.result()) diff --git a/seed/static/seed/partials/data_upload_modal.html b/seed/static/seed/partials/data_upload_modal.html index fbbbfa42c3..16de3f7880 100644 --- a/seed/static/seed/partials/data_upload_modal.html +++ b/seed/static/seed/partials/data_upload_modal.html @@ -100,11 +100,11 @@

PM_METER_IMPORT_RESULTS

-
+

There were one or more warnings for the file(s) uploaded.

{$ file_issues.file $} -
+
Use Case Warnings
{$ warn $}
@@ -267,15 +267,15 @@
Use Case Warnings

BuildingSync file(s) failed to validate. No files were imported. Please fix the errors and re-upload the corrected file(s).

{$ file_issues.file $} -
+
Schema Errors
{$ err.path $}: {$ err.message $}
-
+
Use Case Errors
{$ err $}
-
+
Use Case Warnings
{$ warn $}
From 50d06bc30bff6393f387289bd199e7bc475e2311 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 2 Jun 2020 14:20:28 -0400 Subject: [PATCH 4/5] fix: remove old drf decorator --- seed/data_importer/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seed/data_importer/views.py b/seed/data_importer/views.py index 03a9d440b5..075da67fe6 100644 --- a/seed/data_importer/views.py +++ b/seed/data_importer/views.py @@ -906,7 +906,6 @@ def data_quality_progress(self, request, pk=None): @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) - @detail_route(methods=['POST']) def validate_use_cases(self, request, pk=None): """ Starts a background task to call BuildingSync's use case validation @@ -943,7 +942,7 @@ def validate_use_cases(self, request, pk=None): @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') - @detail_route(methods=['POST']) + @action(detail=True, methods=['POST']) def save_raw_data(self, request, pk=None): """ Starts a background task to import raw data from an ImportFile From d19bc2795c36c4f196e4f2c5d4a35b5003a967be Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 2 Jun 2020 14:20:48 -0400 Subject: [PATCH 5/5] fix(validation_client): use SEED use case for bsync validation --- seed/building_sync/validation_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/building_sync/validation_client.py b/seed/building_sync/validation_client.py index 94e0d6821c..23ddf91cc5 100644 --- a/seed/building_sync/validation_client.py +++ b/seed/building_sync/validation_client.py @@ -5,7 +5,7 @@ VALIDATION_API_URL = "https://selectiontool.buildingsync.net/api/validate" DEFAULT_SCHEMA_VERSION = '2.0.0' -DEFAULT_USE_CASE = 'BRICR_SEED' +DEFAULT_USE_CASE = 'SEED' class ValidationClientException(Exception):