From 7922063769f33e53b31c1ff44354e6fc2568ef78 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 12:53:49 -0600 Subject: [PATCH 01/28] API v3 geocode - copy v2 endpoint to v3 --- seed/api/v3/urls.py | 2 + seed/views/v3/geocode.py | 106 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 seed/views/v3/geocode.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index f2dfd89580..ba80fa97d6 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -13,6 +13,7 @@ from seed.views.v3.data_quality_checks import DataQualityCheckViewSet from seed.views.v3.data_quality_check_rules import DataQualityCheckRuleViewSet from seed.views.v3.datasets import DatasetViewSet +from seed.views.v3.geocode import GeocodeViews from seed.views.v3.import_files import ImportFileViewSet from seed.views.v3.labels import LabelViewSet from seed.views.v3.label_inventories import LabelInventoryViewSet @@ -30,6 +31,7 @@ api_v3_router.register(r'columns', ColumnViewSet, base_name='columns') api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') +api_v3_router.register(r'geocode', GeocodeViews, base_name='geocode') api_v3_router.register(r'labels', LabelViewSet, base_name='labels') api_v3_router.register(r'data_quality_checks', DataQualityCheckViewSet, base_name='data_quality_checks') api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files') diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py new file mode 100644 index 0000000000..40cdbef53c --- /dev/null +++ b/seed/views/v3/geocode.py @@ -0,0 +1,106 @@ +# !/usr/bin/env python +# encoding: utf-8 + +from rest_framework import viewsets +from rest_framework.decorators import action + +from seed.decorators import ajax_request_class + +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import Organization + +from seed.models.properties import PropertyState +from seed.models.tax_lots import TaxLotState + +from seed.utils.api import api_endpoint_class +from seed.utils.geocode import geocode_buildings + + +class GeocodeViews(viewsets.ViewSet): + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=False, methods=['POST']) + def geocode_by_ids(self, request): + body = dict(request.data) + property_ids = body.get('property_ids') + taxlot_ids = body.get('taxlot_ids') + + if property_ids: + properties = PropertyState.objects.filter(id__in=property_ids) + geocode_buildings(properties) + + if taxlot_ids: + taxlots = TaxLotState.objects.filter(id__in=taxlot_ids) + geocode_buildings(taxlots) + + @ajax_request_class + @action(detail=False, methods=['POST']) + def confidence_summary(self, request): + body = dict(request.data) + property_ids = body.get('property_ids') + tax_lot_ids = body.get('tax_lot_ids') + + result = {} + + if property_ids: + result["properties"] = { + 'not_geocoded': len(PropertyState.objects.filter( + id__in=property_ids, + geocoding_confidence__isnull=True + )), + 'high_confidence': len(PropertyState.objects.filter( + id__in=property_ids, + geocoding_confidence__startswith='High' + )), + 'low_confidence': len(PropertyState.objects.filter( + id__in=property_ids, + geocoding_confidence__startswith='Low' + )), + 'manual': len(PropertyState.objects.filter( + id__in=property_ids, + geocoding_confidence='Manually geocoded (N/A)' + )), + 'missing_address_components': len(PropertyState.objects.filter( + id__in=property_ids, + geocoding_confidence='Missing address components (N/A)' + )), + } + + if tax_lot_ids: + result["tax_lots"] = { + 'not_geocoded': len(TaxLotState.objects.filter( + id__in=tax_lot_ids, + geocoding_confidence__isnull=True + )), + 'high_confidence': len(TaxLotState.objects.filter( + id__in=tax_lot_ids, + geocoding_confidence__startswith='High' + )), + 'low_confidence': len(TaxLotState.objects.filter( + id__in=tax_lot_ids, + geocoding_confidence__startswith='Low' + )), + 'manual': len(TaxLotState.objects.filter( + id__in=tax_lot_ids, + geocoding_confidence='Manually geocoded (N/A)' + )), + 'missing_address_components': len(TaxLotState.objects.filter( + id__in=tax_lot_ids, + geocoding_confidence='Missing address components (N/A)' + )), + } + + return result + + @ajax_request_class + @action(detail=False, methods=['GET']) + def api_key_exists(self, request): + org_id = request.GET.get("organization_id") + org = Organization.objects.get(id=org_id) + + if org.mapquest_api_key: + return True + else: + return False From b0babb3aa0c8ae7c8e87d00053f290375ddc6723 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 12:58:20 -0600 Subject: [PATCH 02/28] API v3 geocode - rename class to include "ViewSet" --- seed/api/v3/urls.py | 4 ++-- seed/views/v3/geocode.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index ba80fa97d6..896164c5ae 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -13,7 +13,7 @@ from seed.views.v3.data_quality_checks import DataQualityCheckViewSet from seed.views.v3.data_quality_check_rules import DataQualityCheckRuleViewSet from seed.views.v3.datasets import DatasetViewSet -from seed.views.v3.geocode import GeocodeViews +from seed.views.v3.geocode import GeocodeViewSet from seed.views.v3.import_files import ImportFileViewSet from seed.views.v3.labels import LabelViewSet from seed.views.v3.label_inventories import LabelInventoryViewSet @@ -31,7 +31,7 @@ api_v3_router.register(r'columns', ColumnViewSet, base_name='columns') api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') -api_v3_router.register(r'geocode', GeocodeViews, base_name='geocode') +api_v3_router.register(r'geocode', GeocodeViewSet, base_name='geocode') api_v3_router.register(r'labels', LabelViewSet, base_name='labels') api_v3_router.register(r'data_quality_checks', DataQualityCheckViewSet, base_name='data_quality_checks') api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files') diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 40cdbef53c..6d04ae03b7 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -16,7 +16,7 @@ from seed.utils.geocode import geocode_buildings -class GeocodeViews(viewsets.ViewSet): +class GeocodeViewSet(viewsets.ViewSet): @api_endpoint_class @ajax_request_class From c7a3414815b141a9830cb75caed65e1cf716d777 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 13:35:21 -0600 Subject: [PATCH 03/28] API v3 geocode - geocode_by_ids shift to View IDs. --- seed/views/v3/geocode.py | 42 ++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 6d04ae03b7..97a0f175cf 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -1,6 +1,11 @@ # !/usr/bin/env python # encoding: utf-8 +from django.db.models import Subquery +from django.http import JsonResponse + +from drf_yasg.utils import swagger_auto_schema + from rest_framework import viewsets from rest_framework.decorators import action @@ -9,32 +14,49 @@ from seed.lib.superperms.orgs.decorators import has_perm_class from seed.lib.superperms.orgs.models import Organization -from seed.models.properties import PropertyState -from seed.models.tax_lots import TaxLotState - +from seed.models.properties import PropertyState, PropertyView +from seed.models.tax_lots import TaxLotState, TaxLotView from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper from seed.utils.geocode import geocode_buildings class GeocodeViewSet(viewsets.ViewSet): + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'property_view_ids': ['integer'], + 'taxlot_view_ids': ['integer'], + }, + description='IDs by inventory type for records to be geocoded.' + ) + ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def geocode_by_ids(self, request): body = dict(request.data) - property_ids = body.get('property_ids') - taxlot_ids = body.get('taxlot_ids') - - if property_ids: - properties = PropertyState.objects.filter(id__in=property_ids) + property_view_ids = body.get('property_view_ids') + taxlot_view_ids = body.get('taxlot_view_ids') + + if property_view_ids: + property_views = PropertyView.objects.filter(id__in=property_view_ids) + properties = PropertyState.objects.filter( + id__in=Subquery(property_views.values('state_id')) + ) geocode_buildings(properties) - if taxlot_ids: - taxlots = TaxLotState.objects.filter(id__in=taxlot_ids) + if taxlot_view_ids: + taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlots = TaxLotState.objects.filter( + id__in=Subquery(taxlot_views.values('state_id')) + ) geocode_buildings(taxlots) + return JsonResponse({'status': 'success'}) @ajax_request_class @action(detail=False, methods=['POST']) def confidence_summary(self, request): From f7d54560f0b5bb1a759a6be9c0e75d61a05b382b Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 13:37:29 -0600 Subject: [PATCH 04/28] API v3 geocode - confidence_summary uses View IDs --- seed/views/v3/geocode.py | 43 +++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 97a0f175cf..d67ecea1b0 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -57,59 +57,74 @@ def geocode_by_ids(self, request): geocode_buildings(taxlots) return JsonResponse({'status': 'success'}) + + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'property_view_ids': ['integer'], + 'taxlot_view_ids': ['integer'], + }, + description='IDs by inventory type for records to be used in building a geocoding summary.' + ) + ) + @api_endpoint_class @ajax_request_class + @has_perm_class('can_view_data') @action(detail=False, methods=['POST']) def confidence_summary(self, request): body = dict(request.data) - property_ids = body.get('property_ids') - tax_lot_ids = body.get('tax_lot_ids') + property_view_ids = body.get('property_view_ids') + taxlot_view_ids = body.get('taxlot_view_ids') result = {} - if property_ids: + if property_view_ids: + property_views = PropertyView.objects.filter(id__in=property_view_ids) result["properties"] = { 'not_geocoded': len(PropertyState.objects.filter( - id__in=property_ids, + id__in=Subquery(property_views.values('state_id')), geocoding_confidence__isnull=True )), 'high_confidence': len(PropertyState.objects.filter( - id__in=property_ids, + id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='High' )), 'low_confidence': len(PropertyState.objects.filter( - id__in=property_ids, + id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='Low' )), 'manual': len(PropertyState.objects.filter( - id__in=property_ids, + id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' )), 'missing_address_components': len(PropertyState.objects.filter( - id__in=property_ids, + id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' )), } - if tax_lot_ids: + if taxlot_view_ids: + taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) result["tax_lots"] = { 'not_geocoded': len(TaxLotState.objects.filter( - id__in=tax_lot_ids, + id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__isnull=True )), 'high_confidence': len(TaxLotState.objects.filter( - id__in=tax_lot_ids, + id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='High' )), 'low_confidence': len(TaxLotState.objects.filter( - id__in=tax_lot_ids, + id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='Low' )), 'manual': len(TaxLotState.objects.filter( - id__in=tax_lot_ids, + id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' )), 'missing_address_components': len(TaxLotState.objects.filter( - id__in=tax_lot_ids, + id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' )), } From 37c6ea606711def41eea313d293ea19e14794eaf Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 13:38:27 -0600 Subject: [PATCH 05/28] API v3 geocode - confidence_summary uses QS count() vs len() --- seed/views/v3/geocode.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index d67ecea1b0..4da28a2365 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -82,51 +82,51 @@ def confidence_summary(self, request): if property_view_ids: property_views = PropertyView.objects.filter(id__in=property_view_ids) result["properties"] = { - 'not_geocoded': len(PropertyState.objects.filter( + 'not_geocoded': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__isnull=True - )), - 'high_confidence': len(PropertyState.objects.filter( + ), + 'high_confidence': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='High' - )), - 'low_confidence': len(PropertyState.objects.filter( + ), + 'low_confidence': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='Low' - )), - 'manual': len(PropertyState.objects.filter( + ), + 'manual': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' - )), - 'missing_address_components': len(PropertyState.objects.filter( + ), + 'missing_address_components': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' - )), + ), } if taxlot_view_ids: taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) result["tax_lots"] = { - 'not_geocoded': len(TaxLotState.objects.filter( + 'not_geocoded': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__isnull=True - )), - 'high_confidence': len(TaxLotState.objects.filter( + ), + 'high_confidence': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='High' - )), - 'low_confidence': len(TaxLotState.objects.filter( + ), + 'low_confidence': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='Low' - )), - 'manual': len(TaxLotState.objects.filter( + ), + 'manual': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' - )), - 'missing_address_components': len(TaxLotState.objects.filter( + ), + 'missing_address_components': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' - )), + ), } return result From e3884c14f74c821b35ccc7580b7bceed6e7a80d8 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 13:48:31 -0600 Subject: [PATCH 06/28] API v3 geocode - deprecate api_key_exists, dedup OrgViewSet version --- seed/views/v3/geocode.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 4da28a2365..860e4ed3b9 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -130,14 +130,3 @@ def confidence_summary(self, request): } return result - - @ajax_request_class - @action(detail=False, methods=['GET']) - def api_key_exists(self, request): - org_id = request.GET.get("organization_id") - org = Organization.objects.get(id=org_id) - - if org.mapquest_api_key: - return True - else: - return False From 5483c0989cb58dc8bf2911d8ebf48c2b8707e100 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 14:03:14 -0600 Subject: [PATCH 07/28] API v3 ubid - copy v2 endpoint to v3 --- seed/api/v3/urls.py | 2 ++ seed/views/v3/ubid.py | 73 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 seed/views/v3/ubid.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 896164c5ae..5577ae69df 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -23,6 +23,7 @@ from seed.views.v3.organization_users import OrganizationUserViewSet from seed.views.v3.properties import PropertyViewSet from seed.views.v3.taxlots import TaxlotViewSet +from seed.views.v3.ubid import UbidViews from seed.views.v3.users import UserViewSet api_v3_router = routers.DefaultRouter() @@ -39,6 +40,7 @@ api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations') api_v3_router.register(r'properties', PropertyViewSet, base_name='properties') api_v3_router.register(r'taxlots', TaxlotViewSet, base_name='taxlots') +api_v3_router.register(r'ubid', UbidViews, base_name='ubid') api_v3_router.register(r'users', UserViewSet, base_name='user') data_quality_checks_router = nested_routers.NestedSimpleRouter(api_v3_router, r'data_quality_checks', lookup="nested") diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py new file mode 100644 index 0000000000..ebcf452b5b --- /dev/null +++ b/seed/views/v3/ubid.py @@ -0,0 +1,73 @@ +# !/usr/bin/env python +# encoding: utf-8 + +from rest_framework import viewsets +from rest_framework.decorators import action + +from seed.decorators import ajax_request_class +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.models.properties import PropertyState +from seed.models.tax_lots import TaxLotState +from seed.utils.api import api_endpoint_class +from seed.utils.ubid import decode_unique_ids + + +class UbidViews(viewsets.ViewSet): + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=False, methods=['POST']) + def decode_by_ids(self, request): + body = dict(request.data) + property_ids = body.get('property_ids') + taxlot_ids = body.get('taxlot_ids') + + if property_ids: + properties = PropertyState.objects.filter(id__in=property_ids) + decode_unique_ids(properties) + + if taxlot_ids: + taxlots = TaxLotState.objects.filter(id__in=taxlot_ids) + decode_unique_ids(taxlots) + + @ajax_request_class + @action(detail=False, methods=['POST']) + def decode_results(self, request): + body = dict(request.data) + + ubid_unpopulated = 0 + ubid_successfully_decoded = 0 + ubid_not_decoded = 0 + ulid_unpopulated = 0 + ulid_successfully_decoded = 0 + ulid_not_decoded = 0 + property_ids = body.get('property_ids') + taxlot_ids = body.get('taxlot_ids') + if property_ids: + properties = PropertyState.objects.filter(id__in=property_ids) + ubid_unpopulated = len(properties.filter(ubid__isnull=True)) + ubid_successfully_decoded = len(properties.filter(ubid__isnull=False, + bounding_box__isnull=False, + centroid__isnull=False)) + # for ubid_not_decoded, bounding_box could be populated from a GeoJSON import + ubid_not_decoded = len(properties.filter(ubid__isnull=False, centroid__isnull=True)) + + if taxlot_ids: + taxlots = TaxLotState.objects.filter(id__in=taxlot_ids) + ulid_unpopulated = len(taxlots.filter(ulid__isnull=True)) + ulid_successfully_decoded = len(taxlots.filter(ulid__isnull=False, + bounding_box__isnull=False, + centroid__isnull=False)) + # for ulid_not_decoded, bounding_box could be populated from a GeoJSON import + ulid_not_decoded = len(taxlots.filter(ulid__isnull=False, centroid__isnull=True)) + + result = { + "ubid_unpopulated": ubid_unpopulated, + "ubid_successfully_decoded": ubid_successfully_decoded, + "ubid_not_decoded": ubid_not_decoded, + "ulid_unpopulated": ulid_unpopulated, + "ulid_successfully_decoded": ulid_successfully_decoded, + "ulid_not_decoded": ulid_not_decoded, + } + + return result From bb95a409cb0e52d221080507be01bd5ac1cee64e Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 14:15:17 -0600 Subject: [PATCH 08/28] API v3 ubid - rename class to have "ViewSet" --- seed/api/v3/urls.py | 4 ++-- seed/views/v3/ubid.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 5577ae69df..bc682feacb 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -23,7 +23,7 @@ from seed.views.v3.organization_users import OrganizationUserViewSet from seed.views.v3.properties import PropertyViewSet from seed.views.v3.taxlots import TaxlotViewSet -from seed.views.v3.ubid import UbidViews +from seed.views.v3.ubid import UbidViewSet from seed.views.v3.users import UserViewSet api_v3_router = routers.DefaultRouter() @@ -40,7 +40,7 @@ api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations') api_v3_router.register(r'properties', PropertyViewSet, base_name='properties') api_v3_router.register(r'taxlots', TaxlotViewSet, base_name='taxlots') -api_v3_router.register(r'ubid', UbidViews, base_name='ubid') +api_v3_router.register(r'ubid', UbidViewSet, base_name='ubid') api_v3_router.register(r'users', UserViewSet, base_name='user') data_quality_checks_router = nested_routers.NestedSimpleRouter(api_v3_router, r'data_quality_checks', lookup="nested") diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py index ebcf452b5b..b47a7be04e 100644 --- a/seed/views/v3/ubid.py +++ b/seed/views/v3/ubid.py @@ -12,7 +12,7 @@ from seed.utils.ubid import decode_unique_ids -class UbidViews(viewsets.ViewSet): +class UbidViewSet(viewsets.ViewSet): @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') From 7c0406e4bda4f1c809d84a82c61031f7ad93a7c1 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 14:38:36 -0600 Subject: [PATCH 09/28] API v3 ubid - decode_by_ids shift to View IDs --- seed/views/v3/ubid.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py index b47a7be04e..c0be4d38f0 100644 --- a/seed/views/v3/ubid.py +++ b/seed/views/v3/ubid.py @@ -1,6 +1,8 @@ # !/usr/bin/env python # encoding: utf-8 +from drf_yasg.utils import swagger_auto_schema + from rest_framework import viewsets from rest_framework.decorators import action @@ -9,27 +11,46 @@ from seed.models.properties import PropertyState from seed.models.tax_lots import TaxLotState from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper from seed.utils.ubid import decode_unique_ids class UbidViewSet(viewsets.ViewSet): + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'property_view_ids': ['integer'], + 'taxlot_view_ids': ['integer'], + }, + description='IDs by inventory type for records to have their UBID decoded.' + ) + ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def decode_by_ids(self, request): body = dict(request.data) - property_ids = body.get('property_ids') - taxlot_ids = body.get('taxlot_ids') + property_view_ids = body.get('property_view_ids') + taxlot_view_ids = body.get('taxlot_view_ids') - if property_ids: - properties = PropertyState.objects.filter(id__in=property_ids) + if property_view_ids: + property_views = PropertyView.objects.filter(id__in=property_view_ids) + properties = PropertyState.objects.filter( + id__in=Subquery(property_views.values('state_id')) + ) decode_unique_ids(properties) - if taxlot_ids: - taxlots = TaxLotState.objects.filter(id__in=taxlot_ids) + if taxlot_view_ids: + taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlots = TaxLotState.objects.filter( + id__in=Subquery(taxlot_views.values('state_id')) + ) decode_unique_ids(taxlots) + return JsonResponse({'status': 'success'}) + @ajax_request_class @action(detail=False, methods=['POST']) def decode_results(self, request): From d4efee151457a8946e9c78f1512f7d36efc1b2b5 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 15:06:44 -0600 Subject: [PATCH 10/28] API v3 ubid - decode_results shift to View IDs and add perm --- seed/views/v3/ubid.py | 46 ++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py index c0be4d38f0..4671a657ee 100644 --- a/seed/views/v3/ubid.py +++ b/seed/views/v3/ubid.py @@ -1,6 +1,9 @@ # !/usr/bin/env python # encoding: utf-8 +from django.db.models import Subquery +from django.http import JsonResponse + from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets @@ -8,8 +11,8 @@ from seed.decorators import ajax_request_class from seed.lib.superperms.orgs.decorators import has_perm_class -from seed.models.properties import PropertyState -from seed.models.tax_lots import TaxLotState +from seed.models.properties import PropertyState, PropertyView +from seed.models.tax_lots import TaxLotState, TaxLotView from seed.utils.api import api_endpoint_class from seed.utils.api_schema import AutoSchemaHelper from seed.utils.ubid import decode_unique_ids @@ -51,7 +54,18 @@ def decode_by_ids(self, request): return JsonResponse({'status': 'success'}) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'property_view_ids': ['integer'], + 'taxlot_view_ids': ['integer'], + }, + description='IDs by inventory type for records to be used in building a UBID decoding summary.' + ) + ) @ajax_request_class + @has_perm_class('can_view_data') @action(detail=False, methods=['POST']) def decode_results(self, request): body = dict(request.data) @@ -62,25 +76,29 @@ def decode_results(self, request): ulid_unpopulated = 0 ulid_successfully_decoded = 0 ulid_not_decoded = 0 - property_ids = body.get('property_ids') - taxlot_ids = body.get('taxlot_ids') - if property_ids: - properties = PropertyState.objects.filter(id__in=property_ids) - ubid_unpopulated = len(properties.filter(ubid__isnull=True)) - ubid_successfully_decoded = len(properties.filter(ubid__isnull=False, + property_view_ids = body.get('property_view_ids') + taxlot_view_ids = body.get('taxlot_view_ids') + if property_view_ids: + property_views = PropertyView.objects.filter(id__in=property_view_ids) + property_states = PropertyState.objects.filter(id__in=Subquery(property_views.values('state_id'))) + + ubid_unpopulated = len(property_states.filter(ubid__isnull=True)) + ubid_successfully_decoded = len(property_states.filter(ubid__isnull=False, bounding_box__isnull=False, centroid__isnull=False)) # for ubid_not_decoded, bounding_box could be populated from a GeoJSON import - ubid_not_decoded = len(properties.filter(ubid__isnull=False, centroid__isnull=True)) + ubid_not_decoded = len(property_states.filter(ubid__isnull=False, centroid__isnull=True)) + + if taxlot_view_ids: + taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlot_states = TaxLotState.objects.filter(id__in=Subquery(taxlot_views.values('state_id'))) - if taxlot_ids: - taxlots = TaxLotState.objects.filter(id__in=taxlot_ids) - ulid_unpopulated = len(taxlots.filter(ulid__isnull=True)) - ulid_successfully_decoded = len(taxlots.filter(ulid__isnull=False, + ulid_unpopulated = len(taxlot_states.filter(ulid__isnull=True)) + ulid_successfully_decoded = len(taxlot_states.filter(ulid__isnull=False, bounding_box__isnull=False, centroid__isnull=False)) # for ulid_not_decoded, bounding_box could be populated from a GeoJSON import - ulid_not_decoded = len(taxlots.filter(ulid__isnull=False, centroid__isnull=True)) + ulid_not_decoded = len(taxlot_states.filter(ulid__isnull=False, centroid__isnull=True)) result = { "ubid_unpopulated": ubid_unpopulated, From b626171d13f0f1a864bfcc81e5d1fa317970c834 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 15:14:59 -0600 Subject: [PATCH 11/28] API v3 ubid and geocode - add org_id to record filters --- seed/views/v3/geocode.py | 26 ++++++++++++++++++++------ seed/views/v3/ubid.py | 26 ++++++++++++++++++++------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 860e4ed3b9..6a51c19557 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -16,12 +16,12 @@ from seed.models.properties import PropertyState, PropertyView from seed.models.tax_lots import TaxLotState, TaxLotView -from seed.utils.api import api_endpoint_class +from seed.utils.api import api_endpoint_class, OrgMixin from seed.utils.api_schema import AutoSchemaHelper from seed.utils.geocode import geocode_buildings -class GeocodeViewSet(viewsets.ViewSet): +class GeocodeViewSet(viewsets.ViewSet, OrgMixin): @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], @@ -39,18 +39,25 @@ class GeocodeViewSet(viewsets.ViewSet): @action(detail=False, methods=['POST']) def geocode_by_ids(self, request): body = dict(request.data) + org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') if property_view_ids: - property_views = PropertyView.objects.filter(id__in=property_view_ids) + property_views = PropertyView.objects.filter( + id__in=property_view_ids, + cycle__organization_id=org_id + ) properties = PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')) ) geocode_buildings(properties) if taxlot_view_ids: - taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlot_views = TaxLotView.objects.filter( + id__in=taxlot_view_ids, + cycle__organization_id=org_id + ) taxlots = TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')) ) @@ -74,13 +81,17 @@ def geocode_by_ids(self, request): @action(detail=False, methods=['POST']) def confidence_summary(self, request): body = dict(request.data) + org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') result = {} if property_view_ids: - property_views = PropertyView.objects.filter(id__in=property_view_ids) + property_views = PropertyView.objects.filter( + id__in=property_view_ids, + cycle__organization_id=org_id + ) result["properties"] = { 'not_geocoded': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), @@ -105,7 +116,10 @@ def confidence_summary(self, request): } if taxlot_view_ids: - taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlot_views = TaxLotView.objects.filter( + id__in=taxlot_view_ids, + cycle__organization_id=org_id + ) result["tax_lots"] = { 'not_geocoded': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py index 4671a657ee..6ba81a00e7 100644 --- a/seed/views/v3/ubid.py +++ b/seed/views/v3/ubid.py @@ -13,12 +13,12 @@ from seed.lib.superperms.orgs.decorators import has_perm_class from seed.models.properties import PropertyState, PropertyView from seed.models.tax_lots import TaxLotState, TaxLotView -from seed.utils.api import api_endpoint_class +from seed.utils.api import api_endpoint_class, OrgMixin from seed.utils.api_schema import AutoSchemaHelper from seed.utils.ubid import decode_unique_ids -class UbidViewSet(viewsets.ViewSet): +class UbidViewSet(viewsets.ViewSet, OrgMixin): @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( @@ -35,18 +35,25 @@ class UbidViewSet(viewsets.ViewSet): @action(detail=False, methods=['POST']) def decode_by_ids(self, request): body = dict(request.data) + org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') if property_view_ids: - property_views = PropertyView.objects.filter(id__in=property_view_ids) + property_views = PropertyView.objects.filter( + id__in=property_view_ids, + cycle__organization_id=org_id + ) properties = PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')) ) decode_unique_ids(properties) if taxlot_view_ids: - taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlot_views = TaxLotView.objects.filter( + id__in=taxlot_view_ids, + cycle__organization_id=org_id + ) taxlots = TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')) ) @@ -69,6 +76,7 @@ def decode_by_ids(self, request): @action(detail=False, methods=['POST']) def decode_results(self, request): body = dict(request.data) + org_id = self.get_organization(request) ubid_unpopulated = 0 ubid_successfully_decoded = 0 @@ -79,7 +87,10 @@ def decode_results(self, request): property_view_ids = body.get('property_view_ids') taxlot_view_ids = body.get('taxlot_view_ids') if property_view_ids: - property_views = PropertyView.objects.filter(id__in=property_view_ids) + property_views = PropertyView.objects.filter( + id__in=property_view_ids, + cycle__organization_id=org_id + ) property_states = PropertyState.objects.filter(id__in=Subquery(property_views.values('state_id'))) ubid_unpopulated = len(property_states.filter(ubid__isnull=True)) @@ -90,7 +101,10 @@ def decode_results(self, request): ubid_not_decoded = len(property_states.filter(ubid__isnull=False, centroid__isnull=True)) if taxlot_view_ids: - taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids) + taxlot_views = TaxLotView.objects.filter( + id__in=taxlot_view_ids, + cycle__organization_id=org_id + ) taxlot_states = TaxLotState.objects.filter(id__in=Subquery(taxlot_views.values('state_id'))) ulid_unpopulated = len(taxlot_states.filter(ulid__isnull=True)) From d32c5e25e2043c7c303a1c8bd4535b6e139854ff Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 15:17:37 -0600 Subject: [PATCH 12/28] API v3 geocode - confidence_summary actually use count() --- seed/views/v3/geocode.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 6a51c19557..2afd515ab8 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -96,23 +96,23 @@ def confidence_summary(self, request): 'not_geocoded': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__isnull=True - ), + ).count(), 'high_confidence': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='High' - ), + ).count(), 'low_confidence': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence__startswith='Low' - ), + ).count(), 'manual': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' - ), + ).count(), 'missing_address_components': PropertyState.objects.filter( id__in=Subquery(property_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' - ), + ).count(), } if taxlot_view_ids: @@ -124,23 +124,23 @@ def confidence_summary(self, request): 'not_geocoded': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__isnull=True - ), + ).count(), 'high_confidence': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='High' - ), + ).count(), 'low_confidence': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence__startswith='Low' - ), + ).count(), 'manual': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Manually geocoded (N/A)' - ), + ).count(), 'missing_address_components': TaxLotState.objects.filter( id__in=Subquery(taxlot_views.values('state_id')), geocoding_confidence='Missing address components (N/A)' - ), + ).count(), } return result From b772b57c030783d695507e1b0b1013c67526fcfe Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 15:29:24 -0600 Subject: [PATCH 13/28] API v3 ubid - decode_results uses QS count() vs len() --- seed/views/v3/ubid.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py index 6ba81a00e7..8936635165 100644 --- a/seed/views/v3/ubid.py +++ b/seed/views/v3/ubid.py @@ -93,12 +93,17 @@ def decode_results(self, request): ) property_states = PropertyState.objects.filter(id__in=Subquery(property_views.values('state_id'))) - ubid_unpopulated = len(property_states.filter(ubid__isnull=True)) - ubid_successfully_decoded = len(property_states.filter(ubid__isnull=False, - bounding_box__isnull=False, - centroid__isnull=False)) + ubid_unpopulated = property_states.filter(ubid__isnull=True).count() + ubid_successfully_decoded = property_states.filter( + ubid__isnull=False, + bounding_box__isnull=False, + centroid__isnull=False + ).count() # for ubid_not_decoded, bounding_box could be populated from a GeoJSON import - ubid_not_decoded = len(property_states.filter(ubid__isnull=False, centroid__isnull=True)) + ubid_not_decoded = property_states.filter( + ubid__isnull=False, + centroid__isnull=True + ).count() if taxlot_view_ids: taxlot_views = TaxLotView.objects.filter( @@ -107,12 +112,17 @@ def decode_results(self, request): ) taxlot_states = TaxLotState.objects.filter(id__in=Subquery(taxlot_views.values('state_id'))) - ulid_unpopulated = len(taxlot_states.filter(ulid__isnull=True)) - ulid_successfully_decoded = len(taxlot_states.filter(ulid__isnull=False, - bounding_box__isnull=False, - centroid__isnull=False)) + ulid_unpopulated = taxlot_states.filter(ulid__isnull=True).count() + ulid_successfully_decoded = taxlot_states.filter( + ulid__isnull=False, + bounding_box__isnull=False, + centroid__isnull=False + ).count() # for ulid_not_decoded, bounding_box could be populated from a GeoJSON import - ulid_not_decoded = len(taxlot_states.filter(ulid__isnull=False, centroid__isnull=True)) + ulid_not_decoded = taxlot_states.filter( + ulid__isnull=False, + centroid__isnull=True + ).count() result = { "ubid_unpopulated": ubid_unpopulated, From 49c53b7e6f0a7e9a87b0188701f33d90e3cd97e7 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 15:30:17 -0600 Subject: [PATCH 14/28] API v3 geocode - flake8 remove Organzation import --- seed/views/v3/geocode.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index 2afd515ab8..c1170a927e 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -12,7 +12,6 @@ from seed.decorators import ajax_request_class from seed.lib.superperms.orgs.decorators import has_perm_class -from seed.lib.superperms.orgs.models import Organization from seed.models.properties import PropertyState, PropertyView from seed.models.tax_lots import TaxLotState, TaxLotView From 50dabc8147b72fe4021138cd69e1c1e3760258c8 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Mon, 29 Jun 2020 21:28:35 -0600 Subject: [PATCH 15/28] API v3 geocode and ubid - add descriptions to endpoints --- seed/views/v3/geocode.py | 7 +++++++ seed/views/v3/ubid.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/seed/views/v3/geocode.py b/seed/views/v3/geocode.py index c1170a927e..1827d17f0d 100644 --- a/seed/views/v3/geocode.py +++ b/seed/views/v3/geocode.py @@ -37,6 +37,9 @@ class GeocodeViewSet(viewsets.ViewSet, OrgMixin): @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def geocode_by_ids(self, request): + """ + Submit a request to geocode property and tax lot records. + """ body = dict(request.data) org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') @@ -79,6 +82,10 @@ def geocode_by_ids(self, request): @has_perm_class('can_view_data') @action(detail=False, methods=['POST']) def confidence_summary(self, request): + """ + Generate a summary of geocoding confidence values for property and + tax lot records. + """ body = dict(request.data) org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') diff --git a/seed/views/v3/ubid.py b/seed/views/v3/ubid.py index 8936635165..76967b3ee6 100644 --- a/seed/views/v3/ubid.py +++ b/seed/views/v3/ubid.py @@ -34,6 +34,9 @@ class UbidViewSet(viewsets.ViewSet, OrgMixin): @has_perm_class('can_modify_data') @action(detail=False, methods=['POST']) def decode_by_ids(self, request): + """ + Submit a request to decode UBIDs for property and tax lot records. + """ body = dict(request.data) org_id = self.get_organization(request) property_view_ids = body.get('property_view_ids') @@ -75,6 +78,10 @@ def decode_by_ids(self, request): @has_perm_class('can_view_data') @action(detail=False, methods=['POST']) def decode_results(self, request): + """ + Generate a summary of populated, unpopulated, and decoded UBIDs for + property and tax lot records. + """ body = dict(request.data) org_id = self.get_organization(request) From 0ac13ac9de23bd0ef3bc96f97efa05635ca30cf6 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 11:03:27 -0600 Subject: [PATCH 16/28] feat(v3/properties): move some v2.1 views to v3 This does not include v2.1/properties and v2.1/properties/{id} --- seed/views/v3/properties.py | 258 +++++++++++++++++++++++++++++++++++- 1 file changed, 253 insertions(+), 5 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index c79fb90030..3085df3375 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -2,26 +2,29 @@ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author """ +import os from collections import namedtuple from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator -from django.db.models import Q -from django.db.models import Subquery -from django.http import JsonResponse +from django.db.models import Q, Subquery +from django.http import HttpResponse, JsonResponse from drf_yasg.utils import no_body, swagger_auto_schema from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.parsers import JSONParser from rest_framework.renderers import JSONRenderer +from seed.building_sync.building_sync import BuildingSync from seed.data_importer.utils import usage_point_id from seed.decorators import ajax_request_class +from seed.hpxml.hpxml import HPXML from seed.lib.superperms.orgs.decorators import has_perm_class from seed.lib.superperms.orgs.models import Organization from seed.models import (AUDIT_USER_EDIT, DATA_STATE_MATCHING, MERGE_STATE_DELETE, MERGE_STATE_MERGED, MERGE_STATE_NEW, VIEW_LIST, VIEW_LIST_PROPERTY, - Column, ColumnListSetting, ColumnListSettingColumn, - Cycle, Meter, Note, Property, PropertyAuditLog, + BuildingFile, Column, ColumnListSetting, + ColumnListSettingColumn, ColumnMappingPreset, Cycle, + Meter, Note, Property, PropertyAuditLog, PropertyMeasure, PropertyState, PropertyView, Simulation) from seed.models import StatusLabel as Label @@ -30,6 +33,7 @@ apply_display_unit_preferences) from seed.serializers.properties import (PropertySerializer, PropertyStateSerializer, + PropertyViewAsStateSerializer, PropertyViewSerializer, UpdatePropertyPayloadSerializer) from seed.serializers.taxlots import TaxLotViewSerializer @@ -1010,6 +1014,250 @@ def update(self, request, pk=None): else: return JsonResponse(result, status=status.HTTP_404_NOT_FOUND) + def _get_property_view_for_property(self, pk, cycle_pk): + """ + Return a property view based on the property id and cycle + :param pk: ID of property (not property view) + :param cycle_pk: ID of the cycle + :return: dict, propety view and status + """ + try: + property_view = PropertyView.objects.select_related( + 'property', 'cycle', 'state' + ).get( + property_id=pk, + cycle_id=cycle_pk, + property__organization_id=self.request.GET['organization_id'] + ) + result = { + 'status': 'success', + 'property_view': property_view + } + except PropertyView.DoesNotExist: + result = { + 'status': 'error', + 'message': 'property view with property id {} does not exist'.format(pk) + } + except PropertyView.MultipleObjectsReturned: + result = { + 'status': 'error', + 'message': 'Multiple property views with id {}'.format(pk) + } + return result + + @action(detail=True, methods=['GET']) + def building_sync(self, request, pk): + """ + Return BuildingSync representation of the property + + --- + parameters: + - name: pk + description: The PropertyView to return the BuildingSync file + type: path + required: true + - name: organization_id + type: integer + required: true + paramType: query + """ + preset_pk = request.GET.get('preset_id') + try: + preset_pk = int(preset_pk) + column_mapping_preset = ColumnMappingPreset.objects.get( + pk=preset_pk, + preset_type__in=[ColumnMappingPreset.BUILDINGSYNC_DEFAULT, ColumnMappingPreset.BUILDINGSYNC_CUSTOM]) + except TypeError: + return JsonResponse({ + 'success': False, + 'message': 'Query param `preset_id` is either missing or invalid' + }, status=status.HTTP_400_BAD_REQUEST) + except ColumnMappingPreset.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': f'Cannot find a BuildingSync ColumnMappingPreset with pk={preset_pk}' + }, status=status.HTTP_400_BAD_REQUEST) + + try: + # TODO: not checking organization? Is that right? + # TODO: this needs to call _get_property_view and use the property pk, not the property_view pk. + # or we need to state the v2.1 of API uses property views instead of property + property_view = PropertyView.objects.select_related('state').get(pk=pk) + except PropertyView.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': 'Cannot match a PropertyView with pk=%s' % pk + }, status=status.HTTP_400_BAD_REQUEST) + + bs = BuildingSync() + # Check if there is an existing BuildingSync XML file to merge + bs_file = property_view.state.building_files.order_by('created').last() + if bs_file is not None and os.path.exists(bs_file.file.path): + bs.import_file(bs_file.file.path) + + try: + xml = bs.export_using_preset(property_view.state, column_mapping_preset.mappings) + return HttpResponse(xml, content_type='application/xml') + except Exception as e: + return JsonResponse({ + 'success': False, + 'message': str(e) + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=True, methods=['GET']) + def hpxml(self, request, pk): + """ + Return HPXML representation of the property + + --- + parameters: + - name: pk + description: The PropertyView to return the HPXML file + type: path + required: true + - name: organization_id + type: integer + required: true + paramType: query + """ + # Organization is checked in the orgfilter of the ViewSet + try: + property_view = PropertyView.objects.select_related('state').get(pk=pk) + except PropertyView.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': 'Cannot match a PropertyView with pk=%s' % pk + }) + + hpxml = HPXML() + # Check if there is an existing BuildingSync XML file to merge + hpxml_file = property_view.state.building_files.last() + if hpxml_file is not None and os.path.exists(hpxml_file.file.path): + hpxml.import_file(hpxml_file.file.path) + xml = hpxml.export(property_view.state) + return HttpResponse(xml, content_type='application/xml') + else: + # create a new XML from the record, do not import existing XML + xml = hpxml.export(property_view.state) + return HttpResponse(xml, content_type='application/xml') + + def _merge_relationships(self, old_state, new_state): + """ + Merge the relationships between the old state and the new state. This is different than the version + in views/properties.py because if this is a buildingsync update, then the buildingsync file may + contain new or removed items that we want to merge. + + :param old_state: PropertyState + :param new_state: PropertyState + :return: PropertyState, updated new_state + """ + # for s in old_state.scenarios.all(): + # s.property_state = new_state + # s.save() + # + # # Move the measures to the new state + # for m in PropertyMeasure.objects.filter(property_state=old_state): + # m.property_state = new_state + # m.save() + # + # # Move the old building file to the new state to preserve the history + # for b in old_state.building_files.all(): + # b.property_state = new_state + # b.save() + # + # for s in Simulation.objects.filter(property_state=old_state): + # s.property_state = new_state + # s.save() + + return new_state + + @action(detail=True, methods=['PUT']) + @has_perm_class('can_modify_data') + def update_with_building_sync(self, request, pk): + """ + Does not work in Swagger! + + Update an existing PropertyView with a building file. Currently only supports BuildingSync. + --- + consumes: + - multipart/form-data + parameters: + - name: pk + description: The PropertyView to update with this buildingsync file + type: path + required: true + - name: organization_id + type: integer + required: true + - name: cycle_id + type: integer + required: true + - name: file_type + type: string + enum: ["Unknown", "BuildingSync"] + required: true + - name: file + description: In-memory file object + required: true + type: file + """ + if len(request.FILES) == 0: + return JsonResponse({ + 'success': False, + 'message': "Must pass file in as a Multipart/Form post" + }) + + the_file = request.data['file'] + file_type = BuildingFile.str_to_file_type(request.data.get('file_type', 'Unknown')) + organization_id = request.query_params.get('organization_id', None) + cycle_pk = request.query_params.get('cycle_id', None) + + if not cycle_pk: + return JsonResponse({ + 'success': False, + 'message': "Cycle ID is not defined" + }) + else: + cycle = Cycle.objects.get(pk=cycle_pk) + + result = self._get_property_view_for_property(pk, cycle_pk) + p_status = False + new_pv_state = None + if result.get('status', None) != 'error': + building_file = BuildingFile.objects.create( + file=the_file, + filename=the_file.name, + file_type=file_type, + ) + + property_view = result.pop('property_view') + previous_state = property_view.state + # passing in the existing propertyview allows it to process the buildingsync file and attach it to the + # existing propertyview. + p_status, new_pv_state, new_pv_view, messages = building_file.process( + organization_id, cycle, property_view=property_view + ) + + # merge the relationships from the old property state + self._merge_relationships(previous_state, new_pv_state) + + else: + messages = ['Cannot match a PropertyView with property_id=%s; cycle_id=%s' % (pk, cycle_pk)] + + if p_status and new_pv_state: + return JsonResponse({ + 'success': True, + 'status': 'success', + 'message': 'successfully imported file', + 'data': { + 'property_view': PropertyViewAsStateSerializer(new_pv_view).data, + }, + }) + else: + return JsonResponse({ + 'status': 'error', + 'message': "Could not process building file with messages {}".format(messages) + }, status=status.HTTP_400_BAD_REQUEST) def diffupdate(old, new): """Returns lists of fields changed""" From 722e5cb041b4bae94109cc7e794deda12a7b96a1 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 11:53:39 -0600 Subject: [PATCH 17/28] chore(v3/properties): add swagger docs to update_with_building_sync --- seed/utils/api_schema.py | 20 ++++++++++++++ seed/views/v3/properties.py | 52 ++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 87b287b433..884b995ac8 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -81,6 +81,26 @@ def query_boolean_field(name, required, description): type=openapi.TYPE_BOOLEAN ) + @staticmethod + def form_string_field(name, required, description): + return openapi.Parameter( + name, + openapi.IN_FORM, + description=description, + required=required, + type=openapi.TYPE_STRING + ) + + @staticmethod + def upload_file_field(name, required, description): + return openapi.Parameter( + name, + openapi.IN_FORM, + description=description, + required=required, + type=openapi.TYPE_FILE + ) + @staticmethod def path_id_field(description): return openapi.Parameter( diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 3085df3375..72eaa0c6cb 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -11,7 +11,7 @@ from drf_yasg.utils import no_body, swagger_auto_schema from rest_framework import status, viewsets from rest_framework.decorators import action -from rest_framework.parsers import JSONParser +from rest_framework.parsers import JSONParser, MultiPartParser from rest_framework.renderers import JSONRenderer from seed.building_sync.building_sync import BuildingSync from seed.data_importer.utils import usage_point_id @@ -1171,35 +1171,35 @@ def _merge_relationships(self, old_state, new_state): return new_state - @action(detail=True, methods=['PUT']) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.path_id_field( + description='ID of the property (not property view) to update' + ), + AutoSchemaHelper.query_org_id_field(), + AutoSchemaHelper.query_integer_field( + 'cycle_id', + required=True, + description='ID of the cycle to ' + ), + AutoSchemaHelper.upload_file_field( + 'file', + required=True, + description='BuildingSync file to use', + ), + AutoSchemaHelper.form_string_field( + 'file_type', + required=True, + description='Either "Unknown" or "BuildingSync"', + ), + ], + request_body=no_body, + ) + @action(detail=True, methods=['PUT'], parser_classes=(MultiPartParser,)) @has_perm_class('can_modify_data') def update_with_building_sync(self, request, pk): """ - Does not work in Swagger! - Update an existing PropertyView with a building file. Currently only supports BuildingSync. - --- - consumes: - - multipart/form-data - parameters: - - name: pk - description: The PropertyView to update with this buildingsync file - type: path - required: true - - name: organization_id - type: integer - required: true - - name: cycle_id - type: integer - required: true - - name: file_type - type: string - enum: ["Unknown", "BuildingSync"] - required: true - - name: file - description: In-memory file object - required: true - type: file """ if len(request.FILES) == 0: return JsonResponse({ From 31a00cead78ae0bffe7ebe30cd22614a06186b03 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 12:05:58 -0600 Subject: [PATCH 18/28] chore(v3/properties): add swagger docs for building_sync and hpxml --- seed/views/v3/properties.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 72eaa0c6cb..1a553ed2b7 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1045,21 +1045,21 @@ def _get_property_view_for_property(self, pk, cycle_pk): } return result + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_org_id_field(), + AutoSchemaHelper.query_integer_field( + 'preset_id', + required=True, + description='ID of a BuildingSync ColumnMappingPreset' + ), + ] + ) + @has_perm_class('can_view_data') @action(detail=True, methods=['GET']) def building_sync(self, request, pk): """ Return BuildingSync representation of the property - - --- - parameters: - - name: pk - description: The PropertyView to return the BuildingSync file - type: path - required: true - - name: organization_id - type: integer - required: true - paramType: query """ preset_pk = request.GET.get('preset_id') try: @@ -1104,21 +1104,14 @@ def building_sync(self, request, pk): 'message': str(e) }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()] + ) + @has_perm_class('can_view_data') @action(detail=True, methods=['GET']) def hpxml(self, request, pk): """ Return HPXML representation of the property - - --- - parameters: - - name: pk - description: The PropertyView to return the HPXML file - type: path - required: true - - name: organization_id - type: integer - required: true - paramType: query """ # Organization is checked in the orgfilter of the ViewSet try: From dd3cbb2b1803cadb7d65a3ad39c2de76acb66817 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 13:09:50 -0600 Subject: [PATCH 19/28] feat(v3/property_scenarios): move v2.1 scenario views to v3 --- seed/api/v3/urls.py | 5 +++++ seed/views/v3/property_scenarios.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 seed/views/v3/property_scenarios.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index bc682feacb..fe77113dad 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -22,6 +22,7 @@ from seed.views.v3.organizations import OrganizationViewSet from seed.views.v3.organization_users import OrganizationUserViewSet from seed.views.v3.properties import PropertyViewSet +from seed.views.v3.property_scenarios import PropertyScenarioViewSet from seed.views.v3.taxlots import TaxlotViewSet from seed.views.v3.ubid import UbidViewSet from seed.views.v3.users import UserViewSet @@ -51,6 +52,9 @@ taxlots_router = nested_routers.NestedSimpleRouter(api_v3_router, r'taxlots', lookup='taxlots') taxlots_router.register(r'notes', NoteViewSet, base_name='taxlot-notes') +scenarios_router = nested_routers.NestedSimpleRouter(api_v3_router, r'properties', lookup='property') +scenarios_router.register(r'scenarios', PropertyScenarioViewSet, base_name='property-scenarios') + urlpatterns = [ url(r'^', include(api_v3_router.urls)), url(r'^', include(data_quality_checks_router.urls)), @@ -65,5 +69,6 @@ {'inventory_type': 'taxlot'}, ), url(r'^', include(organizations_router.urls)), + url(r'^', include(scenarios_router.urls)), url(r'^', include(taxlots_router.urls)), ] diff --git a/seed/views/v3/property_scenarios.py b/seed/views/v3/property_scenarios.py new file mode 100644 index 0000000000..11c85a4f32 --- /dev/null +++ b/seed/views/v3/property_scenarios.py @@ -0,0 +1,32 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +from rest_framework.parsers import JSONParser, FormParser +from rest_framework.renderers import JSONRenderer + +from seed.models import ( + Scenario, +) +from seed.serializers.scenarios import ScenarioSerializer +from seed.utils.viewsets import ( + SEEDOrgReadOnlyModelViewSet +) + + +class PropertyScenarioViewSet(SEEDOrgReadOnlyModelViewSet): + """ + API View for Scenarios. This only includes retrieve and list for now. + """ + serializer_class = ScenarioSerializer + parser_classes = (JSONParser, FormParser,) + renderer_classes = (JSONRenderer,) + queryset = Scenario.objects.all() + pagination_class = None + orgfilter = 'property_state__organization_id' + + def get_queryset(self): + org_id = self.get_organization(self.request) + return Scenario.objects.filter(property_state__organization_id=org_id).order_by('id') From 88e4a08bb18dd7effb6cdedd56e374a4176c9f5f Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 13:34:00 -0600 Subject: [PATCH 20/28] refactor(v3/property_scenarios): use property pk for filtering --- seed/views/v3/properties.py | 1 + seed/views/v3/property_scenarios.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 1a553ed2b7..957f64b560 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1252,6 +1252,7 @@ def update_with_building_sync(self, request, pk): 'message': "Could not process building file with messages {}".format(messages) }, status=status.HTTP_400_BAD_REQUEST) + def diffupdate(old, new): """Returns lists of fields changed""" changed_fields = [] diff --git a/seed/views/v3/property_scenarios.py b/seed/views/v3/property_scenarios.py index 11c85a4f32..77855ec146 100644 --- a/seed/views/v3/property_scenarios.py +++ b/seed/views/v3/property_scenarios.py @@ -29,4 +29,9 @@ class PropertyScenarioViewSet(SEEDOrgReadOnlyModelViewSet): def get_queryset(self): org_id = self.get_organization(self.request) - return Scenario.objects.filter(property_state__organization_id=org_id).order_by('id') + property_view_id = self.kwargs.get('property_pk') + + return Scenario.objects.filter( + property_state__organization_id=org_id, + property_state__propertyview=property_view_id, + ).order_by('id') From d0f510f33269fd8bd67f4f9634f0799a178f02d8 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 13:50:19 -0600 Subject: [PATCH 21/28] feat(v3/properties): add nested view for notes Also, change lookups to singular form for property and taxlot nested routers --- seed/api/v3/urls.py | 11 +++++++---- seed/views/v3/notes.py | 18 +++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index fe77113dad..a7267ca013 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -49,11 +49,14 @@ organizations_router = nested_routers.NestedSimpleRouter(api_v3_router, r'organizations', lookup='organization') organizations_router.register(r'users', OrganizationUserViewSet, base_name='organization-users') -taxlots_router = nested_routers.NestedSimpleRouter(api_v3_router, r'taxlots', lookup='taxlots') + +properties_router = nested_routers.NestedSimpleRouter(api_v3_router, r'properties', lookup='property') +properties_router.register(r'notes', NoteViewSet, base_name='property-notes') +properties_router.register(r'scenarios', PropertyScenarioViewSet, base_name='property-scenarios') + +taxlots_router = nested_routers.NestedSimpleRouter(api_v3_router, r'taxlots', lookup='taxlot') taxlots_router.register(r'notes', NoteViewSet, base_name='taxlot-notes') -scenarios_router = nested_routers.NestedSimpleRouter(api_v3_router, r'properties', lookup='property') -scenarios_router.register(r'scenarios', PropertyScenarioViewSet, base_name='property-scenarios') urlpatterns = [ url(r'^', include(api_v3_router.urls)), @@ -69,6 +72,6 @@ {'inventory_type': 'taxlot'}, ), url(r'^', include(organizations_router.urls)), - url(r'^', include(scenarios_router.urls)), + url(r'^', include(properties_router.urls)), url(r'^', include(taxlots_router.urls)), ] diff --git a/seed/views/v3/notes.py b/seed/views/v3/notes.py index 9bcec4f80e..eef620383c 100644 --- a/seed/views/v3/notes.py +++ b/seed/views/v3/notes.py @@ -44,22 +44,22 @@ class NoteViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): def get_queryset(self): # check if the request is properties or taxlots org_id = self.get_organization(self.request) - if self.kwargs.get('properties_pk', None): - return Note.objects.filter(organization_id=org_id, property_view_id=self.kwargs.get('properties_pk')) - elif self.kwargs.get('taxlots_pk', None): - return Note.objects.filter(organization_id=org_id, taxlot_view_id=self.kwargs.get('taxlots_pk')) + if self.kwargs.get('property_pk', None): + return Note.objects.filter(organization_id=org_id, property_view_id=self.kwargs.get('property_pk')) + elif self.kwargs.get('taxlot_pk', None): + return Note.objects.filter(organization_id=org_id, taxlot_view_id=self.kwargs.get('taxlot_pk')) else: return Note.objects.filter(organization_id=org_id) def perform_create(self, serializer): org_id = self.get_organization(self.request) - if self.kwargs.get('properties_pk', None): + if self.kwargs.get('property_pk', None): serializer.save( - organization_id=org_id, user=self.request.user, property_view_id=self.kwargs.get('properties_pk', None) + organization_id=org_id, user=self.request.user, property_view_id=self.kwargs.get('property_pk', None) ) - elif self.kwargs.get('taxlots_pk', None): + elif self.kwargs.get('taxlot_pk', None): serializer.save( - organization_id=org_id, user=self.request.user, taxlot_view_id=self.kwargs.get('taxlots_pk', None) + organization_id=org_id, user=self.request.user, taxlot_view_id=self.kwargs.get('taxlot_pk', None) ) else: - _log.warn("Unable to create model without a property_pk or taxlots_pk") + _log.warn("Unable to create model without a property_pk or taxlot_pk") From 57c17e7ea7cd59103e3e059fc2a518b49b4d6faf Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 14:01:08 -0600 Subject: [PATCH 22/28] refactor(v3/properties): remove vain merge_relationships method --- seed/views/v3/properties.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 957f64b560..5e66bcb59e 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1134,36 +1134,6 @@ def hpxml(self, request, pk): xml = hpxml.export(property_view.state) return HttpResponse(xml, content_type='application/xml') - def _merge_relationships(self, old_state, new_state): - """ - Merge the relationships between the old state and the new state. This is different than the version - in views/properties.py because if this is a buildingsync update, then the buildingsync file may - contain new or removed items that we want to merge. - - :param old_state: PropertyState - :param new_state: PropertyState - :return: PropertyState, updated new_state - """ - # for s in old_state.scenarios.all(): - # s.property_state = new_state - # s.save() - # - # # Move the measures to the new state - # for m in PropertyMeasure.objects.filter(property_state=old_state): - # m.property_state = new_state - # m.save() - # - # # Move the old building file to the new state to preserve the history - # for b in old_state.building_files.all(): - # b.property_state = new_state - # b.save() - # - # for s in Simulation.objects.filter(property_state=old_state): - # s.property_state = new_state - # s.save() - - return new_state - @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.path_id_field( @@ -1224,16 +1194,12 @@ def update_with_building_sync(self, request, pk): ) property_view = result.pop('property_view') - previous_state = property_view.state # passing in the existing propertyview allows it to process the buildingsync file and attach it to the # existing propertyview. p_status, new_pv_state, new_pv_view, messages = building_file.process( organization_id, cycle, property_view=property_view ) - # merge the relationships from the old property state - self._merge_relationships(previous_state, new_pv_state) - else: messages = ['Cannot match a PropertyView with property_id=%s; cycle_id=%s' % (pk, cycle_pk)] From 29dbf7fc6256b2e49bcbd1b13de5c7ddf7aa4131 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 29 Jun 2020 14:27:36 -0600 Subject: [PATCH 23/28] refactor(v3/properties)!: change path id to property view for update_with_building_sync --- seed/views/v3/properties.py | 50 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 5e66bcb59e..e9adb9a3c9 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1137,13 +1137,13 @@ def hpxml(self, request, pk): @swagger_auto_schema( manual_parameters=[ AutoSchemaHelper.path_id_field( - description='ID of the property (not property view) to update' + description='ID of the property view to update' ), AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( 'cycle_id', required=True, - description='ID of the cycle to ' + description='ID of the cycle of the property view' ), AutoSchemaHelper.upload_file_field( 'file', @@ -1175,33 +1175,37 @@ def update_with_building_sync(self, request, pk): organization_id = request.query_params.get('organization_id', None) cycle_pk = request.query_params.get('cycle_id', None) - if not cycle_pk: + try: + cycle = Cycle.objects.get(pk=cycle_pk) + except Cycle.DoesNotExist: return JsonResponse({ 'success': False, - 'message': "Cycle ID is not defined" - }) - else: - cycle = Cycle.objects.get(pk=cycle_pk) + 'message': "Cycle ID is missing or Cycle does not exist" + }, status=status.HTTP_404_NOT_FOUND) + + try: + property_view = PropertyView.objects.select_related( + 'property', 'cycle', 'state' + ).get(pk=pk) + except PropertyView.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'property view does not exist' + }, status=status.HTTP_404_NOT_FOUND) - result = self._get_property_view_for_property(pk, cycle_pk) p_status = False new_pv_state = None - if result.get('status', None) != 'error': - building_file = BuildingFile.objects.create( - file=the_file, - filename=the_file.name, - file_type=file_type, - ) - - property_view = result.pop('property_view') - # passing in the existing propertyview allows it to process the buildingsync file and attach it to the - # existing propertyview. - p_status, new_pv_state, new_pv_view, messages = building_file.process( - organization_id, cycle, property_view=property_view - ) + building_file = BuildingFile.objects.create( + file=the_file, + filename=the_file.name, + file_type=file_type, + ) - else: - messages = ['Cannot match a PropertyView with property_id=%s; cycle_id=%s' % (pk, cycle_pk)] + # passing in the existing propertyview allows it to process the buildingsync file and attach it to the + # existing propertyview. + p_status, new_pv_state, new_pv_view, messages = building_file.process( + organization_id, cycle, property_view=property_view + ) if p_status and new_pv_state: return JsonResponse({ From 9f179b2132b2c31f99f7795f8d2f031bdc0d6eec Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 30 Jun 2020 09:13:42 -0600 Subject: [PATCH 24/28] refactor(v3/properties): make building_sync view fiter with org id --- seed/views/v3/properties.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index e9adb9a3c9..8ca8f8555e 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1062,6 +1062,7 @@ def building_sync(self, request, pk): Return BuildingSync representation of the property """ preset_pk = request.GET.get('preset_id') + org_id=self.get_organization(self.request) try: preset_pk = int(preset_pk) column_mapping_preset = ColumnMappingPreset.objects.get( @@ -1079,10 +1080,8 @@ def building_sync(self, request, pk): }, status=status.HTTP_400_BAD_REQUEST) try: - # TODO: not checking organization? Is that right? - # TODO: this needs to call _get_property_view and use the property pk, not the property_view pk. - # or we need to state the v2.1 of API uses property views instead of property - property_view = PropertyView.objects.select_related('state').get(pk=pk) + property_view = (PropertyView.objects.select_related('state') + .get(pk=pk, cycle__organization_id=org_id)) except PropertyView.DoesNotExist: return JsonResponse({ 'success': False, From 149e44891fbb17de9f9c8c4c7722a945c3db8e07 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 30 Jun 2020 09:15:52 -0600 Subject: [PATCH 25/28] refactor(v3/properties): make hpxml filter with org id --- seed/views/v3/properties.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 8ca8f8555e..4e3b0da747 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1112,9 +1112,10 @@ def hpxml(self, request, pk): """ Return HPXML representation of the property """ - # Organization is checked in the orgfilter of the ViewSet + org_id = self.get_organization(self.request) try: - property_view = PropertyView.objects.select_related('state').get(pk=pk) + property_view = (PropertyView.objects.select_related('state') + .get(pk=pk, cycle__organization_id=org_id)) except PropertyView.DoesNotExist: return JsonResponse({ 'success': False, From 209b1da3d82954119f775b7b3f6cf95dda9a6272 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 30 Jun 2020 09:22:49 -0600 Subject: [PATCH 26/28] refactor(v3/properties): make update_with_buildingsync filter with org id --- seed/views/v3/properties.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 4e3b0da747..9e9dc4906b 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1174,9 +1174,10 @@ def update_with_building_sync(self, request, pk): file_type = BuildingFile.str_to_file_type(request.data.get('file_type', 'Unknown')) organization_id = request.query_params.get('organization_id', None) cycle_pk = request.query_params.get('cycle_id', None) + org_id = self.get_organization(self.request) try: - cycle = Cycle.objects.get(pk=cycle_pk) + cycle = Cycle.objects.get(pk=cycle_pk, organization_id=org_id) except Cycle.DoesNotExist: return JsonResponse({ 'success': False, @@ -1184,9 +1185,11 @@ def update_with_building_sync(self, request, pk): }, status=status.HTTP_404_NOT_FOUND) try: + # note that this is a "safe" query b/c we should have already returned + # if the cycle was not within the user's organization property_view = PropertyView.objects.select_related( 'property', 'cycle', 'state' - ).get(pk=pk) + ).get(pk=pk, cycle_id=cycle_pk) except PropertyView.DoesNotExist: return JsonResponse({ 'status': 'error', From 0e6ca5d3168dc4247b2282edb882debbf4d25c7f Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 30 Jun 2020 09:26:02 -0600 Subject: [PATCH 27/28] chore(v3/property_scenarios): remove queryset attr --- seed/views/v3/property_scenarios.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seed/views/v3/property_scenarios.py b/seed/views/v3/property_scenarios.py index 77855ec146..e3aa3931ad 100644 --- a/seed/views/v3/property_scenarios.py +++ b/seed/views/v3/property_scenarios.py @@ -23,7 +23,6 @@ class PropertyScenarioViewSet(SEEDOrgReadOnlyModelViewSet): serializer_class = ScenarioSerializer parser_classes = (JSONParser, FormParser,) renderer_classes = (JSONRenderer,) - queryset = Scenario.objects.all() pagination_class = None orgfilter = 'property_state__organization_id' From fb38842d70f7af0173bf98bed19576ee3f36052d Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Tue, 30 Jun 2020 10:09:00 -0600 Subject: [PATCH 28/28] chore(v3/properties): flake8 add whitespace around "=" --- seed/views/v3/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 9e9dc4906b..47e7e98334 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1062,7 +1062,7 @@ def building_sync(self, request, pk): Return BuildingSync representation of the property """ preset_pk = request.GET.get('preset_id') - org_id=self.get_organization(self.request) + org_id = self.get_organization(self.request) try: preset_pk = int(preset_pk) column_mapping_preset = ColumnMappingPreset.objects.get(