From f20aad98a9478329f331c0620116e44fa38e7603 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 29 Apr 2020 12:24:12 -0600 Subject: [PATCH 001/135] working on Label endpoint conversion to v3 --- seed/api/v3/urls.py | 4 + seed/views/v3/labels.py | 490 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 seed/views/v3/labels.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index e0cf8e48b1..0f3975b59c 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -5,9 +5,13 @@ from seed.views.v3.data_quality import DataQualityViews +from seed.views.v3.labels import LabelViewSet + api_v3_router = routers.DefaultRouter() api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') +api_v3_router.register(r'labels', LabelViewSet, base_name='labels') + urlpatterns = [ url(r'^', include(api_v3_router.urls)), ] diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py new file mode 100644 index 0000000000..1a17f74c9a --- /dev/null +++ b/seed/views/v3/labels.py @@ -0,0 +1,490 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author 'Piper Merriam ' +""" +from collections import namedtuple + +from django.apps import apps +from django.db import IntegrityError +from django.core.exceptions import ObjectDoesNotExist +from rest_framework import ( + response, + status, + viewsets +) +from rest_framework.decorators import action +from rest_framework.parsers import JSONParser, FormParser +from rest_framework.renderers import JSONRenderer +from rest_framework.views import APIView + +from seed.decorators import DecoratorMixin +from seed.filters import ( + LabelFilterBackend, + InventoryFilterBackend, +) +from seed.models import ( + StatusLabel as Label, + PropertyView, + TaxLotView, +) +from seed.serializers.labels import ( + LabelSerializer, +) +from seed.utils.api import drf_api_endpoint +from seed.utils.api_schema import AutoSchemaHelper + +ErrorState = namedtuple('ErrorState', ['status_code', 'message']) + + +class LabelSchema(AutoSchemaHelper): + def __init__(self, *args): + super().__init__(*args) + + self.manual_fields = { + ('GET', 'get_queryset'): [], + ('POST', 'add_labels'): [ + self.body_field( + name='name', + required=True, + description="Rules information", + params_to_formats={ + 'name': 'string', + 'color': 'string', + 'super_organization': 'integer' + } + ) + ], + ('POST', 'filter'): [self.org_id_field()], + ('GET', 'get_labels'): [self.org_id_field()], + ('PUT', 'put'): [ + self.org_id_field(), + self.body_field( + name='data_quality_rules', + required=True, + description="Rules information" + ) + ], + ('GET', 'results'): [ + self.org_id_field(), + self.query_integer_field( + name='data_quality_id', + required=True, + description="Task ID created when DataQuality task is created." + ), + ], + ('GET', 'csv'): [ + # This will replace the auto-generated field - adds description. + self.path_id_field(description="Import file ID or cache key") + ], + } + + +class LabelViewSet(DecoratorMixin(drf_api_endpoint), viewsets.ModelViewSet): + swagger_schema = LabelSchema + serializer_class = LabelSerializer + renderer_classes = (JSONRenderer,) + parser_classes = (JSONParser, FormParser) + queryset = Label.objects.none() + filter_backends = (LabelFilterBackend,) + pagination_class = None + + _organization = None + + def get_parent_organization(self): + org = self.get_organization() + if org.is_parent: + return org + else: + return org.parent_org + + def get_organization(self): + if self._organization is None: + try: + self._organization = self.request.user.orgs.get( + pk=self.request.query_params["organization_id"], + ) + except (KeyError, ObjectDoesNotExist): + self._organization = self.request.user.orgs.all()[0] + return self._organization + + def get_queryset(self): + labels = Label.objects.filter( + super_organization=self.get_parent_organization() + ).order_by("name").distinct() + return labels + + def get_serializer(self, *args, **kwargs): + kwargs['super_organization'] = self.get_organization() + inventory = InventoryFilterBackend().filter_queryset( + request=self.request, + ) + kwargs['inventory'] = inventory + return super().get_serializer(*args, **kwargs) + + def _get_labels(self, request): + qs = self.get_queryset() + super_organization = self.get_organization() + inventory = InventoryFilterBackend().filter_queryset( + request=self.request, + ) + results = [ + LabelSerializer( + q, + super_organization=super_organization, + inventory=inventory + ).data for q in qs + ] + status_code = status.HTTP_200_OK + return response.Response(results, status=status_code) + + renderer_classes = (JSONRenderer,) + parser_classes = (JSONParser,) + inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} + errors = { + 'disjoint': ErrorState( + status.HTTP_422_UNPROCESSABLE_ENTITY, + 'add_label_ids and remove_label_ids cannot contain elements in common' + ), + 'missing_org': ErrorState( + status.HTTP_422_UNPROCESSABLE_ENTITY, + 'missing organization_id' + ) + } + + @property + def models(self): + """ + Exposes Django's internal models for join table. + + Used for bulk_create operations. + """ + return { + 'property': apps.get_model('seed', 'PropertyView_labels'), + 'taxlot': apps.get_model('seed', 'TaxLotView_labels') + } + + # def get_queryset(self, inventory_type, organization_id): + # Model = self.models[inventory_type] + # return Model.objects.filter( + # statuslabel__super_organization_id=organization_id + # ) + + def get_label_desc(self, add_label_ids, remove_label_ids): + return Label.objects.filter( + pk__in=add_label_ids + remove_label_ids + ).values('id', 'color', 'name') + + def get_inventory_id(self, q, inventory_type): + return getattr(q, "{}view_id".format(inventory_type)) + + def exclude(self, qs, inventory_type, label_ids): + exclude = {label: [] for label in label_ids} + for q in qs: + if q.statuslabel_id in label_ids: + inventory_id = self.get_inventory_id(q, inventory_type) + exclude[q.statuslabel_id].append(inventory_id) + return exclude + + def filter_by_inventory(self, qs, inventory_type, inventory_ids): + if inventory_ids: + filterdict = { + "{}view__pk__in".format(inventory_type): inventory_ids + } + qs = qs.filter(**filterdict) + return qs + + def label_factory(self, inventory_type, label_id, inventory_id): + model = self.models[inventory_type] + + # Ensure the the label org and inventory org are the same + inventory_parent_org_id = getattr(model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id) \ + .cycle.organization.get_parent().id + label_super_org_id = model.statuslabel.get_queryset().get(pk=label_id).super_organization_id + if inventory_parent_org_id == label_super_org_id: + create_dict = { + 'statuslabel_id': label_id, + "{}view_id".format(inventory_type): inventory_id + } + + return model(**create_dict) + else: + raise IntegrityError( + 'Label with super_organization_id={} cannot be applied to a record with parent ' + 'organization_id={}.'.format( + label_super_org_id, + inventory_parent_org_id + ) + ) + + def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids): + added = [] + if add_label_ids: + model = self.models[inventory_type] + inventory_model = self.inventory_models[inventory_type] + exclude = self.exclude(qs, inventory_type, add_label_ids) + inventory_ids = inventory_ids if inventory_ids else [ + m.pk for m in inventory_model.objects.all() + ] + new_inventory_labels = [ + self.label_factory(inventory_type, label_id, pk) + for label_id in add_label_ids for pk in inventory_ids + if pk not in exclude[label_id] + ] + model.objects.bulk_create(new_inventory_labels) + added = [ + self.get_inventory_id(m, inventory_type) + for m in new_inventory_labels + ] + return added + + def remove_labels(self, qs, inventory_type, remove_label_ids): + removed = [] + if remove_label_ids: + rqs = qs.filter( + statuslabel_id__in=remove_label_ids + ) + removed = [self.get_inventory_id(q, inventory_type) for q in rqs] + rqs.delete() + return removed + + def put(self, request, inventory_type): + """ + Updates label assignments to inventory items. + + Payload:: + + { + "add_label_ids": {array} Array of label ids to add + "remove_label_ids": {array} Array of label ids to remove + "inventory_ids": {array} Array property/taxlot ids + "organization_id": {integer} The user's org ID + } + + Returns:: + + { + 'status': {string} 'success' or 'error' + 'message': {string} Error message if error + 'num_updated': {integer} Number of properties/taxlots updated + 'labels': [ List of labels affected. + { + 'color': {string} + 'id': {int} + 'label': {'string'} + 'name': {string} + }... + ] + } + + """ + add_label_ids = request.data.get('add_label_ids', []) + remove_label_ids = request.data.get('remove_label_ids', []) + inventory_ids = request.data.get('inventory_ids', None) + organization_id = request.query_params['organization_id'] + error = None + # ensure add_label_ids and remove_label_ids are different + if not set(add_label_ids).isdisjoint(remove_label_ids): + error = self.errors['disjoint'] + elif not organization_id: + error = self.errors['missing_org'] + if error: + result = { + 'status': 'error', + 'message': str(error) + } + status_code = error.status_code + else: + qs = self.get_queryset(inventory_type, organization_id) + qs = self.filter_by_inventory(qs, inventory_type, inventory_ids) + removed = self.remove_labels(qs, inventory_type, remove_label_ids) + added = self.add_labels(qs, inventory_type, inventory_ids, add_label_ids) + num_updated = len(set(added).union(removed)) + labels = self.get_label_desc(add_label_ids, remove_label_ids) + result = { + 'status': 'success', + 'num_updated': num_updated, + 'labels': labels + } + status_code = status.HTTP_200_OK + return response.Response(result, status=status_code) + + @action(detail=False, methods=['POST']) + def filter(self, request): + return self._get_labels(request) + + def list(self, request): + return self._get_labels(request) + +# class UpdateInventoryLabelsAPIView(APIView): +# renderer_classes = (JSONRenderer,) +# parser_classes = (JSONParser,) +# inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} +# errors = { +# 'disjoint': ErrorState( +# status.HTTP_422_UNPROCESSABLE_ENTITY, +# 'add_label_ids and remove_label_ids cannot contain elements in common' +# ), +# 'missing_org': ErrorState( +# status.HTTP_422_UNPROCESSABLE_ENTITY, +# 'missing organization_id' +# ) +# } +# +# @property +# def models(self): +# """ +# Exposes Django's internal models for join table. +# +# Used for bulk_create operations. +# """ +# return { +# 'property': apps.get_model('seed', 'PropertyView_labels'), +# 'taxlot': apps.get_model('seed', 'TaxLotView_labels') +# } +# +# def get_queryset(self, inventory_type, organization_id): +# Model = self.models[inventory_type] +# return Model.objects.filter( +# statuslabel__super_organization_id=organization_id +# ) +# +# def get_label_desc(self, add_label_ids, remove_label_ids): +# return Label.objects.filter( +# pk__in=add_label_ids + remove_label_ids +# ).values('id', 'color', 'name') +# +# def get_inventory_id(self, q, inventory_type): +# return getattr(q, "{}view_id".format(inventory_type)) +# +# def exclude(self, qs, inventory_type, label_ids): +# exclude = {label: [] for label in label_ids} +# for q in qs: +# if q.statuslabel_id in label_ids: +# inventory_id = self.get_inventory_id(q, inventory_type) +# exclude[q.statuslabel_id].append(inventory_id) +# return exclude +# +# def filter_by_inventory(self, qs, inventory_type, inventory_ids): +# if inventory_ids: +# filterdict = { +# "{}view__pk__in".format(inventory_type): inventory_ids +# } +# qs = qs.filter(**filterdict) +# return qs +# +# def label_factory(self, inventory_type, label_id, inventory_id): +# Model = self.models[inventory_type] +# +# # Ensure the the label org and inventory org are the same +# inventory_parent_org_id = getattr(Model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id)\ +# .cycle.organization.get_parent().id +# label_super_org_id = Model.statuslabel.get_queryset().get(pk=label_id).super_organization_id +# if inventory_parent_org_id == label_super_org_id: +# create_dict = { +# 'statuslabel_id': label_id, +# "{}view_id".format(inventory_type): inventory_id +# } +# +# return Model(**create_dict) +# else: +# raise IntegrityError( +# 'Label with super_organization_id={} cannot be applied to a record with parent ' +# 'organization_id={}.'.format( +# label_super_org_id, +# inventory_parent_org_id +# ) +# ) +# +# def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids): +# added = [] +# if add_label_ids: +# model = self.models[inventory_type] +# inventory_model = self.inventory_models[inventory_type] +# exclude = self.exclude(qs, inventory_type, add_label_ids) +# inventory_ids = inventory_ids if inventory_ids else [ +# m.pk for m in inventory_model.objects.all() +# ] +# new_inventory_labels = [ +# self.label_factory(inventory_type, label_id, pk) +# for label_id in add_label_ids for pk in inventory_ids +# if pk not in exclude[label_id] +# ] +# model.objects.bulk_create(new_inventory_labels) +# added = [ +# self.get_inventory_id(m, inventory_type) +# for m in new_inventory_labels +# ] +# return added +# +# def remove_labels(self, qs, inventory_type, remove_label_ids): +# removed = [] +# if remove_label_ids: +# rqs = qs.filter( +# statuslabel_id__in=remove_label_ids +# ) +# removed = [self.get_inventory_id(q, inventory_type) for q in rqs] +# rqs.delete() +# return removed +# +# def put(self, request, inventory_type): +# """ +# Updates label assignments to inventory items. +# +# Payload:: +# +# { +# "add_label_ids": {array} Array of label ids to add +# "remove_label_ids": {array} Array of label ids to remove +# "inventory_ids": {array} Array property/taxlot ids +# "organization_id": {integer} The user's org ID +# } +# +# Returns:: +# +# { +# 'status': {string} 'success' or 'error' +# 'message': {string} Error message if error +# 'num_updated': {integer} Number of properties/taxlots updated +# 'labels': [ List of labels affected. +# { +# 'color': {string} +# 'id': {int} +# 'label': {'string'} +# 'name': {string} +# }... +# ] +# } +# +# """ +# add_label_ids = request.data.get('add_label_ids', []) +# remove_label_ids = request.data.get('remove_label_ids', []) +# inventory_ids = request.data.get('inventory_ids', None) +# organization_id = request.query_params['organization_id'] +# error = None +# # ensure add_label_ids and remove_label_ids are different +# if not set(add_label_ids).isdisjoint(remove_label_ids): +# error = self.errors['disjoint'] +# elif not organization_id: +# error = self.errors['missing_org'] +# if error: +# result = { +# 'status': 'error', +# 'message': str(error) +# } +# status_code = error.status_code +# else: +# qs = self.get_queryset(inventory_type, organization_id) +# qs = self.filter_by_inventory(qs, inventory_type, inventory_ids) +# removed = self.remove_labels(qs, inventory_type, remove_label_ids) +# added = self.add_labels(qs, inventory_type, inventory_ids, add_label_ids) +# num_updated = len(set(added).union(removed)) +# labels = self.get_label_desc(add_label_ids, remove_label_ids) +# result = { +# 'status': 'success', +# 'num_updated': num_updated, +# 'labels': labels +# } +# status_code = status.HTTP_200_OK +# return response.Response(result, status=status_code) From 6e73d8e00257a2993ca471c24e574fd3ea0b9562 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 4 May 2020 14:04:15 -0600 Subject: [PATCH 002/135] labels almost done need to see how to deprecate PATCH --- seed/utils/api_schema.py | 4 +- seed/utils/viewsets.py | 14 +- seed/views/labels.py | 15 ++ seed/views/v3/labels.py | 395 ++------------------------------------- 4 files changed, 46 insertions(+), 382 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 1426b0dd19..2c7391535c 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -11,7 +11,9 @@ class AutoSchemaHelper(SwaggerAutoSchema): 'interger_list': openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_INTEGER) - ) + ), + 'integer': openapi.Schema(type=openapi.TYPE_INTEGER), + 'string': openapi.Schema(type=openapi.TYPE_STRING) } def base_field(self, name, location_attr, description, required, type): diff --git a/seed/utils/viewsets.py b/seed/utils/viewsets.py index 2248a2eeae..a8f33ff086 100644 --- a/seed/utils/viewsets.py +++ b/seed/utils/viewsets.py @@ -18,6 +18,11 @@ from rest_framework.parsers import FormParser, JSONParser, MultiPartParser from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from oauth2_provider.ext.rest_framework import OAuth2Authentication +from rest_framework.mixins import ( + CreateModelMixin, + DestroyModelMixin, + UpdateModelMixin, +) # Local Imports from seed.authentication import SEEDAuthentication @@ -29,7 +34,6 @@ OrgQuerySetMixin, drf_api_endpoint ) - # Constants AUTHENTICATION_CLASSES = ( OAuth2Authentication, @@ -40,6 +44,14 @@ RENDERER_CLASSES = (JSONRenderer,) PERMISSIONS_CLASSES = (SEEDOrgPermissions,) +class UpdateWithoutPatchModelMixin(object): + # Rebuilds the UpdateModelMixin without the patch action + def update(self, request, *args, **kwargs): + return UpdateModelMixin.update(self, request, *args, **kwargs) + + def perform_update(self, serializer): + return UpdateModelMixin.perform_update(self, serializer) + class SEEDOrgModelViewSet(DecoratorMixin(drf_api_endpoint), OrgQuerySetMixin, ModelViewSet): """Viewset class customized with SEED standard attributes. diff --git a/seed/views/labels.py b/seed/views/labels.py index 9b6870fcb7..70151a3ee0 100644 --- a/seed/views/labels.py +++ b/seed/views/labels.py @@ -118,6 +118,21 @@ def list(self, request): class UpdateInventoryLabelsAPIView(APIView): + """API endpoint for viewing and creating labels. + + Returns:: + [ + { + 'id': Label's primary key + 'name': Name given to label + 'color': Color of label, + 'organization_id': Id of organization label belongs to, + 'is_applied': Will be empty array if not applied to property/taxlots + } + ] + + --- + """ renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 1a17f74c9a..e633e74ec2 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -34,6 +34,7 @@ ) from seed.utils.api import drf_api_endpoint from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.viewsets import UpdateWithoutPatchModelMixin ErrorState = namedtuple('ErrorState', ['status_code', 'message']) @@ -42,52 +43,19 @@ class LabelSchema(AutoSchemaHelper): def __init__(self, *args): super().__init__(*args) - self.manual_fields = { - ('GET', 'get_queryset'): [], - ('POST', 'add_labels'): [ - self.body_field( - name='name', - required=True, - description="Rules information", - params_to_formats={ - 'name': 'string', - 'color': 'string', - 'super_organization': 'integer' - } - ) - ], - ('POST', 'filter'): [self.org_id_field()], - ('GET', 'get_labels'): [self.org_id_field()], - ('PUT', 'put'): [ - self.org_id_field(), - self.body_field( - name='data_quality_rules', - required=True, - description="Rules information" - ) - ], - ('GET', 'results'): [ - self.org_id_field(), - self.query_integer_field( - name='data_quality_id', - required=True, - description="Task ID created when DataQuality task is created." - ), - ], - ('GET', 'csv'): [ - # This will replace the auto-generated field - adds description. - self.path_id_field(description="Import file ID or cache key") - ], - } + self.manual_fields = {} class LabelViewSet(DecoratorMixin(drf_api_endpoint), viewsets.ModelViewSet): - swagger_schema = LabelSchema + """API endpoint for viewing and creating labels. + --- + """ serializer_class = LabelSerializer renderer_classes = (JSONRenderer,) parser_classes = (JSONParser, FormParser) queryset = Label.objects.none() filter_backends = (LabelFilterBackend,) + swagger_schema = LabelSchema pagination_class = None _organization = None @@ -110,6 +78,7 @@ def get_organization(self): return self._organization def get_queryset(self): + labels = Label.objects.filter( super_organization=self.get_parent_organization() ).order_by("name").distinct() @@ -139,352 +108,18 @@ def _get_labels(self, request): status_code = status.HTTP_200_OK return response.Response(results, status=status_code) - renderer_classes = (JSONRenderer,) - parser_classes = (JSONParser,) - inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} - errors = { - 'disjoint': ErrorState( - status.HTTP_422_UNPROCESSABLE_ENTITY, - 'add_label_ids and remove_label_ids cannot contain elements in common' - ), - 'missing_org': ErrorState( - status.HTTP_422_UNPROCESSABLE_ENTITY, - 'missing organization_id' - ) - } - - @property - def models(self): - """ - Exposes Django's internal models for join table. - - Used for bulk_create operations. - """ - return { - 'property': apps.get_model('seed', 'PropertyView_labels'), - 'taxlot': apps.get_model('seed', 'TaxLotView_labels') - } - - # def get_queryset(self, inventory_type, organization_id): - # Model = self.models[inventory_type] - # return Model.objects.filter( - # statuslabel__super_organization_id=organization_id - # ) - - def get_label_desc(self, add_label_ids, remove_label_ids): - return Label.objects.filter( - pk__in=add_label_ids + remove_label_ids - ).values('id', 'color', 'name') - - def get_inventory_id(self, q, inventory_type): - return getattr(q, "{}view_id".format(inventory_type)) - - def exclude(self, qs, inventory_type, label_ids): - exclude = {label: [] for label in label_ids} - for q in qs: - if q.statuslabel_id in label_ids: - inventory_id = self.get_inventory_id(q, inventory_type) - exclude[q.statuslabel_id].append(inventory_id) - return exclude - - def filter_by_inventory(self, qs, inventory_type, inventory_ids): - if inventory_ids: - filterdict = { - "{}view__pk__in".format(inventory_type): inventory_ids - } - qs = qs.filter(**filterdict) - return qs - - def label_factory(self, inventory_type, label_id, inventory_id): - model = self.models[inventory_type] - - # Ensure the the label org and inventory org are the same - inventory_parent_org_id = getattr(model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id) \ - .cycle.organization.get_parent().id - label_super_org_id = model.statuslabel.get_queryset().get(pk=label_id).super_organization_id - if inventory_parent_org_id == label_super_org_id: - create_dict = { - 'statuslabel_id': label_id, - "{}view_id".format(inventory_type): inventory_id - } - - return model(**create_dict) - else: - raise IntegrityError( - 'Label with super_organization_id={} cannot be applied to a record with parent ' - 'organization_id={}.'.format( - label_super_org_id, - inventory_parent_org_id - ) - ) - - def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids): - added = [] - if add_label_ids: - model = self.models[inventory_type] - inventory_model = self.inventory_models[inventory_type] - exclude = self.exclude(qs, inventory_type, add_label_ids) - inventory_ids = inventory_ids if inventory_ids else [ - m.pk for m in inventory_model.objects.all() - ] - new_inventory_labels = [ - self.label_factory(inventory_type, label_id, pk) - for label_id in add_label_ids for pk in inventory_ids - if pk not in exclude[label_id] - ] - model.objects.bulk_create(new_inventory_labels) - added = [ - self.get_inventory_id(m, inventory_type) - for m in new_inventory_labels - ] - return added - - def remove_labels(self, qs, inventory_type, remove_label_ids): - removed = [] - if remove_label_ids: - rqs = qs.filter( - statuslabel_id__in=remove_label_ids - ) - removed = [self.get_inventory_id(q, inventory_type) for q in rqs] - rqs.delete() - return removed - - def put(self, request, inventory_type): - """ - Updates label assignments to inventory items. - - Payload:: - - { - "add_label_ids": {array} Array of label ids to add - "remove_label_ids": {array} Array of label ids to remove - "inventory_ids": {array} Array property/taxlot ids - "organization_id": {integer} The user's org ID - } - - Returns:: - - { - 'status': {string} 'success' or 'error' - 'message': {string} Error message if error - 'num_updated': {integer} Number of properties/taxlots updated - 'labels': [ List of labels affected. - { - 'color': {string} - 'id': {int} - 'label': {'string'} - 'name': {string} - }... - ] - } - - """ - add_label_ids = request.data.get('add_label_ids', []) - remove_label_ids = request.data.get('remove_label_ids', []) - inventory_ids = request.data.get('inventory_ids', None) - organization_id = request.query_params['organization_id'] - error = None - # ensure add_label_ids and remove_label_ids are different - if not set(add_label_ids).isdisjoint(remove_label_ids): - error = self.errors['disjoint'] - elif not organization_id: - error = self.errors['missing_org'] - if error: - result = { - 'status': 'error', - 'message': str(error) - } - status_code = error.status_code - else: - qs = self.get_queryset(inventory_type, organization_id) - qs = self.filter_by_inventory(qs, inventory_type, inventory_ids) - removed = self.remove_labels(qs, inventory_type, remove_label_ids) - added = self.add_labels(qs, inventory_type, inventory_ids, add_label_ids) - num_updated = len(set(added).union(removed)) - labels = self.get_label_desc(add_label_ids, remove_label_ids) - result = { - 'status': 'success', - 'num_updated': num_updated, - 'labels': labels - } - status_code = status.HTTP_200_OK - return response.Response(result, status=status_code) - @action(detail=False, methods=['POST']) def filter(self, request): + """ + Filters a list of all labels + --- + """ return self._get_labels(request) def list(self, request): + """ + Returns a list of all labels + --- + """ return self._get_labels(request) -# class UpdateInventoryLabelsAPIView(APIView): -# renderer_classes = (JSONRenderer,) -# parser_classes = (JSONParser,) -# inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} -# errors = { -# 'disjoint': ErrorState( -# status.HTTP_422_UNPROCESSABLE_ENTITY, -# 'add_label_ids and remove_label_ids cannot contain elements in common' -# ), -# 'missing_org': ErrorState( -# status.HTTP_422_UNPROCESSABLE_ENTITY, -# 'missing organization_id' -# ) -# } -# -# @property -# def models(self): -# """ -# Exposes Django's internal models for join table. -# -# Used for bulk_create operations. -# """ -# return { -# 'property': apps.get_model('seed', 'PropertyView_labels'), -# 'taxlot': apps.get_model('seed', 'TaxLotView_labels') -# } -# -# def get_queryset(self, inventory_type, organization_id): -# Model = self.models[inventory_type] -# return Model.objects.filter( -# statuslabel__super_organization_id=organization_id -# ) -# -# def get_label_desc(self, add_label_ids, remove_label_ids): -# return Label.objects.filter( -# pk__in=add_label_ids + remove_label_ids -# ).values('id', 'color', 'name') -# -# def get_inventory_id(self, q, inventory_type): -# return getattr(q, "{}view_id".format(inventory_type)) -# -# def exclude(self, qs, inventory_type, label_ids): -# exclude = {label: [] for label in label_ids} -# for q in qs: -# if q.statuslabel_id in label_ids: -# inventory_id = self.get_inventory_id(q, inventory_type) -# exclude[q.statuslabel_id].append(inventory_id) -# return exclude -# -# def filter_by_inventory(self, qs, inventory_type, inventory_ids): -# if inventory_ids: -# filterdict = { -# "{}view__pk__in".format(inventory_type): inventory_ids -# } -# qs = qs.filter(**filterdict) -# return qs -# -# def label_factory(self, inventory_type, label_id, inventory_id): -# Model = self.models[inventory_type] -# -# # Ensure the the label org and inventory org are the same -# inventory_parent_org_id = getattr(Model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id)\ -# .cycle.organization.get_parent().id -# label_super_org_id = Model.statuslabel.get_queryset().get(pk=label_id).super_organization_id -# if inventory_parent_org_id == label_super_org_id: -# create_dict = { -# 'statuslabel_id': label_id, -# "{}view_id".format(inventory_type): inventory_id -# } -# -# return Model(**create_dict) -# else: -# raise IntegrityError( -# 'Label with super_organization_id={} cannot be applied to a record with parent ' -# 'organization_id={}.'.format( -# label_super_org_id, -# inventory_parent_org_id -# ) -# ) -# -# def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids): -# added = [] -# if add_label_ids: -# model = self.models[inventory_type] -# inventory_model = self.inventory_models[inventory_type] -# exclude = self.exclude(qs, inventory_type, add_label_ids) -# inventory_ids = inventory_ids if inventory_ids else [ -# m.pk for m in inventory_model.objects.all() -# ] -# new_inventory_labels = [ -# self.label_factory(inventory_type, label_id, pk) -# for label_id in add_label_ids for pk in inventory_ids -# if pk not in exclude[label_id] -# ] -# model.objects.bulk_create(new_inventory_labels) -# added = [ -# self.get_inventory_id(m, inventory_type) -# for m in new_inventory_labels -# ] -# return added -# -# def remove_labels(self, qs, inventory_type, remove_label_ids): -# removed = [] -# if remove_label_ids: -# rqs = qs.filter( -# statuslabel_id__in=remove_label_ids -# ) -# removed = [self.get_inventory_id(q, inventory_type) for q in rqs] -# rqs.delete() -# return removed -# -# def put(self, request, inventory_type): -# """ -# Updates label assignments to inventory items. -# -# Payload:: -# -# { -# "add_label_ids": {array} Array of label ids to add -# "remove_label_ids": {array} Array of label ids to remove -# "inventory_ids": {array} Array property/taxlot ids -# "organization_id": {integer} The user's org ID -# } -# -# Returns:: -# -# { -# 'status': {string} 'success' or 'error' -# 'message': {string} Error message if error -# 'num_updated': {integer} Number of properties/taxlots updated -# 'labels': [ List of labels affected. -# { -# 'color': {string} -# 'id': {int} -# 'label': {'string'} -# 'name': {string} -# }... -# ] -# } -# -# """ -# add_label_ids = request.data.get('add_label_ids', []) -# remove_label_ids = request.data.get('remove_label_ids', []) -# inventory_ids = request.data.get('inventory_ids', None) -# organization_id = request.query_params['organization_id'] -# error = None -# # ensure add_label_ids and remove_label_ids are different -# if not set(add_label_ids).isdisjoint(remove_label_ids): -# error = self.errors['disjoint'] -# elif not organization_id: -# error = self.errors['missing_org'] -# if error: -# result = { -# 'status': 'error', -# 'message': str(error) -# } -# status_code = error.status_code -# else: -# qs = self.get_queryset(inventory_type, organization_id) -# qs = self.filter_by_inventory(qs, inventory_type, inventory_ids) -# removed = self.remove_labels(qs, inventory_type, remove_label_ids) -# added = self.add_labels(qs, inventory_type, inventory_ids, add_label_ids) -# num_updated = len(set(added).union(removed)) -# labels = self.get_label_desc(add_label_ids, remove_label_ids) -# result = { -# 'status': 'success', -# 'num_updated': num_updated, -# 'labels': labels -# } -# status_code = status.HTTP_200_OK -# return response.Response(result, status=status_code) From cbf18212f6618c4a3b6405e67e85b7f7c2efe0c8 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 6 May 2020 10:12:31 -0600 Subject: [PATCH 003/135] all set besides the POST filter - used the class wrapper viewset to deprecate PATCH --- seed/utils/viewsets.py | 12 ++++++++++++ seed/views/v3/labels.py | 5 ++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/seed/utils/viewsets.py b/seed/utils/viewsets.py index a8f33ff086..4925750de6 100644 --- a/seed/utils/viewsets.py +++ b/seed/utils/viewsets.py @@ -34,6 +34,7 @@ OrgQuerySetMixin, drf_api_endpoint ) + # Constants AUTHENTICATION_CLASSES = ( OAuth2Authentication, @@ -44,6 +45,7 @@ RENDERER_CLASSES = (JSONRenderer,) PERMISSIONS_CLASSES = (SEEDOrgPermissions,) + class UpdateWithoutPatchModelMixin(object): # Rebuilds the UpdateModelMixin without the patch action def update(self, request, *args, **kwargs): @@ -97,3 +99,13 @@ class SEEDOrgCreateUpdateModelViewSet(OrgCreateUpdateMixin, SEEDOrgModelViewSet) and/or perform_update overrides appropriate to the model's needs. """ pass + + +class SEEDOrgNoPatchOrOrgCreateModelViewSet(SEEDOrgReadOnlyModelViewSet, + CreateModelMixin, + DestroyModelMixin, + UpdateWithoutPatchModelMixin): + """Extends SEEDOrgReadOnlyModelViewSet to include update (without patch), + create, and destroy actions. + """ + pass diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index e633e74ec2..bf2e788452 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -34,7 +34,7 @@ ) from seed.utils.api import drf_api_endpoint from seed.utils.api_schema import AutoSchemaHelper -from seed.utils.viewsets import UpdateWithoutPatchModelMixin +from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet ErrorState = namedtuple('ErrorState', ['status_code', 'message']) @@ -42,11 +42,10 @@ class LabelSchema(AutoSchemaHelper): def __init__(self, *args): super().__init__(*args) - self.manual_fields = {} -class LabelViewSet(DecoratorMixin(drf_api_endpoint), viewsets.ModelViewSet): +class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateModelViewSet): """API endpoint for viewing and creating labels. --- """ From de8b51bc88d9af2f1f89e18f9798d15702d9b700 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 6 May 2020 11:12:39 -0600 Subject: [PATCH 004/135] added TODOs --- seed/views/v3/labels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index bf2e788452..8391867742 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -108,6 +108,7 @@ def _get_labels(self, request): return response.Response(results, status=status_code) @action(detail=False, methods=['POST']) + #TODO: apply filters for respective views def filter(self, request): """ Filters a list of all labels From 42c60992407408f76660486970f11e8f4106ff04 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Fri, 8 May 2020 08:44:34 -0600 Subject: [PATCH 005/135] resolving conflicts --- seed/utils/api_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 2c7391535c..251dc34c64 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -12,8 +12,8 @@ class AutoSchemaHelper(SwaggerAutoSchema): type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_INTEGER) ), + 'integer': openapi.Schema(type=openapi.TYPE_INTEGER), - 'string': openapi.Schema(type=openapi.TYPE_STRING) } def base_field(self, name, location_attr, description, required, type): From 71135fdea7e9e422b54ba657c8fa3fab1f051b9e Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Fri, 8 May 2020 09:04:16 -0600 Subject: [PATCH 006/135] blank line at end of file --- seed/views/v3/labels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 8391867742..1da8e0fcb1 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -122,4 +122,3 @@ def list(self, request): --- """ return self._get_labels(request) - From 8815b89ef63553993e69145f2c34bbe485e32045 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Fri, 8 May 2020 09:27:03 -0600 Subject: [PATCH 007/135] trying to get travis to pass --- seed/views/v3/labels.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 1da8e0fcb1..5a071a3fd4 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -6,18 +6,14 @@ """ from collections import namedtuple -from django.apps import apps -from django.db import IntegrityError from django.core.exceptions import ObjectDoesNotExist from rest_framework import ( response, status, - viewsets ) from rest_framework.decorators import action from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer -from rest_framework.views import APIView from seed.decorators import DecoratorMixin from seed.filters import ( @@ -112,13 +108,11 @@ def _get_labels(self, request): def filter(self, request): """ Filters a list of all labels - --- """ return self._get_labels(request) def list(self, request): """ Returns a list of all labels - --- """ return self._get_labels(request) From 08f7861d333bd47762db0ac61423b54e009d90ea Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Fri, 8 May 2020 11:38:22 -0600 Subject: [PATCH 008/135] testing travis --- seed/tests/test_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seed/tests/test_api.py b/seed/tests/test_api.py index 4d9d0754cc..64440c9a64 100644 --- a/seed/tests/test_api.py +++ b/seed/tests/test_api.py @@ -50,7 +50,8 @@ def test_get_api_endpoints_utils(self): '/api/v2/gbr_properties', '/api/v2/notes', '/api/v2.1/properties', - '/api/v2.1/scenarios',)): + '/api/v2.1/scenarios', + '/api/v3/labels',)): self.assertTrue( url.endswith('/'), "Endpoint %s does not end with / as expected" % url From d701bf8a21290ddd23ba061b03ea3b85f6f018b6 Mon Sep 17 00:00:00 2001 From: Alex Swindler Date: Sat, 9 May 2020 21:39:06 -0600 Subject: [PATCH 009/135] flake8 --- seed/views/v3/labels.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 5a071a3fd4..caa685de53 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -22,8 +22,6 @@ ) from seed.models import ( StatusLabel as Label, - PropertyView, - TaxLotView, ) from seed.serializers.labels import ( LabelSerializer, @@ -104,7 +102,7 @@ def _get_labels(self, request): return response.Response(results, status=status_code) @action(detail=False, methods=['POST']) - #TODO: apply filters for respective views + # TODO: apply filters for respective views def filter(self, request): """ Filters a list of all labels From 9e1b4ac5ac905cb61bf87cfc90d3c02a58556bdf Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 11 May 2020 12:47:20 -0600 Subject: [PATCH 010/135] users apiv3 working --- seed/api/v3/urls.py | 4 + seed/landing/models.py | 4 + seed/utils/api_schema.py | 7 +- seed/views/v3/users.py | 887 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 901 insertions(+), 1 deletion(-) create mode 100644 seed/views/v3/users.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 1882f228c8..37b3f35b0e 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -6,10 +6,14 @@ from seed.views.v3.data_quality import DataQualityViews from seed.views.v3.datasets import DatasetViewSet +from seed.views.v3.users import UserViewSet + api_v3_router = routers.DefaultRouter() api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') +api_v3_router.register(r'users', UserViewSet, base_name='user') + urlpatterns = [ url(r'^', include(api_v3_router.urls)), diff --git a/seed/landing/models.py b/seed/landing/models.py index be5d2b9dc2..7cfa94c298 100644 --- a/seed/landing/models.py +++ b/seed/landing/models.py @@ -124,6 +124,10 @@ def process_header_request(cls, request): def get_absolute_url(self): return "/users/%s/" % urlquote(self.username) + def deactivate_user(self): + self.is_active = False + self.save() + def get_full_name(self): """ Returns the first_name plus the last_name, with a space in between. diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 50035dc6d1..f3bb19d452 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -12,7 +12,12 @@ class AutoSchemaHelper(SwaggerAutoSchema): type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_INTEGER) ), - 'string': openapi.Schema(type=openapi.TYPE_STRING) + 'string': openapi.Schema(type=openapi.TYPE_STRING), + + 'string_array': openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema(type=openapi.TYPE_STRING) + ) } def base_field(self, name, location_attr, description, required, type): diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py new file mode 100644 index 0000000000..60847ba25b --- /dev/null +++ b/seed/views/v3/users.py @@ -0,0 +1,887 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import logging + +from django.contrib.auth.password_validation import validate_password +from django.contrib.auth.tokens import default_token_generator +from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ValidationError +from django.http import JsonResponse +from rest_framework import viewsets, status, serializers +from rest_framework.decorators import action + +from seed.decorators import ajax_request_class +from seed.landing.models import SEEDUser as User, SEEDUser +from seed.lib.superperms.orgs.decorators import PERMS +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import ( + ROLE_OWNER, + ROLE_MEMBER, + ROLE_VIEWER, + Organization, + OrganizationUser, +) +from seed.models.data_quality import Rule +from seed.tasks import ( + invite_to_seed, +) +from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.organizations import create_organization + +_log = logging.getLogger(__name__) + + +def _get_js_role(role): + """return the JS friendly role name for user + :param role: role as defined in superperms.models + :returns: (string) JS role name + """ + roles = { + ROLE_OWNER: 'owner', + ROLE_VIEWER: 'viewer', + ROLE_MEMBER: 'member', + } + return roles.get(role, 'viewer') + + +def _get_role_from_js(role): + """return the OrganizationUser role_level from the JS friendly role name + + :param role: 'member', 'owner', or 'viewer' + :returns: int role as defined in superperms.models + """ + roles = { + 'owner': ROLE_OWNER, + 'viewer': ROLE_VIEWER, + 'member': ROLE_MEMBER, + } + return roles[role] + + +def _get_js_rule_type(data_type): + """return the JS friendly data type name for the data data_quality rule + + :param data_type: data data_quality rule data type as defined in data_quality.models + :returns: (string) JS data type name + """ + return dict(Rule.DATA_TYPES).get(data_type) + + +def _get_rule_type_from_js(data_type): + """return the Rules TYPE from the JS friendly data type + + :param data_type: 'string', 'number', 'date', or 'year' + :returns: int data type as defined in data_quality.models + """ + d = {v: k for k, v in dict(Rule.DATA_TYPES).items()} + return d.get(data_type) + + +def _get_js_rule_severity(severity): + """return the JS friendly severity name for the data data_quality rule + + :param severity: data data_quality rule severity as defined in data_quality.models + :returns: (string) JS severity name + """ + return dict(Rule.SEVERITY).get(severity) + + +def _get_severity_from_js(severity): + """return the Rules SEVERITY from the JS friendly severity + + :param severity: 'error', or 'warning' + :returns: int severity as defined in data_quality.models + """ + d = {v: k for k, v in dict(Rule.SEVERITY).items()} + return d.get(severity) + + +class EmailAndIDSerializer(serializers.Serializer): + email = serializers.CharField(max_length=100) + user_id = serializers.IntegerField() + + +class ListUsersResponseSerializer(serializers.Serializer): + users = EmailAndIDSerializer(many=True) + +class UserSchema(AutoSchemaHelper): + def __init__(self, *args): + super().__init__(*args) + + self.manual_fields = { + ('GET', 'list'): [], + ('POST', 'create'): [ + self.org_id_field(), + self.body_field( + name='New User Fields', + required=True, + description="An object containing meta data for a new user: " + "required - [last_name, role(viewer/owner/member), email]", + params_to_formats={ + 'first_name': 'string', + 'last_name': 'string', + 'role': 'string', + 'email': 'string' + } + )], + ('GET', 'current_user_id'): [], + ('GET', 'retrieve'): [self.path_id_field(description="users PK ID")], + ('PUT', 'update'): [ + self.path_id_field(description="Updated Users PK ID"), + self.body_field( + name='Updated User Fields', + required=True, + description="An object containing meta data for a updated user: " + "required - [first_name, last_name, email]", + params_to_formats={ + 'first_name': 'string', + 'last_name': 'string', + 'email': 'string' + } + )], + ('PUT', 'default_organization'): [self.org_id_field(), self.path_id_field(description="Updated Users PK ID")], + ('POST', 'is_authorized'): [ + self.org_id_field(), + self.path_id_field(description="Users PK ID"), + self.body_field( + name='actions', + required=True, + description="A list of actions to check: examples include (requires_parent_org_owner, " + "requires_owner, requires_member, requires_viewer, " + "requires_superuser, can_create_sub_org, can_remove_org)", + params_to_formats={ + 'actions': 'string_array', + }) + ], + ('PUT', 'set_password'): [ + self.body_field( + name='Change Password', + required=True, + description="fill in the current and new matching passwords ", + params_to_formats={ + 'current_password': 'string', + 'password_1': 'string', + 'password_2': 'string' + }) + ], + ('PUT', 'role'): [ + self.org_id_field(), + self.path_id_field(description="Users PK ID"), + self.body_field( + name='role', + required=True, + description="fill in the role to be updated", + params_to_formats={ + 'role': 'string', + + }) + ], + ('DELETE', 'deactivate'): [ + self.body_field( + name='User to be deactivated', + required=True, + description="first and last name of user to be deactivated", + params_to_formats={ + 'first_name': 'string', + 'last_name': 'string' + }) + + ] + + + + } + + +class UserViewSet(viewsets.ViewSet): + raise_exception = True + + swagger_schema = UserSchema + + def validate_request_user(self, pk, request): + try: + user = User.objects.get(pk=pk) + except ObjectDoesNotExist: + return False, JsonResponse( + {'status': 'error', 'message': "Could not find user with pk = " + str(pk)}, + status=status.HTTP_404_NOT_FOUND) + if not user == request.user: + return False, JsonResponse( + {'status': 'error', 'message': "Cannot access user with pk = " + str(pk)}, + status=status.HTTP_403_FORBIDDEN) + return True, user + + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_owner') + def create(self, request): + """ + Creates a new SEED user. One of 'organization_id' or 'org_name' is needed. + Sends invitation email to the new user. + --- + parameters: + - name: organization_id + description: Organization ID if adding user to an existing organization + required: false + type: integer + - name: org_name + description: New organization name if creating a new organization for this user + required: false + type: string + - name: first_name + description: First name of new user + required: true + type: string + - name: last_name + description: Last name of new user + required: true + type: string + - name: role + description: one of owner, member, or viewer + required: true + type: string + - name: email + description: Email address of the new user + required: true + type: string + type: + status: + description: success or error + required: true + type: string + message: + description: email address of new user + required: true + type: string + org: + description: name of the new org (or existing org) + required: true + type: string + org_created: + description: True if new org created + required: true + type: string + username: + description: Username of new user + required: true + type: string + user_id: + description: User ID (pk) of new user + required: true + type: integer + """ + body = request.data + org_name = body.get('org_name') + org_id = request.query_params.get('organization_id', None) + if (org_name and org_id) or (not org_name and not org_id): + return JsonResponse({ + 'status': 'error', + 'message': 'Choose either an existing org or provide a new one' + }, status=status.HTTP_409_CONFLICT) + + first_name = body['first_name'] + last_name = body['last_name'] + email = body['email'] + username = body['email'] + user, created = User.objects.get_or_create(username=username.lower()) + + if org_id: + org = Organization.objects.get(pk=org_id) + org_created = False + else: + org, _, _ = create_organization(user, org_name) + org_created = True + + # Add the user to the org. If this is the org's first user, + # the user becomes the owner/admin automatically. + # see Organization.add_member() + if not org.is_member(user): + org.add_member(user) + + if body.get('role'): + # check if this is a dict, if so, grab the value out of 'value' + role = body['role'] + if isinstance(role, dict): + role = role['value'] + + OrganizationUser.objects.filter( + organization_id=org.pk, + user_id=user.pk + ).update(role_level=_get_role_from_js(role)) + + if created: + user.set_unusable_password() + user.email = email + user.first_name = first_name + user.last_name = last_name + user.save() + + try: + domain = request.get_host() + except Exception: + domain = 'seed-platform.org' + invite_to_seed( + domain, user.email, default_token_generator.make_token(user), user.pk, first_name + ) + + return JsonResponse({ + 'status': 'success', + 'message': user.email, + 'org': org.name, + 'org_created': org_created, + 'username': user.username, + 'user_id': user.id + }) + + @ajax_request_class + @has_perm_class('requires_superuser') + def list(self, request): + """ + Retrieves all users' email addresses and IDs. + Only usable by superusers. + --- + response_serializer: ListUsersResponseSerializer + """ + users = [] + for user in User.objects.only('id', 'email'): + users.append({'email': user.email, 'user_id': user.id}) + return JsonResponse({'users': users}) + + @ajax_request_class + @api_endpoint_class + @action(detail=False, methods=['GET']) + def current(self, request): + """ + Returns the id (primary key) for the current user to allow it + to be passed to other user related endpoints + --- + type: + pk: + description: Primary key for the current user + required: true + type: string + """ + return JsonResponse({'pk': request.user.id}) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_owner') + @action(detail=True, methods=['PUT']) + def role(self, request, pk=None): + """ + Updates a user's role within an organization. + --- + parameter_strategy: replace + parameters: + - name: pk + description: ID for the user to modify + type: integer + required: true + paramType: path + - name: organization_id + description: The organization ID to update this user within + type: integer + required: true + - name: role + description: one of owner, member, or viewer + type: string + required: true + type: + status: + required: true + description: success or error + type: string + message: + required: false + description: error message, if any + type: string + """ + body = request.data + role = _get_role_from_js(body['role']) + + user_id = pk + organization_id = request.query_params.get('organization_id', None) + + is_last_member = not OrganizationUser.objects.filter( + organization_id=organization_id, + ).exclude(user_id=user_id).exists() + + if is_last_member: + return JsonResponse({ + 'status': 'error', + 'message': 'an organization must have at least one member' + }, status=status.HTTP_409_CONFLICT) + + is_last_owner = not OrganizationUser.objects.filter( + organization_id=organization_id, + role_level=ROLE_OWNER, + ).exclude(user_id=user_id).exists() + + if is_last_owner: + return JsonResponse({ + 'status': 'error', + 'message': 'an organization must have at least one owner level member' + }, status=status.HTTP_409_CONFLICT) + + OrganizationUser.objects.filter( + user_id=user_id, + organization_id=body['organization_id'] + ).update(role_level=role) + + return JsonResponse({'status': 'success'}) + + @api_endpoint_class + @ajax_request_class + def retrieve(self, request, pk=None): + """ + Retrieves the a user's first_name, last_name, email + and api key if exists by user ID (pk). + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID / primary key + type: integer + required: true + paramType: path + type: + status: + description: success or error + type: string + required: true + first_name: + description: user first name + type: string + required: true + last_name: + description: user last name + type: string + required: true + email: + description: user email + type: string + required: true + api_key: + description: user API key + type: string + required: true + """ + + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + return JsonResponse({ + 'status': 'success', + 'first_name': user.first_name, + 'last_name': user.last_name, + 'email': user.email, + 'api_key': user.api_key, + }) + + @ajax_request_class + @action(detail=True, methods=['POST']) + def generate_api_key(self, request, pk=None): + """ + Generates a new API key + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID / primary key + type: integer + required: true + paramType: path + type: + status: + description: success or error + type: string + required: true + api_key: + description: the new API key for this user + type: string + required: true + """ + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + user.generate_key() + return { + 'status': 'success', + 'api_key': User.objects.get(pk=pk).api_key + } + + @api_endpoint_class + @ajax_request_class + def update(self, request, pk=None): + """ + Updates the user's first name, last name, and email + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID / primary key + type: integer + required: true + paramType: path + - name: first_name + description: New first name + type: string + required: true + - name: last_name + description: New last name + type: string + required: true + - name: email + description: New user email + type: string + required: true + type: + status: + description: success or error + type: string + required: true + first_name: + description: user first name + type: string + required: true + last_name: + description: user last name + type: string + required: true + email: + description: user email + type: string + required: true + api_key: + description: user API key + type: string + required: true + """ + body = request.data + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + json_user = body + user.first_name = json_user.get('first_name') + user.last_name = json_user.get('last_name') + user.email = json_user.get('email') + user.username = json_user.get('email') + user.save() + return JsonResponse({ + 'status': 'success', + 'first_name': user.first_name, + 'last_name': user.last_name, + 'email': user.email, + 'api_key': user.api_key, + }) + + @ajax_request_class + @action(detail=True, methods=['PUT']) + def set_password(self, request, pk=None): + """ + sets/updates a user's password, follows the min requirement of + django password validation settings in config/settings/common.py + --- + parameter_strategy: replace + parameters: + - name: current_password + description: Users current password + type: string + required: true + - name: password_1 + description: Users new password 1 + type: string + required: true + - name: password_2 + description: Users new password 2 + type: string + required: true + type: + status: + type: string + description: success or error + required: true + message: + type: string + description: error message, if any + required: false + """ + body = request.data + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + current_password = body.get('current_password') + p1 = body.get('password_1') + p2 = body.get('password_2') + if not user.check_password(current_password): + return JsonResponse({'status': 'error', 'message': 'current password is not valid'}, + status=status.HTTP_400_BAD_REQUEST) + if p1 is None or p1 != p2: + return JsonResponse({'status': 'error', 'message': 'entered password do not match'}, + status=status.HTTP_400_BAD_REQUEST) + try: + validate_password(p2) + except ValidationError as e: + return JsonResponse({'status': 'error', 'message': e.messages[0]}, + status=status.HTTP_400_BAD_REQUEST) + user.set_password(p1) + user.save() + return JsonResponse({'status': 'success'}) + + @ajax_request_class + def get_actions(self, request): + """returns all actions""" + return { + 'status': 'success', + 'actions': list(PERMS.keys()), + } + + @ajax_request_class + @action(detail=True, methods=['POST']) + def is_authorized(self, request, pk=None): + """ + Checks the auth for a given action, if user is the owner of the parent + org then True is returned for each action + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID (primary key) + type: integer + required: true + paramType: path + - name: organization_id + description: ID (primary key) for organization + type: integer + required: true + paramType: query + - name: actions + type: array[string] + required: true + description: a list of actions to check + type: + status: + type: string + description: success or error + required: true + message: + type: string + description: error message, if any + required: false + auth: + type: object + description: a dict of with keys equal to the actions, and values as bool + required: true + """ + actions, org, error, message = self._parse_is_authenticated_params(request) + if error: + return JsonResponse({ + 'status': 'error', + 'message': message + }, status=status.HTTP_400_BAD_REQUEST) + + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + + # If the only action requested is 'requires_superuser' no need to check an org affiliation + if len(actions) == 1 and actions[0] == 'requires_superuser': + return JsonResponse( + {'status': 'success', 'auth': {'requires_superuser': user.is_superuser}}) + + auth = self._try_parent_org_auth(user, org, actions) + if auth: + return JsonResponse({'status': 'success', 'auth': auth}) + + try: + ou = OrganizationUser.objects.get(user=user, organization=org) + except OrganizationUser.DoesNotExist: + return JsonResponse({'status': 'error', 'message': 'user does not exist'}) + + auth = {action: PERMS[action](ou) for action in actions} + return JsonResponse({'status': 'success', 'auth': auth}) + + def _parse_is_authenticated_params(self, request): + """checks if the org exists and if the actions are present + + :param request: the request + :returns: tuple (actions, org, error, message) + """ + error = False + message = "" + body = request.data + if not body.get('actions'): + message = 'no actions to check' + error = True + + org_id = request.query_params.get('organization_id', None) + if org_id == '': + message = 'organization id is undefined' + error = True + org = None + else: + try: + org = Organization.objects.get(pk=org_id) + except Organization.DoesNotExist: + message = 'organization does not exist' + error = True + org = None + + return body.get('actions'), org, error, message + + def _try_parent_org_auth(self, user, organization, actions): + """checks the parent org for permissions, if the user is not the owner of + the parent org, then None is returned. + + :param user: the request user + :param organization: org to check its parent + :param actions: list of str actions to check + :returns: a dict of action permission resolutions or None + """ + try: + ou = OrganizationUser.objects.get( + user=user, + organization=organization.parent_org, + role_level=ROLE_OWNER + ) + except OrganizationUser.DoesNotExist: + return None + + return { + action: PERMS['requires_owner'](ou) for action in actions + } + + @ajax_request_class + @action(detail=True, methods=['GET']) + def shared_buildings(self, request, pk=None): + """ + Get the request user's ``show_shared_buildings`` attr + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID (primary key) + type: integer + required: true + paramType: path + type: + status: + type: string + description: success or error + required: true + show_shared_buildings: + type: string + description: the user show shared buildings attribute + required: true + message: + type: string + description: error message, if any + required: false + """ + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + + return JsonResponse({ + 'status': 'success', + 'show_shared_buildings': user.show_shared_buildings, + }) + + @ajax_request_class + @action(detail=True, methods=['PUT']) + def default_organization(self, request, pk=None): + """ + Sets the user's default organization + --- + parameter_strategy: replace + parameters: + - name: pk + description: User ID (primary key) + type: integer + required: true + paramType: path + - name: organization_id + description: The new default organization ID to use for this user + type: integer + required: true + type: + status: + type: string + description: success or error + required: true + message: + type: string + description: error message, if any + required: false + """ + ok, content = self.validate_request_user(pk, request) + if ok: + user = content + else: + return content + user.default_organization_id = request.query_params.get('organization_id', None) + user.save() + return {'status': 'success'} + + @ajax_request_class + @action(detail=True, methods=['DELETE']) + def deactivate(self, request, pk=None): + """ + Deactivates a user + --- + parameter_strategy: replace + parameters: + - name: + description: User ID (primary key) + type: integer + required: true + paramType: path + - name: organization_id + description: The new default organization ID to use for this user + type: integer + required: true + type: + status: + type: string + description: success or error + required: true + message: + type: string + description: error message, if any + required: false + """ + body = request.data + first_name = body['first_name'] + last_name = body['last_name'] + + print(first_name, last_name) + # check if user exists + user = SEEDUser.objects.filter( + first_name=first_name, last_name=last_name + ) + print(user) + if not user.exists(): + return JsonResponse({ + 'status': 'error', + 'message': 'user does not exist', + }, status=status.HTTP_403_FORBIDDEN) + + user[0].deactivate_user() + return JsonResponse({'status': 'success'}) From 5adb233493598cb90c0589ffd658c0a2402fd7a2 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 11 May 2020 12:53:47 -0600 Subject: [PATCH 011/135] fixed flake8 error --- seed/views/v3/users.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 60847ba25b..13c19bfaaa 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -109,6 +109,7 @@ class EmailAndIDSerializer(serializers.Serializer): class ListUsersResponseSerializer(serializers.Serializer): users = EmailAndIDSerializer(many=True) + class UserSchema(AutoSchemaHelper): def __init__(self, *args): super().__init__(*args) @@ -144,7 +145,8 @@ def __init__(self, *args): 'email': 'string' } )], - ('PUT', 'default_organization'): [self.org_id_field(), self.path_id_field(description="Updated Users PK ID")], + ('PUT', 'default_organization'): [self.org_id_field(), + self.path_id_field(description="Updated Users PK ID")], ('POST', 'is_authorized'): [ self.org_id_field(), self.path_id_field(description="Users PK ID"), @@ -193,8 +195,6 @@ def __init__(self, *args): ] - - } From 059ff7c823a43a73f41e97c9d1353367d64c4e9c Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 12 May 2020 08:20:52 -0600 Subject: [PATCH 012/135] finnished cycles api v3 besides patch --- seed/api/v3/urls.py | 6 ++ seed/views/v3/cycles.py | 157 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 seed/views/v3/cycles.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 1882f228c8..788b561169 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -6,11 +6,17 @@ from seed.views.v3.data_quality import DataQualityViews from seed.views.v3.datasets import DatasetViewSet + +from seed.views.v3.cycles import CycleViewSet + api_v3_router = routers.DefaultRouter() api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') +api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles') + + urlpatterns = [ url(r'^', include(api_v3_router.urls)), ] diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py new file mode 100644 index 0000000000..d345b13dfb --- /dev/null +++ b/seed/views/v3/cycles.py @@ -0,0 +1,157 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, +through Lawrence Berkeley National Laboratory (subject to receipt of any +required approvals from the U.S. Department of Energy) and contributors. +All rights reserved. # NOQA +:authors Paul Munday Fable Turas +""" +from seed.filtersets import CycleFilterSet +from seed.models import Cycle + +from seed.serializers.cycles import CycleSerializer +from seed.utils.viewsets import SEEDOrgModelViewSet + + +class CycleViewSet(SEEDOrgModelViewSet): + """API endpoint for viewing and creating cycles (time periods). + + Returns:: + { + 'status': 'success', + 'cycles': [ + { + 'id': Cycle`s primary key, + 'name': Name given to cycle, + 'start': Start date of cycle, + 'end': End date of cycle, + 'created': Created date of cycle, + 'properties_count': Count of properties in cycle, + 'taxlots_count': Count of tax lots in cycle, + 'organization': Id of organization cycle belongs to, + 'user': Id of user who created cycle + } + ] + } + + + retrieve: + Return a cycle instance by pk if it is within user`s specified org. + + :GET: Expects organization_id in query string. + :Parameters: + :Parameter: organization_id + :Description: organization_id for this user`s organization + :required: true + :Parameter: cycle pk + :Description: id for desired cycle + :required: true + + list: + Return all cycles available to user through user`s specified org. + + :GET: Expects organization_id in query string. + :Parameters: + :Parameter: organization_id + :Description: organization_id for this user`s organization + :required: true + :Parameter: name + :Description: optional name for filtering cycles + :required: false + :Parameter: start_lte + :Description: optional iso date for filtering by cycles + that start on or before the given date + :required: false + :Parameter: end_gte + :Description: optional iso date for filtering by cycles + that end on or after the given date + :required: false + + create: + Create a new cycle within user`s specified org. + + :POST: Expects organization_id in query string. + :Parameters: + :Parameter: organization_id + :Description: organization_id for this user`s organization + :required: true + :Parameter: name + :Description: cycle name + :required: true + :Parameter: start + :Description: cycle start date. format: ``YYYY-MM-DDThh:mm`` + :required: true + :Parameter: end + :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` + :required: true + + delete: + Remove an existing cycle. + + :DELETE: Expects organization_id in query string. + :Parameters: + :Parameter: organization_id + :Description: organization_id for this user`s organization + :required: true + :Parameter: cycle pk + :Description: id for desired cycle + :required: true + + update: + Update a cycle record. + + :PUT: Expects organization_id in query string. + :Parameters: + :Parameter: organization_id + :Description: organization_id for this user`s organization + :required: true + :Parameter: cycle pk + :Description: id for desired cycle + :required: true + :Parameter: name + :Description: cycle name + :required: true + :Parameter: start + :Description: cycle start date. format: ``YYYY-MM-DDThh:mm`` + :required: true + :Parameter: end + :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` + :required: true + + partial_update: + Update one or more fields on an existing cycle. + + :PUT: Expects organization_id in query string. + :Parameters: + :Parameter: organization_id + :Description: organization_id for this user`s organization + :required: true + :Parameter: cycle pk + :Description: id for desired cycle + :required: true + :Parameter: name + :Description: cycle name + :required: false + :Parameter: start + :Description: cycle start date. format: ``YYYY-MM-DDThh:mm`` + :required: false + :Parameter: end + :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` + :required: false + """ + serializer_class = CycleSerializer + pagination_class = None + model = Cycle + data_name = 'cycles' + filter_class = CycleFilterSet + + def get_queryset(self): + org_id = self.get_organization(self.request) + # Order cycles by name because if the user hasn't specified then the front end WILL default to the first + return Cycle.objects.filter(organization_id=org_id).order_by('name') + + def perform_create(self, serializer): + org_id = self.get_organization(self.request) + user = self.request.user + serializer.save(organization_id=org_id, user=user) From 830efa099451da2f2c82b13a84b7d626e45f84d4 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 13 May 2020 07:54:12 -0600 Subject: [PATCH 013/135] implemented the wrapper class with no patch --- seed/utils/viewsets.py | 25 +++++++++++++++++++++++++ seed/views/v3/cycles.py | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/seed/utils/viewsets.py b/seed/utils/viewsets.py index 2248a2eeae..5ebe54aed2 100644 --- a/seed/utils/viewsets.py +++ b/seed/utils/viewsets.py @@ -19,6 +19,12 @@ from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from oauth2_provider.ext.rest_framework import OAuth2Authentication +from rest_framework.mixins import ( + CreateModelMixin, + DestroyModelMixin, + UpdateModelMixin, +) + # Local Imports from seed.authentication import SEEDAuthentication from seed.decorators import DecoratorMixin @@ -41,6 +47,15 @@ PERMISSIONS_CLASSES = (SEEDOrgPermissions,) +class UpdateWithoutPatchModelMixin(object): + # Rebuilds the UpdateModelMixin without the patch action + def update(self, request, *args, **kwargs): + return UpdateModelMixin.update(self, request, *args, **kwargs) + + def perform_update(self, serializer): + return UpdateModelMixin.perform_update(self, serializer) + + class SEEDOrgModelViewSet(DecoratorMixin(drf_api_endpoint), OrgQuerySetMixin, ModelViewSet): """Viewset class customized with SEED standard attributes. @@ -85,3 +100,13 @@ class SEEDOrgCreateUpdateModelViewSet(OrgCreateUpdateMixin, SEEDOrgModelViewSet) and/or perform_update overrides appropriate to the model's needs. """ pass + + +class SEEDOrgNoPatchOrOrgCreateModelViewSet(SEEDOrgReadOnlyModelViewSet, + CreateModelMixin, + DestroyModelMixin, + UpdateWithoutPatchModelMixin): + """Extends SEEDOrgReadOnlyModelViewSet to include update (without patch), + create, and destroy actions. + """ + pass diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index d345b13dfb..3de5446da6 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -11,10 +11,10 @@ from seed.models import Cycle from seed.serializers.cycles import CycleSerializer -from seed.utils.viewsets import SEEDOrgModelViewSet +from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet -class CycleViewSet(SEEDOrgModelViewSet): +class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): """API endpoint for viewing and creating cycles (time periods). Returns:: From 0ca2131eebe250b244421780bc2d07c1888ae5a0 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 13 May 2020 08:16:29 -0600 Subject: [PATCH 014/135] added api/v3/cycles to test_api.py --- seed/tests/test_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seed/tests/test_api.py b/seed/tests/test_api.py index 4d9d0754cc..61a753e68e 100644 --- a/seed/tests/test_api.py +++ b/seed/tests/test_api.py @@ -50,7 +50,8 @@ def test_get_api_endpoints_utils(self): '/api/v2/gbr_properties', '/api/v2/notes', '/api/v2.1/properties', - '/api/v2.1/scenarios',)): + '/api/v2.1/scenarios', + '/api/v3/cycles')): self.assertTrue( url.endswith('/'), "Endpoint %s does not end with / as expected" % url From 4ac2bf55fef12460958e8111cdc406aea136cf2e Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 13 May 2020 08:42:23 -0600 Subject: [PATCH 015/135] removed partial update comments --- seed/views/v3/cycles.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 3de5446da6..87958f0f89 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -118,27 +118,6 @@ class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): :Parameter: end :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` :required: true - - partial_update: - Update one or more fields on an existing cycle. - - :PUT: Expects organization_id in query string. - :Parameters: - :Parameter: organization_id - :Description: organization_id for this user`s organization - :required: true - :Parameter: cycle pk - :Description: id for desired cycle - :required: true - :Parameter: name - :Description: cycle name - :required: false - :Parameter: start - :Description: cycle start date. format: ``YYYY-MM-DDThh:mm`` - :required: false - :Parameter: end - :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` - :required: false """ serializer_class = CycleSerializer pagination_class = None From a63baed0d394337ef4868887e539475ac0c3bdd1 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 13 May 2020 11:36:47 -0600 Subject: [PATCH 016/135] added columns v3 --- seed/api/v3/urls.py | 4 + seed/utils/viewsets.py | 25 ++++ seed/views/v3/columns.py | 282 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 311 insertions(+) create mode 100644 seed/views/v3/columns.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 1882f228c8..6c0cef8297 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -5,12 +5,16 @@ from seed.views.v3.data_quality import DataQualityViews from seed.views.v3.datasets import DatasetViewSet +from seed.views.v3.columns import ColumnViewSet api_v3_router = routers.DefaultRouter() api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') +api_v3_router.register(r'columns', ColumnViewSet, base_name='columns') + + urlpatterns = [ url(r'^', include(api_v3_router.urls)), ] diff --git a/seed/utils/viewsets.py b/seed/utils/viewsets.py index 2248a2eeae..5ebe54aed2 100644 --- a/seed/utils/viewsets.py +++ b/seed/utils/viewsets.py @@ -19,6 +19,12 @@ from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from oauth2_provider.ext.rest_framework import OAuth2Authentication +from rest_framework.mixins import ( + CreateModelMixin, + DestroyModelMixin, + UpdateModelMixin, +) + # Local Imports from seed.authentication import SEEDAuthentication from seed.decorators import DecoratorMixin @@ -41,6 +47,15 @@ PERMISSIONS_CLASSES = (SEEDOrgPermissions,) +class UpdateWithoutPatchModelMixin(object): + # Rebuilds the UpdateModelMixin without the patch action + def update(self, request, *args, **kwargs): + return UpdateModelMixin.update(self, request, *args, **kwargs) + + def perform_update(self, serializer): + return UpdateModelMixin.perform_update(self, serializer) + + class SEEDOrgModelViewSet(DecoratorMixin(drf_api_endpoint), OrgQuerySetMixin, ModelViewSet): """Viewset class customized with SEED standard attributes. @@ -85,3 +100,13 @@ class SEEDOrgCreateUpdateModelViewSet(OrgCreateUpdateMixin, SEEDOrgModelViewSet) and/or perform_update overrides appropriate to the model's needs. """ pass + + +class SEEDOrgNoPatchOrOrgCreateModelViewSet(SEEDOrgReadOnlyModelViewSet, + CreateModelMixin, + DestroyModelMixin, + UpdateWithoutPatchModelMixin): + """Extends SEEDOrgReadOnlyModelViewSet to include update (without patch), + create, and destroy actions. + """ + pass diff --git a/seed/views/v3/columns.py b/seed/views/v3/columns.py new file mode 100644 index 0000000000..79ecb24bf9 --- /dev/null +++ b/seed/views/v3/columns.py @@ -0,0 +1,282 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" + +import logging + +import coreapi +from django.http import JsonResponse +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.exceptions import NotFound, ParseError +from rest_framework.filters import BaseFilterBackend +from rest_framework.parsers import JSONParser, FormParser +from rest_framework.renderers import JSONRenderer +from rest_framework.response import Response + +from seed.decorators import ajax_request_class +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import Organization +from seed.models import PropertyState, TaxLotState +from seed.models.columns import Column +from seed.renderers import SEEDJSONRenderer +from seed.serializers.columns import ColumnSerializer +from seed.utils.api import OrgValidateMixin +from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet + +_log = logging.getLogger(__name__) + + +class ColumnViewSetFilterBackend(BaseFilterBackend): + """ + Specify the schema for the column view set. This allows the user to see the other + required columns in Swagger. + """ + + def get_schema_fields(self, view): + return [ + coreapi.Field('organization_id', location='query', required=True, type='integer'), + coreapi.Field('inventory_type', location='query', required=False, type='string'), + coreapi.Field('only_used', location='query', required=False, type='boolean'), + ] + + def filter_queryset(self, request, queryset, view): + return queryset + + +class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet): + raise_exception = True + serializer_class = ColumnSerializer + renderer_classes = (JSONRenderer,) + model = Column + pagination_class = None + parser_classes = (JSONParser, FormParser) + filter_backends = (ColumnViewSetFilterBackend,) + + def get_queryset(self): + # check if the request is properties or taxlots + org_id = self.get_organization(self.request) + return Column.objects.filter(organization_id=org_id) + + @ajax_request_class + def list(self, request): + """ + Retrieves all columns for the user's organization including the raw database columns. Will + return all the columns across both the Property and Tax Lot tables. The related field will + be true if the column came from the other table that is not the 'inventory_type' (which + defaults to Property) + + This is the same results as calling /api/v2//columns/?organization_id={} + + Example: /api/v2/columns/?inventory_type=(property|taxlot)/&organization_id={} + + type: + status: + required: true + type: string + description: Either success or error + columns: + required: true + type: array[column] + description: Returns an array where each item is a full column structure. + parameters: + - name: organization_id + description: The organization_id for this user's organization + required: true + paramType: query + - name: inventory_type + description: Which inventory type is being matched (for related fields and naming). + property or taxlot + required: true + paramType: query + - name: used_only + description: Determine whether or not to show only the used fields (i.e. only columns that have been mapped) + type: boolean + required: false + paramType: query + """ + organization_id = self.get_organization(self.request) + inventory_type = request.query_params.get('inventory_type', 'property') + only_used = request.query_params.get('only_used', False) + columns = Column.retrieve_all(organization_id, inventory_type, only_used) + return JsonResponse({ + 'status': 'success', + 'columns': columns, + }) + + @ajax_request_class + def retrieve(self, request, pk=None): + """ + Retrieves a column (Column) + + type: + status: + required: true + type: string + description: Either success or error + column: + required: true + type: dictionary + description: Returns a dictionary of a full column structure with keys such as + keys ''name'', ''id'', ''is_extra_data'', ''column_name'', + ''table_name'',... + parameters: + - name: organization_id + description: The organization_id for this user's organization + required: true + paramType: query + """ + organization_id = self.get_organization(self.request) + + # check if column exists for the organization + try: + c = Column.objects.get(pk=pk) + except Column.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'column with id {} does not exist'.format(pk) + }, status=status.HTTP_404_NOT_FOUND) + + if c.organization.id != organization_id: + return JsonResponse({ + 'status': 'error', + 'message': 'Organization ID mismatch between column and organization' + }, status=status.HTTP_400_BAD_REQUEST) + + return JsonResponse({ + 'status': 'success', + 'column': ColumnSerializer(c).data + }) + + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=False, methods=['POST']) + def delete_all(self, request): + """ + Delete all columns for an organization. This method is typically not recommended if there + are data in the inventory as it will invalidate all extra_data fields. This also removes + all the column mappings that existed. + + --- + parameters: + - name: organization_id + description: The organization_id + required: true + paramType: query + type: + status: + description: success or error + type: string + required: true + column_mappings_deleted_count: + description: Number of column_mappings that were deleted + type: integer + required: true + columns_deleted_count: + description: Number of columns that were deleted + type: integer + required: true + """ + organization_id = request.query_params.get('organization_id', None) + + try: + org = Organization.objects.get(pk=organization_id) + c_count, cm_count = Column.delete_all(org) + return JsonResponse( + { + 'status': 'success', + 'column_mappings_deleted_count': cm_count, + 'columns_deleted_count': c_count, + } + ) + except Organization.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'organization with with id {} does not exist'.format(organization_id) + }, status=status.HTTP_404_NOT_FOUND) + + @action(detail=False, renderer_classes=(SEEDJSONRenderer,)) + def add_column_names(self, request): + """ + Allow columns to be added based on an existing record. + This may be necessary to make column selections available when + records are upload through API endpoint rather than the frontend. + """ + model_obj = None + org = self.get_organization(request, return_obj=True) + inventory_pk = request.query_params.get('inventory_pk') + inventory_type = request.query_params.get('inventory_type', 'property') + if inventory_type in ['property', 'propertystate']: + if not inventory_pk: + model_obj = PropertyState.objects.filter( + organization=org + ).order_by('-id').first() + try: + model_obj = PropertyState.objects.get(id=inventory_pk) + except PropertyState.DoesNotExist: + pass + elif inventory_type in ['taxlot', 'taxlotstate']: + if not inventory_pk: + model_obj = TaxLotState.objects.filter( + organization=org + ).order_by('-id').first() + else: + try: + model_obj = TaxLotState.objects.get(id=inventory_pk) + inventory_type = 'taxlotstate' + except TaxLotState.DoesNotExist: + pass + else: + msg = "{} is not a valid inventory type".format(inventory_type) + raise ParseError(msg) + if not model_obj: + msg = "No {} was found matching {}".format( + inventory_type, inventory_pk + ) + raise NotFound(msg) + Column.save_column_names(model_obj) + + columns = Column.objects.filter( + organization=model_obj.organization, + table_name=model_obj.__class__.__name__, + is_extra_data=True, + + ) + columns = ColumnSerializer(columns, many=True) + return Response(columns.data, status=status.HTTP_200_OK) + + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def rename(self, request, pk=None): + org_id = self.get_organization(request) + try: + column = Column.objects.get(id=pk, organization_id=org_id) + except Column.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': 'Cannot find column in org=%s with pk=%s' % (org_id, pk) + }, status=status.HTTP_404_NOT_FOUND) + + new_column_name = request.data.get('new_column_name', None) + overwrite = request.data.get('overwrite', False) + if not new_column_name: + return JsonResponse({ + 'success': False, + 'message': 'You must specify the name of the new column as "new_column_name"' + }, status=status.HTTP_400_BAD_REQUEST) + + result = column.rename_column(new_column_name, overwrite) + if not result[0]: + return JsonResponse({ + 'success': False, + 'message': 'Unable to rename column with message: "%s"' % result[1] + }, status=status.HTTP_400_BAD_REQUEST) + else: + return JsonResponse({ + 'success': True, + 'message': result[1] + }) From 6fd9a52ca86aefa6c7b6794bc9025cdce4c4d8a8 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Thu, 14 May 2020 14:25:45 -0600 Subject: [PATCH 017/135] PropertyViewViewSet - remove select_related slowing down swagger page --- seed/views/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/views/properties.py b/seed/views/properties.py index e4d6597fd6..281253f3aa 100644 --- a/seed/views/properties.py +++ b/seed/views/properties.py @@ -199,7 +199,7 @@ class PropertyViewViewSet(SEEDOrgModelViewSet): filter_class = PropertyViewFilterSet orgfilter = 'property__organization_id' data_name = "property_views" - queryset = PropertyView.objects.all().select_related('state') + queryset = PropertyView.objects.all() class PropertyViewSet(GenericViewSet, ProfileIdMixin): From 36a72a071ea5d10fea8852ba3d45e85adb465155 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Fri, 15 May 2020 07:46:02 -0600 Subject: [PATCH 018/135] Fixed reviewed changes from Adrian --- seed/utils/api_schema.py | 3 +-- seed/views/v3/data_quality.py | 4 ++-- seed/views/v3/users.py | 30 +++++++++++++----------------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index f3bb19d452..1d6dcd2b84 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -8,12 +8,11 @@ class AutoSchemaHelper(SwaggerAutoSchema): # Used to easily build out example values displayed on Swagger page. body_parameter_formats = { - 'interger_list': openapi.Schema( + 'interger_array': openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_INTEGER) ), 'string': openapi.Schema(type=openapi.TYPE_STRING), - 'string_array': openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_STRING) diff --git a/seed/views/v3/data_quality.py b/seed/views/v3/data_quality.py index 02a45a39c0..8fb930b2fa 100644 --- a/seed/views/v3/data_quality.py +++ b/seed/views/v3/data_quality.py @@ -105,8 +105,8 @@ def __init__(self, *args): required=True, description="An object containing IDs of the records to perform data quality checks on. Should contain two keys- property_state_ids and taxlot_state_ids, each of which is an array of appropriate IDs.", params_to_formats={ - 'property_state_ids': 'interger_list', - 'taxlot_state_ids': 'interger_list' + 'property_state_ids': 'interger_array', + 'taxlot_state_ids': 'interger_array' } ), ], diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 13c19bfaaa..edddc15609 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -183,15 +183,16 @@ def __init__(self, *args): }) ], - ('DELETE', 'deactivate'): [ - self.body_field( - name='User to be deactivated', - required=True, - description="first and last name of user to be deactivated", - params_to_formats={ - 'first_name': 'string', - 'last_name': 'string' - }) + ('PUT', 'deactivate'): [ + self.path_id_field(description="Users PK ID") + # self.body_field( + # name='User to be deactivated', + # required=True, + # description="first and last name of user to be deactivated", + # params_to_formats={ + # 'first_name': 'string', + # 'last_name': 'string' + # }) ] @@ -841,7 +842,7 @@ def default_organization(self, request, pk=None): return {'status': 'success'} @ajax_request_class - @action(detail=True, methods=['DELETE']) + @action(detail=True, methods=['PUT']) def deactivate(self, request, pk=None): """ Deactivates a user @@ -867,16 +868,11 @@ def deactivate(self, request, pk=None): description: error message, if any required: false """ - body = request.data - first_name = body['first_name'] - last_name = body['last_name'] - - print(first_name, last_name) + user_id = pk # check if user exists user = SEEDUser.objects.filter( - first_name=first_name, last_name=last_name + id=user_id ) - print(user) if not user.exists(): return JsonResponse({ 'status': 'error', From d341fa2b360cb3febef4f79f2aeda1419b0b7c57 Mon Sep 17 00:00:00 2001 From: Austin <31779424+aviveiros11@users.noreply.github.com> Date: Fri, 15 May 2020 14:23:59 -0600 Subject: [PATCH 019/135] Update seed/utils/viewsets.py Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com> --- seed/utils/viewsets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seed/utils/viewsets.py b/seed/utils/viewsets.py index 5ebe54aed2..c7addbf4d6 100644 --- a/seed/utils/viewsets.py +++ b/seed/utils/viewsets.py @@ -48,6 +48,7 @@ class UpdateWithoutPatchModelMixin(object): + # Taken from: https://github.com/encode/django-rest-framework/pull/3081#issuecomment-518396378 # Rebuilds the UpdateModelMixin without the patch action def update(self, request, *args, **kwargs): return UpdateModelMixin.update(self, request, *args, **kwargs) From bf3fe3bd928c97ab8a16daf28ce715a9c720d08e Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 18 May 2020 07:35:25 -0600 Subject: [PATCH 020/135] Adrian review changes --- seed/views/v3/users.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index edddc15609..3ef00b80a8 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -32,6 +32,7 @@ from seed.utils.api import api_endpoint_class from seed.utils.api_schema import AutoSchemaHelper from seed.utils.organizations import create_organization +from rest_framework.status import HTTP_400_BAD_REQUEST _log = logging.getLogger(__name__) @@ -868,16 +869,19 @@ def deactivate(self, request, pk=None): description: error message, if any required: false """ - user_id = pk - # check if user exists - user = SEEDUser.objects.filter( - id=user_id - ) - if not user.exists(): + try: + user_id = pk + user = SEEDUser.objects.get( + id=user_id + ) + user.deactivate_user() + return JsonResponse({ + 'status': 'successfully deactivated', + 'data': user.email + }) + except Exception as e: return JsonResponse({ 'status': 'error', - 'message': 'user does not exist', - }, status=status.HTTP_403_FORBIDDEN) + 'data': str(e), + }, status=HTTP_400_BAD_REQUEST) - user[0].deactivate_user() - return JsonResponse({'status': 'success'}) From 57f62285c1b9c36236d6ece7ef38c25f7274def2 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 18 May 2020 07:48:18 -0600 Subject: [PATCH 021/135] flake8 --- seed/views/v3/users.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 3ef00b80a8..21a53114b2 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -884,4 +884,3 @@ def deactivate(self, request, pk=None): 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST) - From de8160290e144dcceda0f82abb4c8068b772e401 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 18 May 2020 08:46:14 -0600 Subject: [PATCH 022/135] Cleaned up cycles comments --- seed/views/v3/cycles.py | 88 +++++++++-------------------------------- 1 file changed, 18 insertions(+), 70 deletions(-) diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 87958f0f89..71ed86744f 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -7,11 +7,25 @@ All rights reserved. # NOQA :authors Paul Munday Fable Turas """ -from seed.filtersets import CycleFilterSet from seed.models import Cycle from seed.serializers.cycles import CycleSerializer from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet +from seed.utils.api_schema import AutoSchemaHelper + + +class CycleSchema(AutoSchemaHelper): + def __init__(self, *args): + super().__init__(*args) + + self.manual_fields = { + ('GET', 'list'): [], + ('POST', 'create'): [self.org_id_field()], + ('GET', 'retrieve'): [self.org_id_field()], + ('PUT', 'update'): [], + ('DELETE', 'delete'): [self.org_id_field()], + } + class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): @@ -37,93 +51,27 @@ class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): retrieve: - Return a cycle instance by pk if it is within user`s specified org. - - :GET: Expects organization_id in query string. - :Parameters: - :Parameter: organization_id - :Description: organization_id for this user`s organization - :required: true - :Parameter: cycle pk - :Description: id for desired cycle - :required: true + Return a cycle instance by pk if it is within user`s specified org. list: - Return all cycles available to user through user`s specified org. - :GET: Expects organization_id in query string. - :Parameters: - :Parameter: organization_id - :Description: organization_id for this user`s organization - :required: true - :Parameter: name - :Description: optional name for filtering cycles - :required: false - :Parameter: start_lte - :Description: optional iso date for filtering by cycles - that start on or before the given date - :required: false - :Parameter: end_gte - :Description: optional iso date for filtering by cycles - that end on or after the given date - :required: false + Return all cycles available to user through user`s specified org. create: Create a new cycle within user`s specified org. - :POST: Expects organization_id in query string. - :Parameters: - :Parameter: organization_id - :Description: organization_id for this user`s organization - :required: true - :Parameter: name - :Description: cycle name - :required: true - :Parameter: start - :Description: cycle start date. format: ``YYYY-MM-DDThh:mm`` - :required: true - :Parameter: end - :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` - :required: true - delete: Remove an existing cycle. - :DELETE: Expects organization_id in query string. - :Parameters: - :Parameter: organization_id - :Description: organization_id for this user`s organization - :required: true - :Parameter: cycle pk - :Description: id for desired cycle - :required: true - update: Update a cycle record. - :PUT: Expects organization_id in query string. - :Parameters: - :Parameter: organization_id - :Description: organization_id for this user`s organization - :required: true - :Parameter: cycle pk - :Description: id for desired cycle - :required: true - :Parameter: name - :Description: cycle name - :required: true - :Parameter: start - :Description: cycle start date. format: ``YYYY-MM-DDThh:mm`` - :required: true - :Parameter: end - :Description: cycle end date. format: ``YYYY-MM-DDThh:mm`` - :required: true """ serializer_class = CycleSerializer + swagger_schema = CycleSchema pagination_class = None model = Cycle data_name = 'cycles' - filter_class = CycleFilterSet def get_queryset(self): org_id = self.get_organization(self.request) From f3a510d9d09a4e56e768cb20e2a26c466667c29c Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 18 May 2020 09:10:51 -0600 Subject: [PATCH 023/135] flake8 --- seed/views/v3/cycles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 71ed86744f..2e3e917a85 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -27,7 +27,6 @@ def __init__(self, *args): } - class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): """API endpoint for viewing and creating cycles (time periods). From 715cd2b6cb4d2832a546f38581bac5cc2546d5a0 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 18 May 2020 10:06:59 -0600 Subject: [PATCH 024/135] small changes --- seed/views/v3/columns.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/seed/views/v3/columns.py b/seed/views/v3/columns.py index 79ecb24bf9..4a850c2add 100644 --- a/seed/views/v3/columns.py +++ b/seed/views/v3/columns.py @@ -70,9 +70,8 @@ def list(self, request): defaults to Property) This is the same results as calling /api/v2//columns/?organization_id={} - Example: /api/v2/columns/?inventory_type=(property|taxlot)/&organization_id={} - + ___ type: status: required: true @@ -110,24 +109,24 @@ def list(self, request): @ajax_request_class def retrieve(self, request, pk=None): """ - Retrieves a column (Column) - - type: - status: - required: true - type: string - description: Either success or error - column: - required: true - type: dictionary - description: Returns a dictionary of a full column structure with keys such as - keys ''name'', ''id'', ''is_extra_data'', ''column_name'', - ''table_name'',... - parameters: - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query + This API endpoint retrieves a column (Column) + --- + parameters: + - name: organization_id + description: Organization ID + type: integer + required: true + type: + status: + type: string + description: success or error + required: true + column: + required: true + type: dictionary + description: Returns a dictionary of a full column structure with keys such as + keys ''name'', ''id'', ''is_extra_data'', ''column_name'', + ''table_name'',.. """ organization_id = self.get_organization(self.request) From 814f7313150a1f9def7b4d4100d9bad172832fed Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 18 May 2020 13:47:19 -0600 Subject: [PATCH 025/135] more review changes --- seed/api/v3/urls.py | 11 ++++------- seed/views/v3/cycles.py | 6 +++--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 788b561169..1f6ae3c655 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -3,18 +3,15 @@ from django.conf.urls import url, include from rest_framework import routers -from seed.views.v3.data_quality import DataQualityViews +from seed.views.v3.cycles import CycleViewSet from seed.views.v3.datasets import DatasetViewSet +from seed.views.v3.data_quality import DataQualityViews -from seed.views.v3.cycles import CycleViewSet - api_v3_router = routers.DefaultRouter() -api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') -api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') - - api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles') +api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') +api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') urlpatterns = [ diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 2e3e917a85..22b6d03951 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -19,11 +19,11 @@ def __init__(self, *args): super().__init__(*args) self.manual_fields = { - ('GET', 'list'): [], + ('GET', 'list'): [self.org_id_field()], ('POST', 'create'): [self.org_id_field()], ('GET', 'retrieve'): [self.org_id_field()], - ('PUT', 'update'): [], - ('DELETE', 'delete'): [self.org_id_field()], + ('PUT', 'update'): [self.org_id_field()], + ('DELETE', 'destroy'): [self.org_id_field()], } From 03e7061728eb9c562e14d0a8a4ce69b7202a19e1 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 19 May 2020 07:50:57 -0600 Subject: [PATCH 026/135] review changes --- seed/api/v3/urls.py | 1 - seed/views/v3/users.py | 16 +++------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 37b3f35b0e..af1fb8d757 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -11,7 +11,6 @@ api_v3_router = routers.DefaultRouter() api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') - api_v3_router.register(r'users', UserViewSet, base_name='user') diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 21a53114b2..d398774528 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -129,8 +129,8 @@ def __init__(self, *args): 'last_name': 'string', 'role': 'string', 'email': 'string' - } - )], + }) + ], ('GET', 'current_user_id'): [], ('GET', 'retrieve'): [self.path_id_field(description="users PK ID")], ('PUT', 'update'): [ @@ -186,17 +186,7 @@ def __init__(self, *args): ], ('PUT', 'deactivate'): [ self.path_id_field(description="Users PK ID") - # self.body_field( - # name='User to be deactivated', - # required=True, - # description="first and last name of user to be deactivated", - # params_to_formats={ - # 'first_name': 'string', - # 'last_name': 'string' - # }) - ] - } @@ -432,7 +422,7 @@ def role(self, request, pk=None): OrganizationUser.objects.filter( user_id=user_id, - organization_id=body['organization_id'] + organization_id=organization_id ).update(role_level=role) return JsonResponse({'status': 'success'}) From 7de120982c15c8cb28abc4ee82abee7ef4950db3 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 19 May 2020 07:55:28 -0600 Subject: [PATCH 027/135] removed empty line --- seed/views/v3/datasets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/seed/views/v3/datasets.py b/seed/views/v3/datasets.py index eb4cfd9780..9ab7e2d472 100644 --- a/seed/views/v3/datasets.py +++ b/seed/views/v3/datasets.py @@ -57,7 +57,6 @@ def __init__(self, *args): ) ], ('DELETE', 'destroy'): [self.org_id_field()] - } From 31d241f974ef976762d6971b8510666cac072146 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 19 May 2020 12:32:07 -0600 Subject: [PATCH 028/135] added clarification and default role assignment --- seed/views/v3/users.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index d398774528..d1842b18e0 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -122,8 +122,9 @@ def __init__(self, *args): self.body_field( name='New User Fields', required=True, - description="An object containing meta data for a new user: " - "required - [last_name, role(viewer/owner/member), email]", + description="An object containing meta data for a new user: \n" + "- Required - first_name, last_name, email \n" + "- Optional - role viewer(default), member, owner", params_to_formats={ 'first_name': 'string', 'last_name': 'string', @@ -138,8 +139,8 @@ def __init__(self, *args): self.body_field( name='Updated User Fields', required=True, - description="An object containing meta data for a updated user: " - "required - [first_name, last_name, email]", + description="An object containing meta data for a updated user: \n" + "- Required - first_name, last_name, email", params_to_formats={ 'first_name': 'string', 'last_name': 'string', @@ -300,6 +301,8 @@ def create(self, request): role = body['role'] if isinstance(role, dict): role = role['value'] + elif role == 'string': + role = 'viewer' OrganizationUser.objects.filter( organization_id=org.pk, From dd98d2478d8165841186ca4732398b799af06fcf Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 19 May 2020 12:58:02 -0600 Subject: [PATCH 029/135] moved to post and added properties/taxlots to labels endpoint --- seed/api/v3/urls.py | 17 +++- seed/views/v3/labels.py | 198 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 204 insertions(+), 11 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index a8d49171ae..10b5630f88 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -5,16 +5,27 @@ from seed.views.v3.data_quality import DataQualityViews from seed.views.v3.datasets import DatasetViewSet - from seed.views.v3.labels import LabelViewSet +from seed.views.v3.labels import UpdateInventoryLabelsAPIView api_v3_router = routers.DefaultRouter() api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') - - api_v3_router.register(r'labels', LabelViewSet, base_name='labels') + urlpatterns = [ url(r'^', include(api_v3_router.urls)), + url( + r'labels-property/$', + UpdateInventoryLabelsAPIView.as_view(), + {'inventory_type': 'property'}, + name="property-labels", + ), + url( + r'labels-taxlot/$', + UpdateInventoryLabelsAPIView.as_view(), + {'inventory_type': 'taxlot'}, + name="taxlot-labels", + ), ] diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 5a071a3fd4..3af7c3d1bd 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -5,7 +5,8 @@ :author 'Piper Merriam ' """ from collections import namedtuple - +from django.apps import apps +from django.db import IntegrityError from django.core.exceptions import ObjectDoesNotExist from rest_framework import ( response, @@ -14,6 +15,7 @@ from rest_framework.decorators import action from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer +from rest_framework.views import APIView from seed.decorators import DecoratorMixin from seed.filters import ( @@ -103,16 +105,196 @@ def _get_labels(self, request): status_code = status.HTTP_200_OK return response.Response(results, status=status_code) - @action(detail=False, methods=['POST']) - #TODO: apply filters for respective views - def filter(self, request): + def list(self, request): """ - Filters a list of all labels + Returns a list of all labels """ return self._get_labels(request) - def list(self, request): + +class UpdateInventoryLabelsAPIView(APIView): + """API endpoint for viewing and creating labels. + + Returns:: + [ + { + 'id': Label's primary key + 'name': Name given to label + 'color': Color of label, + 'organization_id': Id of organization label belongs to, + 'is_applied': Will be empty array if not applied to property/taxlots + } + ] + + --- + """ + renderer_classes = (JSONRenderer,) + parser_classes = (JSONParser,) + inventory_models = {'property': PropertyView, 'taxlot': TaxLotView} + errors = { + 'disjoint': ErrorState( + status.HTTP_422_UNPROCESSABLE_ENTITY, + 'add_label_ids and remove_label_ids cannot contain elements in common' + ), + 'missing_org': ErrorState( + status.HTTP_422_UNPROCESSABLE_ENTITY, + 'missing organization_id' + ) + } + + @property + def models(self): """ - Returns a list of all labels + Exposes Django's internal models for join table. + + Used for bulk_create operations. """ - return self._get_labels(request) + return { + 'property': apps.get_model('seed', 'PropertyView_labels'), + 'taxlot': apps.get_model('seed', 'TaxLotView_labels') + } + + def get_queryset(self, inventory_type, organization_id): + Model = self.models[inventory_type] + return Model.objects.filter( + statuslabel__super_organization_id=organization_id + ) + + def get_label_desc(self, add_label_ids, remove_label_ids): + return Label.objects.filter( + pk__in=add_label_ids + remove_label_ids + ).values('id', 'color', 'name') + + def get_inventory_id(self, q, inventory_type): + return getattr(q, "{}view_id".format(inventory_type)) + + def exclude(self, qs, inventory_type, label_ids): + exclude = {label: [] for label in label_ids} + for q in qs: + if q.statuslabel_id in label_ids: + inventory_id = self.get_inventory_id(q, inventory_type) + exclude[q.statuslabel_id].append(inventory_id) + return exclude + + def filter_by_inventory(self, qs, inventory_type, inventory_ids): + if inventory_ids: + filterdict = { + "{}view__pk__in".format(inventory_type): inventory_ids + } + qs = qs.filter(**filterdict) + return qs + + def label_factory(self, inventory_type, label_id, inventory_id): + Model = self.models[inventory_type] + + # Ensure the the label org and inventory org are the same + inventory_parent_org_id = getattr(Model, "{}view".format(inventory_type)).get_queryset().get(pk=inventory_id)\ + .cycle.organization.get_parent().id + label_super_org_id = Model.statuslabel.get_queryset().get(pk=label_id).super_organization_id + if inventory_parent_org_id == label_super_org_id: + create_dict = { + 'statuslabel_id': label_id, + "{}view_id".format(inventory_type): inventory_id + } + + return Model(**create_dict) + else: + raise IntegrityError( + 'Label with super_organization_id={} cannot be applied to a record with parent ' + 'organization_id={}.'.format( + label_super_org_id, + inventory_parent_org_id + ) + ) + + def add_labels(self, qs, inventory_type, inventory_ids, add_label_ids): + added = [] + if add_label_ids: + model = self.models[inventory_type] + inventory_model = self.inventory_models[inventory_type] + exclude = self.exclude(qs, inventory_type, add_label_ids) + inventory_ids = inventory_ids if inventory_ids else [ + m.pk for m in inventory_model.objects.all() + ] + new_inventory_labels = [ + self.label_factory(inventory_type, label_id, pk) + for label_id in add_label_ids for pk in inventory_ids + if pk not in exclude[label_id] + ] + model.objects.bulk_create(new_inventory_labels) + added = [ + self.get_inventory_id(m, inventory_type) + for m in new_inventory_labels + ] + return added + + def remove_labels(self, qs, inventory_type, remove_label_ids): + removed = [] + if remove_label_ids: + rqs = qs.filter( + statuslabel_id__in=remove_label_ids + ) + removed = [self.get_inventory_id(q, inventory_type) for q in rqs] + rqs.delete() + return removed + + def post(self, request, inventory_type): + """ + Updates label assignments to inventory items. + + Payload:: + + { + "add_label_ids": {array} Array of label ids to add + "remove_label_ids": {array} Array of label ids to remove + "inventory_ids": {array} Array property/taxlot ids + "organization_id": {integer} The user's org ID + } + + Returns:: + + { + 'status': {string} 'success' or 'error' + 'message': {string} Error message if error + 'num_updated': {integer} Number of properties/taxlots updated + 'labels': [ List of labels affected. + { + 'color': {string} + 'id': {int} + 'label': {'string'} + 'name': {string} + }... + ] + } + + """ + add_label_ids = request.data.get('add_label_ids', []) + remove_label_ids = request.data.get('remove_label_ids', []) + inventory_ids = request.data.get('inventory_ids', None) + organization_id = request.query_params['organization_id'] + error = None + # ensure add_label_ids and remove_label_ids are different + if not set(add_label_ids).isdisjoint(remove_label_ids): + error = self.errors['disjoint'] + elif not organization_id: + error = self.errors['missing_org'] + if error: + result = { + 'status': 'error', + 'message': str(error) + } + status_code = error.status_code + else: + qs = self.get_queryset(inventory_type, organization_id) + qs = self.filter_by_inventory(qs, inventory_type, inventory_ids) + removed = self.remove_labels(qs, inventory_type, remove_label_ids) + added = self.add_labels(qs, inventory_type, inventory_ids, add_label_ids) + num_updated = len(set(added).union(removed)) + labels = self.get_label_desc(add_label_ids, remove_label_ids) + result = { + 'status': 'success', + 'num_updated': num_updated, + 'labels': labels + } + status_code = status.HTTP_200_OK + return response.Response(result, status=status_code) From ca89e2fd548d1be54383c493e82ce4e00d064fd1 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 19 May 2020 13:11:04 -0600 Subject: [PATCH 030/135] navigating merge conflict --- seed/views/v3/labels.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index a06b4b8527..ffc21f867b 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -1,3 +1,5 @@ + + # !/usr/bin/env python # encoding: utf-8 """ @@ -12,7 +14,6 @@ response, status, ) -from rest_framework.decorators import action from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer from rest_framework.views import APIView @@ -24,6 +25,8 @@ ) from seed.models import ( StatusLabel as Label, + PropertyView, + TaxLotView, ) from seed.serializers.labels import ( LabelSerializer, @@ -103,17 +106,11 @@ def _get_labels(self, request): status_code = status.HTTP_200_OK return response.Response(results, status=status_code) - - - - - - - - - - - + def list(self, request): + """ + Returns a list of all labels + """ + return self._get_labels(request) class UpdateInventoryLabelsAPIView(APIView): From fc0c3e611df2246301eeb6176a3a446bc8db4a0a Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 19 May 2020 13:13:12 -0600 Subject: [PATCH 031/135] travis --- seed/views/v3/users.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index d1842b18e0..f90ae5e65b 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -122,8 +122,8 @@ def __init__(self, *args): self.body_field( name='New User Fields', required=True, - description="An object containing meta data for a new user: \n" - "- Required - first_name, last_name, email \n" + description="An object containing meta data for a new user: " + "- Required - first_name, last_name, email " "- Optional - role viewer(default), member, owner", params_to_formats={ 'first_name': 'string', @@ -139,7 +139,7 @@ def __init__(self, *args): self.body_field( name='Updated User Fields', required=True, - description="An object containing meta data for a updated user: \n" + description="An object containing meta data for a updated user: " "- Required - first_name, last_name, email", params_to_formats={ 'first_name': 'string', From f15fa05f77737dd542b8bc1e21a5056de0485c0e Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 19 May 2020 17:06:39 -0400 Subject: [PATCH 032/135] feat: buildingsync column mapping presets (#2213) * feat: column mapping presets for buildingsync * fix: address PR comments * chore: add units for bsync columns * chore: fix migrations * chore: address PR comments * chore: allow changing from_units for bsync custom presets Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com> --- seed/api/v2_1/views.py | 33 ++- seed/building_sync/building_sync.py | 47 +-- seed/building_sync/mappings.py | 1 + .../tests/test_buildingsync_views.py | 10 +- .../tests/integration/test_data_import.py | 10 +- seed/data_importer/tests/util.py | 16 -- seed/lib/xml_mapping/mapper.py | 40 ++- .../0126_columnmappingpreset_preset_type.py | 34 +++ seed/models/column_mapping_presets.py | 27 ++ seed/serializers/column_mapping_presets.py | 20 ++ seed/static/seed/js/constants.js | 8 + .../column_mapping_preset_modal_controller.js | 1 + .../controllers/column_mappings_controller.js | 40 ++- .../export_buildingsync_modal_controller.js | 49 ++++ .../inventory_detail_controller.js | 36 ++- .../seed/js/controllers/mapping_controller.js | 89 ++++-- seed/static/seed/js/seed.js | 35 ++- .../js/services/column_mappings_service.js | 12 +- .../static/seed/partials/column_mappings.html | 31 +- .../partials/export_buildingsync_modal.html | 27 ++ seed/static/seed/partials/mapping.html | 21 +- seed/templates/seed/_scripts.html | 3 + .../tests/test_column_mapping_preset_views.py | 271 +++++++++++++++++- seed/tests/test_property_views.py | 91 +++++- seed/utils/organizations.py | 12 +- seed/views/column_mapping_presets.py | 104 +++++-- 26 files changed, 909 insertions(+), 159 deletions(-) create mode 100644 seed/migrations/0126_columnmappingpreset_preset_type.py create mode 100644 seed/static/seed/js/constants.js create mode 100644 seed/static/seed/js/controllers/export_buildingsync_modal_controller.js create mode 100644 seed/static/seed/partials/export_buildingsync_modal.html diff --git a/seed/api/v2_1/views.py b/seed/api/v2_1/views.py index e20f6266ee..694741da29 100644 --- a/seed/api/v2_1/views.py +++ b/seed/api/v2_1/views.py @@ -24,6 +24,7 @@ PropertyState, BuildingFile, Cycle, + ColumnMappingPreset, ) from seed.serializers.properties import ( PropertyViewAsStateSerializer, @@ -169,6 +170,23 @@ def building_sync(self, request, pk): required: true paramType: query """ + preset_pk = request.GET.get('preset_id') + try: + preset_pk = int(preset_pk) + column_mapping_preset = ColumnMappingPreset.objects.get( + pk=preset_pk, + preset_type__in=[ColumnMappingPreset.BUILDINGSYNC_DEFAULT, ColumnMappingPreset.BUILDINGSYNC_CUSTOM]) + except TypeError: + return JsonResponse({ + 'success': False, + 'message': 'Query param `preset_id` is either missing or invalid' + }, status=status.HTTP_400_BAD_REQUEST) + except ColumnMappingPreset.DoesNotExist: + return JsonResponse({ + 'success': False, + 'message': f'Cannot find a BuildingSync ColumnMappingPreset with pk={preset_pk}' + }, status=status.HTTP_400_BAD_REQUEST) + try: # TODO: not checking organization? Is that right? # TODO: this needs to call _get_property_view and use the property pk, not the property_view pk. @@ -178,19 +196,22 @@ def building_sync(self, request, pk): return JsonResponse({ 'success': False, 'message': 'Cannot match a PropertyView with pk=%s' % pk - }) + }, status=status.HTTP_400_BAD_REQUEST) bs = BuildingSync() # Check if there is an existing BuildingSync XML file to merge bs_file = property_view.state.building_files.order_by('created').last() if bs_file is not None and os.path.exists(bs_file.file.path): bs.import_file(bs_file.file.path) - xml = bs.export(property_view.state) - return HttpResponse(xml, content_type='application/xml') - else: - # create a new XML from the record, do not import existing XML - xml = bs.export(property_view.state) + + try: + xml = bs.export_using_preset(property_view.state, column_mapping_preset.mappings) return HttpResponse(xml, content_type='application/xml') + except Exception as e: + return JsonResponse({ + 'success': False, + 'message': str(e) + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @action(detail=True, methods=['GET']) def hpxml(self, request, pk): diff --git a/seed/building_sync/building_sync.py b/seed/building_sync/building_sync.py index f5cbeb9826..5f5417fdfe 100644 --- a/seed/building_sync/building_sync.py +++ b/seed/building_sync/building_sync.py @@ -117,12 +117,21 @@ def init_tree(self, version=BUILDINGSYNC_V2_0): self.element_tree = etree.parse(StringIO(xml_string)) self.version = version - def export(self, property_state, custom_mapping=None): + def export_using_preset(self, property_state, column_mapping_preset=None): """Export BuildingSync file from an existing BuildingSync file (from import), property_state and a custom mapping. + expected column_mapping_preset structure + [ + {from_field: , from_field_value: 'text' | @ | ..., to_field: }, + {from_field: , from_field_value: 'text' | @ | ..., to_field: }, + . + . + . + ] + :param property_state: object, PropertyState to merge into BuildingSync - :param custom_mapping: dict, user-defined mapping (used with higher priority over the default mapping) + :param column_mapping_preset: list, mappings from ColumnMappingPreset :return: string, as XML """ if not property_state: @@ -131,39 +140,33 @@ def export(self, property_state, custom_mapping=None): if not self.element_tree: self.init_tree(version=BuildingSync.BUILDINGSYNC_V2_0) - merged_mappings = merge_mappings(self.VERSION_MAPPINGS_DICT[self.version], custom_mapping) schema = self.get_schema(self.version) - # iterate through the 'property' field mappings doing the following + # iterate through the mappings doing the following # - if the property_state has the field, update the xml with that value # - else, ignore it - base_path = merged_mappings['property']['xpath'] - field_mappings = merged_mappings['property']['properties'] - for field, mapping in field_mappings.items(): - value = None + for mapping in column_mapping_preset: + field = mapping['to_field'] + xml_element_xpath = mapping['from_field'] + xml_element_value = mapping['from_field_value'] + seed_value = None try: property_state._meta.get_field(field) - value = getattr(property_state, field) + seed_value = getattr(property_state, field) except FieldDoesNotExist: _log.debug("Field {} is not a db field, trying read from extra data".format(field)) - value = property_state.extra_data.get(field, None) + seed_value = property_state.extra_data.get(field, None) - if value is None: + if seed_value is None: continue - if isinstance(value, ureg.Quantity): - value = value.magnitude - - if mapping['xpath'].startswith('./'): - mapping_path = mapping['xpath'][2:] - else: - mapping_path = mapping['xpath'] - absolute_xpath = os.path.join(base_path, mapping_path) + if isinstance(seed_value, ureg.Quantity): + seed_value = seed_value.magnitude - update_tree(schema, self.element_tree, absolute_xpath, - mapping['value'], str(value), NAMESPACES) + update_tree(schema, self.element_tree, xml_element_xpath, + xml_element_value, str(seed_value), NAMESPACES) # Not sure why, but lxml was not pretty printing if the tree was updated - # a hack to fix this, we just export the tree, parse it, then export again + # As a hack to fix this, we just export the tree, parse it, then export again xml_bytes = etree.tostring(self.element_tree, pretty_print=True) tree = etree.parse(BytesIO(xml_bytes)) return etree.tostring(tree, pretty_print=True).decode() diff --git a/seed/building_sync/mappings.py b/seed/building_sync/mappings.py index 0557349a87..23d2268785 100644 --- a/seed/building_sync/mappings.py +++ b/seed/building_sync/mappings.py @@ -507,6 +507,7 @@ def update_tree(schema, tree, xpath, target, value, namespaces): 'type': 'value', 'value': 'text', 'formatter': to_float, + 'units': 'ft**2', }, 'net_floor_area': { 'xpath': './auc:Buildings/auc:Building/auc:FloorAreas/auc:FloorArea[auc:FloorAreaType="Net"]/auc:FloorAreaValue', diff --git a/seed/building_sync/tests/test_buildingsync_views.py b/seed/building_sync/tests/test_buildingsync_views.py index e98444d767..9d9ee80aa7 100644 --- a/seed/building_sync/tests/test_buildingsync_views.py +++ b/seed/building_sync/tests/test_buildingsync_views.py @@ -16,6 +16,7 @@ from seed.models import ( PropertyView, StatusLabel, + ColumnMappingPreset, ) from seed.test_helpers.fake import ( FakeCycleFactory, FakeColumnFactory, @@ -50,6 +51,8 @@ def setUp(self): start=datetime(2010, 10, 10, tzinfo=timezone.get_current_timezone()) ) + self.default_bsync_preset = ColumnMappingPreset.objects.get(preset_type=ColumnMappingPreset.BUILDINGSYNC_DEFAULT) + self.client.login(**user_details) def test_get_building_sync(self): @@ -61,7 +64,8 @@ def test_get_building_sync(self): # go to buildingsync endpoint params = { - 'organization_id': self.org.pk + 'organization_id': self.org.pk, + 'preset_id': self.default_bsync_preset.id } url = reverse('api:v2.1:properties-building-sync', args=[pv.id]) response = self.client.get(url, params) @@ -90,7 +94,7 @@ def test_upload_and_get_building_sync(self): # now get the building sync that was just uploaded property_id = result['data']['property_view']['id'] url = reverse('api:v2.1:properties-building-sync', args=[property_id]) - response = self.client.get(url) + response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id}) self.assertIn('1967', response.content.decode("utf-8")) @@ -182,6 +186,6 @@ def test_upload_and_get_building_sync_diff_ns(self): # now get the building sync that was just uploaded property_id = result['data']['property_view']['id'] url = reverse('api:v2.1:properties-building-sync', args=[property_id]) - response = self.client.get(url) + response = self.client.get(url, {'organization_id': self.org.pk, 'preset_id': self.default_bsync_preset.id}) self.assertIn('1889', response.content.decode('utf-8')) diff --git a/seed/data_importer/tests/integration/test_data_import.py b/seed/data_importer/tests/integration/test_data_import.py index 29d336dd13..5b24f266b2 100644 --- a/seed/data_importer/tests/integration/test_data_import.py +++ b/seed/data_importer/tests/integration/test_data_import.py @@ -26,7 +26,6 @@ FAKE_EXTRA_DATA, FAKE_MAPPINGS, FAKE_ROW, - mock_buildingsync_mapping ) from seed.models import ( ASSESSED_RAW, @@ -43,6 +42,7 @@ BuildingFile, ) from seed.tests.util import DataMappingBaseTestCase +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings _log = logging.getLogger(__name__) @@ -240,7 +240,7 @@ def test_map_data_zip(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 2) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # -- Act @@ -260,7 +260,7 @@ def test_map_all_models_zip(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 2) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # map the data @@ -327,7 +327,7 @@ def test_map_all_models_xml(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 1) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # map the data @@ -386,7 +386,7 @@ def test_map_all_models_xml(self): self.assertEqual(PropertyState.objects.filter(import_file=self.import_file).count(), 1) # make the column mappings - self.fake_mappings = mock_buildingsync_mapping() + self.fake_mappings = default_buildingsync_preset_mappings() Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # map the data diff --git a/seed/data_importer/tests/util.py b/seed/data_importer/tests/util.py index e741260550..25ec0c4735 100644 --- a/seed/data_importer/tests/util.py +++ b/seed/data_importer/tests/util.py @@ -7,7 +7,6 @@ import logging -from seed.building_sync.mappings import BASE_MAPPING_V2_0, xpath_to_column_map logger = logging.getLogger(__name__) @@ -269,18 +268,3 @@ "to_table_name": 'PropertyState', "to_field": 'property_footprint', } - - -def mock_buildingsync_mapping(): - # returns a column mapping for bsync files - xpath_to_col = xpath_to_column_map(BASE_MAPPING_V2_0) - result = [] - for xpath, col in xpath_to_col.items(): - result.append( - { - 'from_field': xpath, - 'to_field': col, - 'to_table_name': 'PropertyState' - } - ) - return result diff --git a/seed/lib/xml_mapping/mapper.py b/seed/lib/xml_mapping/mapper.py index 32cfc94203..51733df26b 100644 --- a/seed/lib/xml_mapping/mapper.py +++ b/seed/lib/xml_mapping/mapper.py @@ -9,7 +9,7 @@ import logging from seed.building_sync.building_sync import BuildingSync -from seed.building_sync.mappings import merge_mappings, xpath_to_column_map +from seed.building_sync.mappings import merge_mappings, xpath_to_column_map, BASE_MAPPING_V2_0 _log = logging.getLogger(__name__) @@ -23,3 +23,41 @@ def build_column_mapping(base_mapping=None, custom_mapping=None): xpath: ('PropertyState', db_column, 100) for xpath, db_column in column_mapping.items() } + + +def default_buildingsync_preset_mappings(): + """Returns the default ColumnMappingPreset mappings for BuildingSync + :return: list + """ + # taken from mapping partial (./static/seed/partials/mapping.html) + valid_units = [ + # area units + "ft**2", + "m**2", + # eui_units + "kBtu/ft**2/year", + "kWh/m**2/year", + "GJ/m**2/year", + "MJ/m**2/year", + "kBtu/m**2/year", + ] + + mapping = BASE_MAPPING_V2_0.copy() + base_path = mapping['property']['xpath'].rstrip('/') + result = [] + for col_name, col_info in mapping['property']['properties'].items(): + from_units = col_info.get('units') + if from_units not in valid_units: + from_units = None + + sub_path = col_info['xpath'].replace('./', '') + absolute_xpath = f'{base_path}/{sub_path}' + result.append({ + 'from_field': absolute_xpath, + 'from_field_value': col_info['value'], + 'from_units': from_units, + 'to_field': col_name, + 'to_table_name': 'PropertyState' + }) + + return result diff --git a/seed/migrations/0126_columnmappingpreset_preset_type.py b/seed/migrations/0126_columnmappingpreset_preset_type.py new file mode 100644 index 0000000000..471196d9b7 --- /dev/null +++ b/seed/migrations/0126_columnmappingpreset_preset_type.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.10 on 2020-05-01 21:24 + +from django.db import migrations, models + +from seed.lib.xml_mapping.mapper import default_buildingsync_preset_mappings + + +def create_default_bsync_presets(apps, schema_editor): + """create a default BuildingSync column mapping preset for each organization""" + Organization = apps.get_model("orgs", "Organization") + + for org in Organization.objects.all(): + bsync_mapping_name = 'BuildingSync v2.0 Defaults' + org.columnmappingpreset_set.create( + name=bsync_mapping_name, + mappings=default_buildingsync_preset_mappings(), + preset_type=1 + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('seed', '0125_dq_refactor'), + ] + + operations = [ + migrations.AddField( + model_name='columnmappingpreset', + name='preset_type', + field=models.IntegerField(choices=[(0, 'Normal'), (1, 'BuildingSync Default'), (2, 'BuildingSync Custom')], default=0), + ), + migrations.RunPython(create_default_bsync_presets), + ] diff --git a/seed/models/column_mapping_presets.py b/seed/models/column_mapping_presets.py index 6badf1dec5..a65b04d249 100644 --- a/seed/models/column_mapping_presets.py +++ b/seed/models/column_mapping_presets.py @@ -8,6 +8,16 @@ class ColumnMappingPreset(models.Model): + NORMAL = 0 + BUILDINGSYNC_DEFAULT = 1 + BUILDINGSYNC_CUSTOM = 2 + + COLUMN_MAPPING_PRESET_TYPES = ( + (NORMAL, 'Normal'), + (BUILDINGSYNC_DEFAULT, 'BuildingSync Default'), + (BUILDINGSYNC_CUSTOM, 'BuildingSync Custom') + ) + name = models.CharField(max_length=255, blank=False) mappings = JSONField(default=list, blank=True) @@ -15,3 +25,20 @@ class ColumnMappingPreset(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + + preset_type = models.IntegerField(choices=COLUMN_MAPPING_PRESET_TYPES, default=NORMAL) + + @classmethod + def get_preset_type(cls, preset_type): + """Returns the integer value for a preset type. Raises exception when + preset_type is invalid. + + :param preset_type: int | str + :return: str + """ + if isinstance(preset_type, int): + return preset_type + types_dict = dict((v, k) for k, v in cls.COLUMN_MAPPING_PRESET_TYPES) + if preset_type in types_dict: + return types_dict[preset_type] + raise Exception(f'Invalid preset type "{preset_type}"') diff --git a/seed/serializers/column_mapping_presets.py b/seed/serializers/column_mapping_presets.py index 9d14aaa584..b9b466ab93 100644 --- a/seed/serializers/column_mapping_presets.py +++ b/seed/serializers/column_mapping_presets.py @@ -3,10 +3,30 @@ from rest_framework import serializers +from seed.serializers.base import ChoiceField from seed.models import ColumnMappingPreset class ColumnMappingPresetSerializer(serializers.ModelSerializer): + preset_type = ChoiceField(choices=ColumnMappingPreset.COLUMN_MAPPING_PRESET_TYPES, default=ColumnMappingPreset.NORMAL) + class Meta: model = ColumnMappingPreset fields = '__all__' + + def validate_mappings(self, mappings): + """if the preset is for BuildingSync, make sure it has valid mappings""" + preset_types_dict = dict(ColumnMappingPreset.COLUMN_MAPPING_PRESET_TYPES) + bsync_presets = [ + preset_types_dict[ColumnMappingPreset.BUILDINGSYNC_DEFAULT], + preset_types_dict[ColumnMappingPreset.BUILDINGSYNC_CUSTOM] + ] + preset_type = self.initial_data.get('preset_type') + if preset_type is None or preset_type not in bsync_presets: + return mappings + + for mapping in mappings: + if mapping.get('from_field_value') is None: + raise serializers.ValidationError(f'All BuildingSync mappings must include "from_field_value": for mapping {mapping["from_field"]}') + + return mappings diff --git a/seed/static/seed/js/constants.js b/seed/static/seed/js/constants.js new file mode 100644 index 0000000000..fc79d76ef1 --- /dev/null +++ b/seed/static/seed/js/constants.js @@ -0,0 +1,8 @@ +/** + * :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. + * :author + */ +angular.module('BE.seed.constants', []) + .constant('COLUMN_MAPPING_PRESET_TYPE_NORMAL', 'Normal') + .constant('COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', 'BuildingSync Default') + .constant('COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', 'BuildingSync Custom') diff --git a/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js b/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js index 36dfbff8cb..d051da52a5 100644 --- a/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js +++ b/seed/static/seed/js/controllers/column_mapping_preset_modal_controller.js @@ -36,6 +36,7 @@ angular.module('BE.seed.controller.column_mapping_preset_modal', []) column_mappings_service.new_column_mapping_preset_for_org($scope.org_id, { name: $scope.newName, mappings: $scope.data.mappings, + preset_type: $scope.data.preset_type, }).then(function (result) { $uibModalInstance.close(result.data); }); diff --git a/seed/static/seed/js/controllers/column_mappings_controller.js b/seed/static/seed/js/controllers/column_mappings_controller.js index ff9aaac5fa..29362fcf7a 100644 --- a/seed/static/seed/js/controllers/column_mappings_controller.js +++ b/seed/static/seed/js/controllers/column_mappings_controller.js @@ -16,6 +16,9 @@ angular.module('BE.seed.controller.column_mappings', []) 'mappable_taxlot_columns_payload', 'organization_payload', 'urls', + 'COLUMN_MAPPING_PRESET_TYPE_NORMAL', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', function ( $scope, $state, @@ -28,7 +31,10 @@ angular.module('BE.seed.controller.column_mappings', []) mappable_property_columns_payload, mappable_taxlot_columns_payload, organization_payload, - urls + urls, + COLUMN_MAPPING_PRESET_TYPE_NORMAL, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, ) { $scope.org = organization_payload.organization; $scope.auth = auth_payload.auth; @@ -120,12 +126,18 @@ angular.module('BE.seed.controller.column_mappings', []) // Preset-level CRUD modal-rending actions $scope.new_preset = function () { + const presetData = JSON.parse(JSON.stringify($scope.current_preset)) + // change the preset type to custom if we've edited a default preset + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT) { + presetData.preset_type = COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + } + var modalInstance = $uibModal.open({ templateUrl: urls.static_url + 'seed/partials/column_mapping_preset_modal.html', controller: 'column_mapping_preset_modal_controller', resolve: { action: _.constant('new'), - data: _.constant($scope.current_preset), + data: _.constant(presetData), org_id: _.constant($scope.org.id), } }); @@ -317,4 +329,28 @@ angular.module('BE.seed.controller.column_mappings', []) _.values(grouped_by_from_field), function (group) {return group.length > 1}) ); }; + + $scope.preset_action_ok = (action) => { + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_NORMAL) { + return true + } + + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT) { + return false + } + + if ($scope.current_preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM) { + const allowed_actions = [ + 'update', + 'rename', + 'delete', + 'change_to_field', + 'change_from_units', + ] + return allowed_actions.includes(action) + } + + console.warn(`Unknown preset type "${$scope.current_preset.preset_type}"`) + return false + } }]); diff --git a/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js b/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js new file mode 100644 index 0000000000..b691f634f8 --- /dev/null +++ b/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js @@ -0,0 +1,49 @@ +/** + * :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. + * :author + */ +angular.module('BE.seed.controller.export_buildingsync_modal', []) + .controller('export_buildingsync_modal_controller', [ + '$http', + '$window', + '$scope', + '$uibModalInstance', + 'property_view_id', + 'column_mapping_presets', + function ( + $http, + $window, + $scope, + $uibModalInstance, + property_view_id, + column_mapping_presets, + ) { + $scope.column_mapping_presets = column_mapping_presets + $scope.current_column_mapping_preset = column_mapping_presets[0] + + $scope.download_file = () => { + let the_url = '/api/v2_1/properties/' + property_view_id + '/building_sync/'; + $http.get( + the_url, + { params: { preset_id: $scope.current_column_mapping_preset.id } } + ).then(response => { + let blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'}); + let downloadLink = angular.element(''); + let filename = 'buildingsync_property_' + property_view_id + '.xml'; + downloadLink.attr('href', $window.URL.createObjectURL(blob)); + downloadLink.attr('download', filename); + downloadLink[0].click(); + $uibModalInstance.close() + }, err => { + $scope.download_error_message = err.data ? err.data : err.toString() + }) + } + + $scope.close = () => { + $uibModalInstance.close(); + }; + + $scope.cancel = () => { + $uibModalInstance.dismiss(); + }; + }]); diff --git a/seed/static/seed/js/controllers/inventory_detail_controller.js b/seed/static/seed/js/controllers/inventory_detail_controller.js index d53c4d7e47..99ac3fb30d 100644 --- a/seed/static/seed/js/controllers/inventory_detail_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_controller.js @@ -500,16 +500,32 @@ angular.module('BE.seed.controller.inventory_detail', []) }; $scope.export_building_sync = function () { - var the_url = '/api/v2_1/properties/' + $stateParams.view_id + '/building_sync/'; - $http.get(the_url, {}) - .then(function (response) { - var blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'}); - var downloadLink = angular.element(''); - var filename = 'buildingsync_property_' + $stateParams.view_id + '.xml'; - downloadLink.attr('href', $window.URL.createObjectURL(blob)); - downloadLink.attr('download', filename); - downloadLink[0].click(); - }); + const modalInstance = $uibModal.open({ + templateUrl: urls.static_url + 'seed/partials/export_buildingsync_modal.html', + controller: 'export_buildingsync_modal_controller', + resolve: { + property_view_id: function() { return $stateParams.view_id }, + column_mapping_presets: [ + 'column_mappings_service', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', + function ( + column_mappings_service, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + ) { + const filter_preset_types = [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + ] + return column_mappings_service.get_column_mapping_presets_for_org( + $scope.organization.id, + filter_preset_types, + ).then(response => response.data); + }] + } + }) + modalInstance.result.then(() => { return; }) }; $scope.export_building_sync_xlsx = function () { diff --git a/seed/static/seed/js/controllers/mapping_controller.js b/seed/static/seed/js/controllers/mapping_controller.js index 0fdcfb7783..ebed94d107 100644 --- a/seed/static/seed/js/controllers/mapping_controller.js +++ b/seed/static/seed/js/controllers/mapping_controller.js @@ -30,6 +30,9 @@ angular.module('BE.seed.controller.mapping', []) 'i18nService', // from ui-grid 'simple_modal_service', 'Notification', + 'COLUMN_MAPPING_PRESET_TYPE_NORMAL', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', function ( $scope, $log, @@ -56,7 +59,10 @@ angular.module('BE.seed.controller.mapping', []) $translate, i18nService, simple_modal_service, - Notification + Notification, + COLUMN_MAPPING_PRESET_TYPE_NORMAL, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, ) { $scope.presets = [ {id: 0, mappings: [], name: ""} @@ -76,6 +82,16 @@ angular.module('BE.seed.controller.mapping', []) $scope.mappings_change_possible = true; }; + $scope.is_buildingsync_and_preset_not_ok = function() { + if (!$scope.mappingBuildingSync) { + return false + } + // BuildingSync requires a saved preset to be applied + return $scope.current_preset.id === 0 + || $scope.preset_change_possible + || $scope.mappings_change_possible + } + var analyze_chosen_inventory_types = function () { var chosenTypes = _.uniq(_.map($scope.mappings, 'suggestion_table_name')); var all_cols_have_table_name = !_.find($scope.mappings, {suggestion_table_name: undefined}); @@ -113,26 +129,63 @@ angular.module('BE.seed.controller.mapping', []) var preset_mappings_from_working_mappings = function () { // for to_field, try DB col name, if not use col display name return _.reduce($scope.mappings, function (preset_mapping_data, mapping) { - preset_mapping_data.push({ + const this_mapping = { from_field: mapping.name, from_units: mapping.from_units, to_field: mapping.suggestion_column_name || mapping.suggestion || '', to_table_name: mapping.suggestion_table_name - }); - + }; + const isBuildingSyncPreset = $scope.current_preset.preset_type !== undefined + && [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ].includes($scope.current_preset.preset_type) + + if (isBuildingSyncPreset) { + this_mapping.from_field_value = mapping.from_field_value + } + preset_mapping_data.push(this_mapping) return preset_mapping_data; }, []); }; $scope.new_preset = function () { - var preset_mapping_data = preset_mappings_from_working_mappings(); + let preset_mapping_data = preset_mappings_from_working_mappings(); + + let presetType; + if (!$scope.mappingBuildingSync) { + presetType = COLUMN_MAPPING_PRESET_TYPE_NORMAL + } else { + presetType = COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + + // make sure the new preset mapping data has the required data + const currentPresetForBuildingSync = + $scope.current_preset.preset_type !== undefined + && [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ].includes($scope.current_preset.preset_type) + + if (!currentPresetForBuildingSync) { + // we need to add mapping data, from_field_value, using the default mapping + const defaultPreset = $scope.presets.find(preset => preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT) + preset_mapping_data = preset_mapping_data.map(mapping => { + // find the corresponding mapping in the default preset + const defaultMapping = defaultPreset.mappings.find(defaultMapping => defaultMapping.from_field === mapping.from_field) + return { + ...mapping, + from_field_value: defaultMapping.from_field_value + } + }) + } + } var modalInstance = $uibModal.open({ templateUrl: urls.static_url + 'seed/partials/column_mapping_preset_modal.html', controller: 'column_mapping_preset_modal_controller', resolve: { action: _.constant('new'), - data: _.constant({mappings: preset_mapping_data}), + data: _.constant({mappings: preset_mapping_data, preset_type: presetType}), org_id: _.constant(user_service.get_organization().id) } }); @@ -197,6 +250,8 @@ angular.module('BE.seed.controller.mapping', []) $scope.isValidCycle = Boolean(_.find(cycles.cycles, {id: $scope.import_file.cycle})); + $scope.mappingBuildingSync = $scope.import_file.source_type === 'BuildingSync Raw' + matching_criteria_columns_payload.PropertyState = _.map(matching_criteria_columns_payload.PropertyState, function (column_name) { var display_name = _.find($scope.mappable_property_columns, {column_name: column_name}).display_name; return { @@ -219,10 +274,13 @@ angular.module('BE.seed.controller.mapping', []) $scope.setAllFieldsOptions = [{ name: 'Property', value: 'PropertyState' - }, { - name: 'Tax Lot', - value: 'TaxLotState' }]; + if (!$scope.mappingBuildingSync) { + $scope.setAllFieldsOptions.push({ + name: 'Tax Lot', + value: 'TaxLotState' + }) + } var eui_columns = _.filter($scope.mappable_property_columns, {data_type: 'eui'}); $scope.is_eui_column = function (col) { @@ -332,22 +390,13 @@ angular.module('BE.seed.controller.mapping', []) }; const get_col_from_suggestion = (name) => { - if ($scope.import_file.source_type === "BuildingSync Raw") { - const suggestion = $scope.suggested_mappings[name]; - - return { - name: name, - suggestion_column_name: suggestion[1], - suggestion_table_name: suggestion[0], - raw_data: _.map(first_five_rows_payload.first_five_rows, name) - }; - } const suggestion = _.find($scope.current_preset.mappings, {from_field: name}) || {}; return { from_units: suggestion.from_units, name: name, + from_field_value: suggestion.from_field_value, raw_data: _.map(first_five_rows_payload.first_five_rows, name), suggestion: suggestion.to_field, suggestion_column_name: suggestion.to_field, @@ -378,7 +427,7 @@ angular.module('BE.seed.controller.mapping', []) } if (match) { col.suggestion = match.display_name; - } else if ($scope.import_file.source_type === "BuildingSync Raw") { + } else if ($scope.mappingBuildingSync) { col.suggestion = $filter('titleCase')(col.suggestion_column_name); } diff --git a/seed/static/seed/js/seed.js b/seed/static/seed/js/seed.js index 4f41f331eb..ddad6a7e08 100644 --- a/seed/static/seed/js/seed.js +++ b/seed/static/seed/js/seed.js @@ -55,6 +55,7 @@ angular.module('BE.seed.controllers', [ 'BE.seed.controller.delete_modal', 'BE.seed.controller.delete_user_modal', 'BE.seed.controller.developer', + 'BE.seed.controller.export_buildingsync_modal', 'BE.seed.controller.export_report_modal', 'BE.seed.controller.export_inventory_modal', 'BE.seed.controller.geocode_modal', @@ -152,7 +153,8 @@ var SEED_app = angular.module('BE.seed', [ 'BE.seed.directives', 'BE.seed.services', 'BE.seed.controllers', - 'BE.seed.utilities' + 'BE.seed.utilities', + 'BE.seed.constants', ], ['$interpolateProvider', '$qProvider', function ($interpolateProvider, $qProvider) { $interpolateProvider.startSymbol('{$'); $interpolateProvider.endSymbol('$}'); @@ -529,11 +531,34 @@ SEED_app.config(['stateHelperProvider', '$urlRouterProvider', '$locationProvider templateUrl: static_url + 'seed/partials/mapping.html', controller: 'mapping_controller', resolve: { - column_mapping_presets_payload: ['column_mappings_service', 'user_service', function (column_mappings_service, user_service) { + column_mapping_presets_payload: [ + 'column_mappings_service', + 'user_service', + 'COLUMN_MAPPING_PRESET_TYPE_NORMAL', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', + 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM', + 'import_file_payload', + function ( + column_mappings_service, + user_service, + COLUMN_MAPPING_PRESET_TYPE_NORMAL, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + import_file_payload) { + let filter_preset_types + if (import_file_payload.import_file.source_type === "BuildingSync Raw") { + filter_preset_types = [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + ] + } else { + filter_preset_types = [COLUMN_MAPPING_PRESET_TYPE_NORMAL] + } var organization_id = user_service.get_organization().id; - return column_mappings_service.get_column_mapping_presets_for_org(organization_id).then(function (response) { - return response.data; - }); + return column_mappings_service.get_column_mapping_presets_for_org( + organization_id, + filter_preset_types + ).then(response => response.data) }], import_file_payload: ['dataset_service', '$stateParams', function (dataset_service, $stateParams) { var importfile_id = $stateParams.importfile_id; diff --git a/seed/static/seed/js/services/column_mappings_service.js b/seed/static/seed/js/services/column_mappings_service.js index f67bbd1890..9e75dad426 100644 --- a/seed/static/seed/js/services/column_mappings_service.js +++ b/seed/static/seed/js/services/column_mappings_service.js @@ -28,11 +28,15 @@ angular.module('BE.seed.service.column_mappings', []).factory('column_mappings_s }); }; - column_mappings_factory.get_column_mapping_presets_for_org = function (org_id) { + column_mappings_factory.get_column_mapping_presets_for_org = function (org_id, filter_preset_types) { + const params = { + organization_id: org_id, + } + if (filter_preset_types != null) { + params.preset_type = filter_preset_types + } return $http.get('/api/v2/column_mapping_presets/', { - params: { - organization_id: org_id - } + params }).then(function (response) { return response.data; }); diff --git a/seed/static/seed/partials/column_mappings.html b/seed/static/seed/partials/column_mappings.html index 5cb6d4eee9..a9c8dc90b6 100644 --- a/seed/static/seed/partials/column_mappings.html +++ b/seed/static/seed/partials/column_mappings.html @@ -31,13 +31,13 @@

Column Mappings

{$:: 'Column Mapping Preset' | translate $}: - - - -
+
@@ -95,8 +95,10 @@

Column Mappings

+ {$:: 'Set all fields to' | translate $}: +

Mapped Fields

@@ -110,7 +112,7 @@

Column Mappings

Measurement Units Data File Header - + @@ -119,7 +121,7 @@

Column Mappings

@@ -130,14 +132,15 @@

Column Mappings

class="form-control input-sm tcm_field" ng-change="flag_change(col)" typeahead-on-select="seed_header_change(col)" - ng-attr-id="mapped-row-input-box-{$ $index $}"> + ng-attr-id="mapped-row-input-box-{$ $index $}" + ng-disabled="!preset_action_ok('change_to_field')"> - - @@ -146,13 +149,13 @@

Column Mappings

- + {$:: cell_value $} - @@ -160,7 +163,7 @@

Column Mappings

- diff --git a/seed/static/seed/partials/export_buildingsync_modal.html b/seed/static/seed/partials/export_buildingsync_modal.html new file mode 100644 index 0000000000..2f57777b97 --- /dev/null +++ b/seed/static/seed/partials/export_buildingsync_modal.html @@ -0,0 +1,27 @@ +
+

Select Preset for Export

+
+ +
+
Please select the preset used when importing the data, else the exported data may be incorrect.
+
+ Unexpected error downloading file + 
{$ download_error_message | json $}
+
+
+ +
+ +
+
+
+ +
+ + +
diff --git a/seed/static/seed/partials/mapping.html b/seed/static/seed/partials/mapping.html index 6a42dd5fa9..37ef1a8c59 100644 --- a/seed/static/seed/partials/mapping.html +++ b/seed/static/seed/partials/mapping.html @@ -83,23 +83,23 @@
- +
- +
-
+
Current Column Mapping Preset: -
Please review SEED column names or the column names of the file being imported. Duplicate values are not allowed in either case.
Please review SEED headers. Empty values are not allowed.
+
Please apply a column mapping preset or save your current mapping as a preset.
@@ -126,8 +127,8 @@ @@ -150,13 +151,13 @@
- {$:: 'Set all fields to' | translate $}: - + {$:: 'Set all fields to' | translate $}: +

Mapped Fields
- - + - + ' + '' + - '', + '' }, { field: 'data_type', - displayName: 'Data Type Change', + displayName: 'Data Type Change' }, { field: 'merge_protection', displayName: 'Merge Protection Change', cellTemplate: '
' + '' + - '
', + '' }, { field: 'recognize_empty', displayName: 'Recognize Empty', cellTemplate: '
' + '' + - '
', + '' }, { field: 'is_matching_criteria', displayName: 'Matching Criteria Change', cellTemplate: '
' + '' + - '
', - }, - ] + '' + } + ]; var unique_summary_columns = _.uniq(all_changed_settings); $scope.change_summary_column_defs = _.filter(base_summary_column_defs, function (column_def) { @@ -115,25 +115,25 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) columnDefs: $scope.change_summary_column_defs, enableColumnResizing: true, rowHeight: 40, - minRowsToShow: Math.min($scope.change_summary_data.length, 5), + minRowsToShow: Math.min($scope.change_summary_data.length, 5) }; // By default, assume matching criteria isn't being updated to exclude PM Property ID // And since warning wouldn't be shown in that case, set "acknowledged" to true. $scope.checks = { matching_criteria_excludes_pm_property_id: false, - warnings_acknowledged: true, - } + warnings_acknowledged: true + }; // Check if PM Property ID is actually being removed from matching criteria - if (_.find($scope.change_summary_data, {column_name: "pm_property_id", is_matching_criteria: false})) { + if (_.find($scope.change_summary_data, {column_name: 'pm_property_id', is_matching_criteria: false})) { $scope.checks.matching_criteria_excludes_pm_property_id = true; $scope.checks.warnings_acknowledged = false; } // Preview // Agg function returning last value of matching criteria field (all should be the same if they match) - $scope.matching_field_value = function(aggregation, fieldValue) { + $scope.matching_field_value = function (aggregation, fieldValue) { aggregation.value = fieldValue; }; @@ -143,7 +143,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) // Lastly, non-matching columns are given next priority so that users can sort within a grouped set. if (sortColumns.length > 1) { var matching_cols = _.filter(sortColumns, function (col) { - return col.colDef.is_matching_criteria; + return col.colDef.is_matching_criteria; }); var linking_id_col = _.find(sortColumns, ['name', 'id']); var remaining_cols = _.filter(sortColumns, function (col) { @@ -152,20 +152,20 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) sortColumns = matching_cols.concat(linking_id_col).concat(remaining_cols); _.forEach(sortColumns, function (col, index) { col.sort.priority = index; - }) + }); } }; // Takes raw cycle-partitioned records and returns array of cycle-aware records - var format_preview_records = function(raw_inventory) { - return _.reduce(raw_inventory, function(all_records, records, cycle_id) { + var format_preview_records = function (raw_inventory) { + return _.reduce(raw_inventory, function (all_records, records, cycle_id) { var cycle = _.find($scope.cycles, { id: parseInt(cycle_id) }); - _.forEach(records, function(record) { + _.forEach(records, function (record) { record.cycle_name = cycle.name; record.cycle_start = cycle.start; - all_records.push(record) - }) - return all_records + all_records.push(record); + }); + return all_records; }, []); }; @@ -179,7 +179,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) headerCellFilter: 'translate', minWidth: default_min_width, width: 125, - groupingShowAggregationMenu: false, + groupingShowAggregationMenu: false }; _.map(preview_column_defs, function (col) { @@ -197,7 +197,7 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) // Help indicate matching columns are given preferred sort priority col.displayName = col.displayName + '*'; - options.headerCellClass = "matching-column-header"; + options.headerCellClass = 'matching-column-header'; options.customTreeAggregationFn = $scope.matching_field_value; options.width = autopin_width; @@ -216,23 +216,23 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) visible: false, suppressRemoveSort: true, // since grouping relies on sorting minWidth: default_min_width, - width: autopin_width, + width: autopin_width }, { - name: "cycle_name", - displayName: "Cycle", + name: 'cycle_name', + displayName: 'Cycle', pinnedLeft: true, treeAggregationType: uiGridGroupingConstants.aggregation.COUNT, customTreeAggregationFinalizerFn: function (aggregation) { - aggregation.rendered = "total cycles: " + aggregation.value; + aggregation.rendered = 'total cycles: ' + aggregation.value; }, minWidth: default_min_width, width: autopin_width, - groupingShowAggregationMenu: false, + groupingShowAggregationMenu: false }, { - name: "cycle_start", - displayName: "Cycle Start", + name: 'cycle_start', + displayName: 'Cycle Start', cellFilter: 'date:\'yyyy-MM-dd\'', filter: inventory_service.dateFilter(), type: 'date', @@ -240,11 +240,11 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) pinnedLeft: true, minWidth: default_min_width, width: autopin_width, - groupingShowAggregationMenu: false, - }, - ) + groupingShowAggregationMenu: false + } + ); - return preview_column_defs; + return preview_column_defs; }; // Initialize preview table as empty for now. @@ -260,31 +260,31 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) $scope.gridApi.core.on.filterChanged($scope, function () { // This is a workaround for losing the state of expanded rows during filtering. - _.delay($scope.gridApi.treeBase.expandAllRows, 500); - }) + _.delay($scope.gridApi.treeBase.expandAllRows, 500); + }); // Prioritized to maintain grouping. $scope.gridApi.core.on.sortChanged($scope, prioritize_sort); - }, + } }; // Preview Loading Helpers var build_proposed_matching_columns = function (result) { // Summarize proposed matching_criteria_columns for pinning and to create preview var criteria_additions = _.filter($scope.change_summary_data, function (change) { - return change.is_matching_criteria + return change.is_matching_criteria; }); var criteria_removals = _.filter($scope.change_summary_data, function (change) { - return change.is_matching_criteria === false; + return change.is_matching_criteria === false; }); $scope.criteria_changes = { add: _.map(criteria_additions, 'column_name'), - remove: _.map(criteria_removals, 'column_name'), - } + remove: _.map(criteria_removals, 'column_name') + }; var base_and_add; - if ($scope.inventory_type == "properties") { + if ($scope.inventory_type == 'properties') { base_and_add = _.union(result.PropertyState, $scope.criteria_changes.add); } else { base_and_add = _.union(result.TaxLotState, $scope.criteria_changes.add); @@ -307,24 +307,24 @@ angular.module('BE.seed.controller.confirm_column_settings_modal', []) // Use new proposed matching_criteria_columns to request a preview then render this preview. var spinner_options = { scale: 0.40, - position: "relative", - left: "100%", - } + position: 'relative', + left: '100%' + }; spinner_utility.show(spinner_options, $('#spinner_placeholder')[0]); organization_service.match_merge_link_preview($scope.org_id, $scope.inventory_type, $scope.criteria_changes) .then(function (response) { organization_service.check_match_merge_link_status(response.progress_key) - .then(function (completion_notice) { - organization_service.get_match_merge_link_result($scope.org_id, completion_notice.unique_id) - .then(build_preview) - .then(preview_loading_complete); - }); + .then(function (completion_notice) { + organization_service.get_match_merge_link_result($scope.org_id, completion_notice.unique_id) + .then(build_preview) + .then(preview_loading_complete); + }); }); }; // Get and Show Preview (If matching criteria changes exist.) - $scope.matching_criteria_exists = _.find(_.values($scope.change_summary_data), function(delta) { + $scope.matching_criteria_exists = _.find(_.values($scope.change_summary_data), function (delta) { return _.has(delta, 'is_matching_criteria'); }); diff --git a/seed/static/seed/js/controllers/data_quality_admin_controller.js b/seed/static/seed/js/controllers/data_quality_admin_controller.js index 99df6660fc..5dea303ebe 100644 --- a/seed/static/seed/js/controllers/data_quality_admin_controller.js +++ b/seed/static/seed/js/controllers/data_quality_admin_controller.js @@ -141,16 +141,16 @@ angular.module('BE.seed.controller.data_quality_admin', []) $scope.ruleGroups = ruleGroups; $scope.rule_count_property = 0; $scope.rule_count_taxlot = 0; - _.map($scope.ruleGroups['properties'], function (rule) { + _.map($scope.ruleGroups.properties, function (rule) { $scope.rule_count_property += rule.length; }); - _.map($scope.ruleGroups['taxlots'], function (rule) { + _.map($scope.ruleGroups.taxlots, function (rule) { $scope.rule_count_taxlot += rule.length; }); }; loadRules(data_quality_rules_payload); - $scope.isModified = function() { + $scope.isModified = function () { return modified_service.isModified(); }; var originalRules = angular.copy(data_quality_rules_payload.rules); @@ -424,8 +424,7 @@ angular.module('BE.seed.controller.data_quality_admin', []) $scope.delete_rule = function (rule, index) { if ($scope.ruleGroups[$scope.inventory_type][rule.field].length === 1) { delete $scope.ruleGroups[$scope.inventory_type][rule.field]; - } - else $scope.ruleGroups[$scope.inventory_type][rule.field].splice(index, 1); + } else $scope.ruleGroups[$scope.inventory_type][rule.field].splice(index, 1); $scope.change_rules(); if ($scope.inventory_type === 'properties') $scope.rule_count_property -= 1; else $scope.rule_count_taxlot -= 1; diff --git a/seed/static/seed/js/controllers/data_upload_modal_controller.js b/seed/static/seed/js/controllers/data_upload_modal_controller.js index aa8bd5440d..1a8c7221f4 100644 --- a/seed/static/seed/js/controllers/data_upload_modal_controller.js +++ b/seed/static/seed/js/controllers/data_upload_modal_controller.js @@ -578,7 +578,7 @@ angular.module('BE.seed.controller.data_upload_modal', []) // this only occurs in buildingsync, where we are not actually merging properties // thus we will always end up at step 10 $scope.step_10_style = 'danger'; - $scope.step_10_file_message = 'Warning(s)/Error(s) occurred while processing the file(s):\n' + JSON.stringify(progress_data.file_info, null, 2) + $scope.step_10_file_message = 'Warning(s)/Error(s) occurred while processing the file(s):\n' + JSON.stringify(progress_data.file_info, null, 2); } // If merges against existing exist, provide slightly different feedback diff --git a/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js b/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js index b691f634f8..fb02d8d1f4 100644 --- a/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js +++ b/seed/static/seed/js/controllers/export_buildingsync_modal_controller.js @@ -16,34 +16,34 @@ angular.module('BE.seed.controller.export_buildingsync_modal', []) $scope, $uibModalInstance, property_view_id, - column_mapping_presets, + column_mapping_presets ) { - $scope.column_mapping_presets = column_mapping_presets - $scope.current_column_mapping_preset = column_mapping_presets[0] + $scope.column_mapping_presets = column_mapping_presets; + $scope.current_column_mapping_preset = column_mapping_presets[0]; - $scope.download_file = () => { - let the_url = '/api/v2_1/properties/' + property_view_id + '/building_sync/'; + $scope.download_file = function () { + var the_url = '/api/v2_1/properties/' + property_view_id + '/building_sync/'; $http.get( the_url, { params: { preset_id: $scope.current_column_mapping_preset.id } } - ).then(response => { - let blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'}); - let downloadLink = angular.element(''); - let filename = 'buildingsync_property_' + property_view_id + '.xml'; + ).then(function (response) { + var blob = new Blob([response.data], {type: 'application/xml;charset=utf-8;'}); + var downloadLink = angular.element(''); + var filename = 'buildingsync_property_' + property_view_id + '.xml'; downloadLink.attr('href', $window.URL.createObjectURL(blob)); downloadLink.attr('download', filename); downloadLink[0].click(); - $uibModalInstance.close() - }, err => { - $scope.download_error_message = err.data ? err.data : err.toString() - }) - } + $uibModalInstance.close(); + }, function (err) { + $scope.download_error_message = err.data ? err.data : err.toString(); + }); + }; - $scope.close = () => { + $scope.close = function () { $uibModalInstance.close(); }; - $scope.cancel = () => { + $scope.cancel = function () { $uibModalInstance.dismiss(); }; }]); diff --git a/seed/static/seed/js/controllers/inventory_cycles_controller.js b/seed/static/seed/js/controllers/inventory_cycles_controller.js index 8642ce7b91..706ca429f9 100644 --- a/seed/static/seed/js/controllers/inventory_cycles_controller.js +++ b/seed/static/seed/js/controllers/inventory_cycles_controller.js @@ -44,7 +44,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.currentProfile = current_profile; // Scope columns/data to only those of the given inventory_type - var state_type = $scope.inventory_type == "properties" ? "PropertyState" : "TaxLotState"; + var state_type = $scope.inventory_type == 'properties' ? 'PropertyState' : 'TaxLotState'; $scope.all_columns = _.filter(all_columns, {table_name: state_type}); // set up i18n @@ -61,13 +61,13 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.included_cycle_ids = _.map(_.keys(inventory), function (cycle_id) { return parseInt(cycle_id); }); - $scope.cycle_options = _.map(cycles.cycles, function(cycle) { + $scope.cycle_options = _.map(cycles.cycles, function (cycle) { var selected = $scope.included_cycle_ids.includes(cycle.id); return { selected: selected, label: cycle.name, value: cycle.id, - start: cycle.start, + start: cycle.start }; }); @@ -84,15 +84,15 @@ angular.module('BE.seed.controller.inventory_cycles', []) }; // Takes raw cycle-partitioned records and returns array of cycle-aware records - $scope.format_records = function(raw_inventory) { - return _.reduce(raw_inventory, function(all_records, records, cycle_id) { + $scope.format_records = function (raw_inventory) { + return _.reduce(raw_inventory, function (all_records, records, cycle_id) { var cycle = _.find($scope.cycle_options, { value: parseInt(cycle_id) }); - _.forEach(records, function(record) { + _.forEach(records, function (record) { record.cycle_name = cycle.label; record.cycle_start = cycle.start; - all_records.push(record) - }) - return all_records + all_records.push(record); + }); + return all_records; }, []); }; @@ -100,16 +100,16 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.data = $scope.format_records(inventory); // Refreshes inventory by making API call - $scope.refresh_objects = function() { + $scope.refresh_objects = function () { spinner_utility.show(); var profile_id = _.has($scope.currentProfile, 'id') ? $scope.currentProfile.id : undefined; - if ($scope.inventory_type == "properties") { - inventory_service.properties_cycle(profile_id, $scope.included_cycle_ids).then(function(refreshed_inventory) { + if ($scope.inventory_type == 'properties') { + inventory_service.properties_cycle(profile_id, $scope.included_cycle_ids).then(function (refreshed_inventory) { $scope.data = $scope.format_records(refreshed_inventory); spinner_utility.hide(); }); } else { - inventory_service.taxlots_cycle(profile_id, $scope.included_cycle_ids).then(function(refreshed_inventory) { + inventory_service.taxlots_cycle(profile_id, $scope.included_cycle_ids).then(function (refreshed_inventory) { $scope.data = $scope.format_records(refreshed_inventory); spinner_utility.hide(); }); @@ -117,34 +117,34 @@ angular.module('BE.seed.controller.inventory_cycles', []) }; // On profile change, refreshes objects and rebuild columns - $scope.profile_change = function() { + $scope.profile_change = function () { inventory_service.save_last_profile($scope.currentProfile.id, $scope.inventory_type); $scope.refresh_objects(); // uiGrid doesn't recognize complete columnDefs swap unless it's removed and refreshed and notified for each $scope.gridOptions.columnDefs = []; - $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN) + $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN); $scope.build_columns(); $scope.gridOptions.columnDefs = $scope.columns; - $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN) + $scope.gridApi.core.notifyDataChange(uiGridConstants.dataChange.COLUMN); }; // Agg function returning last value of matching criteria field (all should be the same if they match) - $scope.matching_field_value = function(aggregation, fieldValue) { + $scope.matching_field_value = function (aggregation, fieldValue) { aggregation.value = fieldValue; }; // matching_criteria_columns identified here to pin left on table - if ($scope.inventory_type == "properties") { + if ($scope.inventory_type == 'properties') { $scope.matching_criteria_columns = matching_criteria_columns.PropertyState; } else { $scope.matching_criteria_columns = matching_criteria_columns.TaxLotState; } // Builds columns with profile, default, and grouping settings - $scope.build_columns = function() { + $scope.build_columns = function () { $scope.columns = []; // Profile Settings @@ -167,7 +167,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) var column_def_defaults = { headerCellFilter: 'translate', minWidth: default_min_width, - width: 150, + width: 150 }; _.map($scope.columns, function (col) { @@ -185,7 +185,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) // Help indicate matching columns are given preferred sort priority col.displayName = col.displayName + '*'; - options.headerCellClass = "matching-column-header"; + options.headerCellClass = 'matching-column-header'; options.customTreeAggregationFn = $scope.matching_field_value; options.width = autopin_width; @@ -204,7 +204,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) visible: false, suppressRemoveSort: true, // since grouping relies on sorting minWidth: default_min_width, - width: autopin_width, + width: autopin_width }, { name: 'inventory detail link icon', @@ -222,31 +222,31 @@ angular.module('BE.seed.controller.inventory_cycles', []) enableSorting: false, pinnedLeft: true, visible: true, - width: 30, + width: 30 }, { - name: "cycle_name", - displayName: "Cycle", + name: 'cycle_name', + displayName: 'Cycle', pinnedLeft: true, treeAggregationType: uiGridGroupingConstants.aggregation.COUNT, customTreeAggregationFinalizerFn: function (aggregation) { - aggregation.rendered = "total cycles: " + aggregation.value; + aggregation.rendered = 'total cycles: ' + aggregation.value; }, minWidth: default_min_width, - width: autopin_width, + width: autopin_width }, { - name: "cycle_start", - displayName: "Cycle Start", + name: 'cycle_start', + displayName: 'Cycle Start', cellFilter: 'date:\'yyyy-MM-dd\'', filter: inventory_service.dateFilter(), type: 'date', sort: { priority: 1, direction: 'asc' }, pinnedLeft: true, minWidth: default_min_width, - width: autopin_width, - }, - ) + width: autopin_width + } + ); }; $scope.build_columns(); @@ -257,7 +257,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) // Lastly, non-matching columns are given next priority so that users can sort within a grouped set. if (sortColumns.length > 1) { var matching_cols = _.filter(sortColumns, function (col) { - return col.colDef.is_matching_criteria; + return col.colDef.is_matching_criteria; }); var linking_id_col = _.find(sortColumns, ['name', 'id']); var remaining_cols = _.filter(sortColumns, function (col) { @@ -266,7 +266,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) sortColumns = matching_cols.concat(linking_id_col).concat(remaining_cols); _.forEach(sortColumns, function (col, index) { col.sort.priority = index; - }) + }); } }; @@ -283,8 +283,8 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.gridApi.core.on.filterChanged($scope, function () { // This is a workaround for losing the state of expanded rows during filtering. - _.delay($scope.gridApi.treeBase.expandAllRows, 500); - }) + _.delay($scope.gridApi.treeBase.expandAllRows, 500); + }); // Prioritized to maintain grouping. $scope.gridApi.core.on.sortChanged($scope, prioritize_sort); @@ -296,7 +296,7 @@ angular.module('BE.seed.controller.inventory_cycles', []) $scope.$on('$destroy', function () { angular.element($window).off('resize', debouncedHeightUpdate); }); - }, + } }; $scope.updateHeight = function () { diff --git a/seed/static/seed/js/controllers/inventory_detail_controller.js b/seed/static/seed/js/controllers/inventory_detail_controller.js index 99ac3fb30d..da3e906bb6 100644 --- a/seed/static/seed/js/controllers/inventory_detail_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_controller.js @@ -115,7 +115,7 @@ angular.module('BE.seed.controller.inventory_detail', []) } }; - function populated_columns_modal () { + function populated_columns_modal() { $uibModal.open({ backdrop: 'static', templateUrl: urls.static_url + 'seed/partials/show_populated_columns_modal.html', @@ -127,9 +127,7 @@ angular.module('BE.seed.controller.inventory_detail', []) currentProfile: function () { return $scope.currentProfile; }, - cycle: function () { - return null; - }, + cycle: _.constant(null), inventory_type: function () { return $stateParams.inventory_type; }, @@ -144,7 +142,7 @@ angular.module('BE.seed.controller.inventory_detail', []) }); // add "master" copy - item_copy = angular.copy($scope.item_state); + var item_copy = angular.copy($scope.item_state); _.defaults(item_copy, $scope.item_state.extra_data); return provided_inventory; @@ -500,11 +498,13 @@ angular.module('BE.seed.controller.inventory_detail', []) }; $scope.export_building_sync = function () { - const modalInstance = $uibModal.open({ + var modalInstance = $uibModal.open({ templateUrl: urls.static_url + 'seed/partials/export_buildingsync_modal.html', controller: 'export_buildingsync_modal_controller', resolve: { - property_view_id: function() { return $stateParams.view_id }, + property_view_id: function () { + return $stateParams.view_id; + }, column_mapping_presets: [ 'column_mappings_service', 'COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT', @@ -512,20 +512,23 @@ angular.module('BE.seed.controller.inventory_detail', []) function ( column_mappings_service, COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM ) { - const filter_preset_types = [ - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, - ] - return column_mappings_service.get_column_mapping_presets_for_org( - $scope.organization.id, - filter_preset_types, - ).then(response => response.data); - }] + var filter_preset_types = [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ]; + return column_mappings_service.get_column_mapping_presets_for_org( + $scope.organization.id, + filter_preset_types + ).then(function (response) { + return response.data; + }); + }] } - }) - modalInstance.result.then(() => { return; }) + }); + modalInstance.result.then(function () { + }); }; $scope.export_building_sync_xlsx = function () { diff --git a/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js b/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js index fbe850b55c..e45b479abc 100644 --- a/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_cycles_controller.js @@ -30,19 +30,19 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) ) { $scope.inventory_type = $stateParams.inventory_type; $scope.inventory = { - view_id: $stateParams.view_id, + view_id: $stateParams.view_id }; $scope.states = inventory_payload.data; $scope.base_state = _.find(inventory_payload.data, {view_id: $stateParams.view_id}); - $scope.cycles = _.reduce(cycles.cycles, function(cycles_by_id, cycle) { + $scope.cycles = _.reduce(cycles.cycles, function (cycles_by_id, cycle) { cycles_by_id[cycle.id] = cycle; return cycles_by_id; }, {}); // Flag columns whose values have changed between cycles. - var changes_check = function(column) { + var changes_check = function (column) { var uniq_column_values; if (column.is_extra_data) { @@ -53,7 +53,7 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) uniq_column_values = _.uniqBy($scope.states, column.column_name); } - column['changed'] = uniq_column_values.length > 1; + column.changed = uniq_column_values.length > 1; return column; }; @@ -69,7 +69,7 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) }); } else { // No profiles exist - $scope.columns = _.map(_.reject(columns, 'is_extra_data'), function(col) { + $scope.columns = _.map(_.reject(columns, 'is_extra_data'), function (col) { return changes_check(col); }); } @@ -89,7 +89,7 @@ angular.module('BE.seed.controller.inventory_detail_cycles', []) // Horizontal scroll for "2 tables" that scroll together for fixed header effect. var table_container = $('.table-xscroll-fixed-header-container'); - table_container.scroll(function() { + table_container.scroll(function () { $('.table-xscroll-fixed-header-container > .table-body-x-scroll').width( table_container.width() + table_container.scrollLeft() ); diff --git a/seed/static/seed/js/controllers/inventory_detail_meters_controller.js b/seed/static/seed/js/controllers/inventory_detail_meters_controller.js index ae086ffc7d..00ec61db36 100644 --- a/seed/static/seed/js/controllers/inventory_detail_meters_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_meters_controller.js @@ -32,30 +32,32 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) spinner_utility, urls, user_service, - $log, + $log ) { spinner_utility.show(); $scope.item_state = inventory_payload.state; $scope.inventory_type = $stateParams.inventory_type; $scope.organization = user_service.get_organization(); $scope.filler_cycle = cycles.cycles[0].id; - $scope.scenarios = _.uniqBy(_.map(meters, function(meter) { + $scope.scenarios = _.uniqBy(_.map(meters, function (meter) { return { id: meter.scenario_id, name: meter.scenario_name - } - }), 'id').filter(scenario => scenario.id !== undefined && scenario.id !== null) + }; + }), 'id').filter(function (scenario) { + return !_.isNil(scenario.id); + }); $scope.inventory = { view_id: $stateParams.view_id }; - const getMeterLabel = (meter) => { - return meter.type + ' - ' + meter.source + ' - ' + meter.source_id - } + var getMeterLabel = function (meter) { + return meter.type + ' - ' + meter.source + ' - ' + meter.source_id; + }; - const resetSelections = () => { - $scope.meter_selections = _.map(sorted_meters, function(meter) { + var resetSelections = function () { + $scope.meter_selections = _.map(sorted_meters, function (meter) { return { selected: true, label: getMeterLabel(meter), @@ -63,14 +65,14 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) }; }); - $scope.scenario_selections = _.map($scope.scenarios, function(scenario) { + $scope.scenario_selections = _.map($scope.scenarios, function (scenario) { return { selected: true, label: scenario.name, value: scenario.id - } + }; }); - } + }; // On page load, all meters and readings $scope.data = property_meter_usage.readings; @@ -85,9 +87,9 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) } }; - $scope.scenario_selection_toggled = (is_open) => { + $scope.scenario_selection_toggled = function (is_open) { if (!is_open) { - $scope.applyFilters() + $scope.applyFilters(); } }; @@ -150,70 +152,85 @@ angular.module('BE.seed.controller.inventory_detail_meters', []) }; // remove option to filter by scenario if there are no scenarios if ($scope.scenarios.length === 0) { - $scope.filterMethod.options = ['meter'] + $scope.filterMethod.options = ['meter']; } // given a list of meter labels, it returns the filtered readings and column defs // This is used by the primary filterBy... functions - const filterByMeterLabels = (readings, columnDefs, meterLabels) => { - const timeColumns = ['start_time', 'end_time', 'month', 'year'] - const selectedColumns = meterLabels.concat(timeColumns) + var filterByMeterLabels = function filterByMeterLabels (readings, columnDefs, meterLabels) { + var timeColumns = ['start_time', 'end_time', 'month', 'year']; + var selectedColumns = meterLabels.concat(timeColumns); + var filteredReadings = readings.map(function (reading) { + return Object.entries(reading).reduce(function (newReading, _ref) { + var key = _ref[0], + value = _ref[1]; - const filteredReadings = readings.map(reading => Object.entries(reading).reduce((newReading, [key, value]) => { - if (selectedColumns.includes(key)) { - newReading[key] = value; - } - return newReading; - }, {})); + if (selectedColumns.includes(key)) { + newReading[key] = value; + } - const filteredColumnDefs = columnDefs.filter(columnDef => selectedColumns.includes(columnDef.field)) - return { readings: filteredReadings, columnDefs: filteredColumnDefs} - } + return newReading; + }, {}); + }); + var filteredColumnDefs = columnDefs.filter(function (columnDef) { + return selectedColumns.includes(columnDef.field); + }); + return { + readings: filteredReadings, + columnDefs: filteredColumnDefs + }; + }; // given the meter selections, it returns the filtered readings and column defs - const filterByMeterSelections = (readings, columnDefs, meterSelections) => { + var filterByMeterSelections = function (readings, columnDefs, meterSelections) { // filter according to meter selections - const selectedMeterLabels = meterSelections.filter(selection => selection.selected) - .map(selection => selection.label); + var selectedMeterLabels = meterSelections.filter(function (selection) { + return selection.selected; + }) + .map(function (selection) { + return selection.label; + }); - return filterByMeterLabels(readings, columnDefs, selectedMeterLabels) - } + return filterByMeterLabels(readings, columnDefs, selectedMeterLabels); + }; // given the scenario selections, it returns the filtered readings and column defs - const filterByScenarioSelections = (readings, columnDefs, meters, scenarioSelections) => { - const selectedScenarioIds = scenarioSelections.filter(selection => selection.selected).map(selection => selection.value); - const selectedMeterLabels = meters.filter(meter => selectedScenarioIds.includes(meter.scenario_id)) - .map(meter => getMeterLabel(meter)) + var filterByScenarioSelections = function (readings, columnDefs, meters, scenarioSelections) { + var selectedScenarioIds = scenarioSelections.filter(function (selection) { + return selection.selected; + }).map(function (selection) { + return selection.value; + }); + var selectedMeterLabels = meters.filter(function (meter) { + return selectedScenarioIds.includes(meter.scenario_id); + }).map(function (meter) { + return getMeterLabel(meter); + }); - return filterByMeterLabels(readings, columnDefs, selectedMeterLabels) - } + return filterByMeterLabels(readings, columnDefs, selectedMeterLabels); + }; // filters the meter readings by selected meters or scenarios and updates the table - $scope.applyFilters = () => { - let readings, columnDefs; + $scope.applyFilters = function () { + var results, readings, columnDefs; if ($scope.filterMethod.selected === 'meter') { - ({readings, columnDefs} = filterByMeterSelections( - property_meter_usage.readings, - property_meter_usage.column_defs, - $scope.meter_selections - )) + results = filterByMeterSelections(property_meter_usage.readings, property_meter_usage.column_defs, $scope.meter_selections); + readings = results.readings; + columnDefs = results.columnDefs; } else if ($scope.filterMethod.selected === 'scenario') { - ({readings, columnDefs} = filterByScenarioSelections( - property_meter_usage.readings, - property_meter_usage.column_defs, - sorted_meters, - $scope.scenario_selections - )) + results = filterByScenarioSelections(property_meter_usage.readings, property_meter_usage.column_defs, sorted_meters, $scope.scenario_selections); + readings = results.readings; + columnDefs = results.columnDefs; } else { - $log.error("Invalid filter method selected: ", $scope.filterMethod) - return + $log.error('Invalid filter method selected: ', $scope.filterMethod); + return; } $scope.data = readings; $scope.gridOptions.columnDefs = columnDefs; $scope.has_readings = $scope.data.length > 0; $scope.apply_column_settings(); - } + }; // refresh_readings make an API call to refresh the base readings data // according to the selected interval diff --git a/seed/static/seed/js/controllers/inventory_detail_notes_controller.js b/seed/static/seed/js/controllers/inventory_detail_notes_controller.js index ae3e9d204d..5211861534 100644 --- a/seed/static/seed/js/controllers/inventory_detail_notes_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_notes_controller.js @@ -70,7 +70,7 @@ angular.module('BE.seed.controller.inventory_detail_notes', []) return $scope.org_id; }, note: _.constant({text: ''}), - action: _.constant('new'), + action: _.constant('new') } }); @@ -97,7 +97,7 @@ angular.module('BE.seed.controller.inventory_detail_notes', []) note: function () { return note; }, - action: _.constant('update'), + action: _.constant('update') } }); @@ -124,7 +124,7 @@ angular.module('BE.seed.controller.inventory_detail_notes', []) note: function () { return note; }, - action: _.constant('delete'), + action: _.constant('delete') } }); diff --git a/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js b/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js index 4a1a549ee4..1fa720e4b8 100644 --- a/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_notes_modal_controller.js @@ -36,7 +36,7 @@ angular.module('BE.seed.controller.inventory_detail_notes_modal', []) var data = { name: 'Manually Created', note_type: 'Note', - text: note.text, + text: note.text }; note_service.create_note($scope.orgId, $scope.inventoryType, $scope.viewId, data).then(function () { $uibModalInstance.close(); @@ -47,7 +47,7 @@ angular.module('BE.seed.controller.inventory_detail_notes_modal', []) var data = { name: note.name, note_type: note.note_type, - text: note.text, + text: note.text }; note_service.update_note($scope.orgId, $scope.inventoryType, $scope.viewId, note.id, data).then(function () { $uibModalInstance.close(); diff --git a/seed/static/seed/js/controllers/inventory_list_controller.js b/seed/static/seed/js/controllers/inventory_list_controller.js index 3271a5c485..aefe8d6218 100644 --- a/seed/static/seed/js/controllers/inventory_list_controller.js +++ b/seed/static/seed/js/controllers/inventory_list_controller.js @@ -59,7 +59,7 @@ angular.module('BE.seed.controller.inventory_list', []) $scope.data = []; var lastCycleId = inventory_service.get_last_cycle(); $scope.cycle = { - selected_cycle: _.find(cycles.cycles, {id: lastCycleId}) || _.first(cycles.cycles), + selected_cycle: _.find(cycles.cycles, {id: lastCycleId}) || _.first(cycles.cycles), cycles: cycles.cycles }; @@ -155,7 +155,7 @@ angular.module('BE.seed.controller.inventory_list', []) } }; - function populated_columns_modal() { + function populated_columns_modal () { $uibModal.open({ backdrop: 'static', templateUrl: urls.static_url + 'seed/partials/show_populated_columns_modal.html', @@ -173,9 +173,7 @@ angular.module('BE.seed.controller.inventory_list', []) inventory_type: function () { return $stateParams.inventory_type; }, - provided_inventory: function () { - return null; - } + provided_inventory: _.constant(null) } }); } @@ -192,7 +190,7 @@ angular.module('BE.seed.controller.inventory_list', []) }); }; - function updateApplicableLabels(current_labels) { + function updateApplicableLabels (current_labels) { var inventoryIds; if ($scope.inventory_type === 'properties') { inventoryIds = _.map($scope.data, 'property_view_id').sort(); @@ -829,7 +827,7 @@ angular.module('BE.seed.controller.inventory_list', []) }); }; - function currentColumns() { + function currentColumns () { // Save all columns except first 3 var gridCols = _.filter($scope.gridApi.grid.columns, function (col) { return !_.includes(['treeBaseRowHeaderCol', 'selectionRowHeaderCol', 'notes_count', 'merged_indicator', 'id'], col.name) && col.visible; diff --git a/seed/static/seed/js/controllers/inventory_map_controller.js b/seed/static/seed/js/controllers/inventory_map_controller.js index 724e5f8ff5..8d13575d21 100644 --- a/seed/static/seed/js/controllers/inventory_map_controller.js +++ b/seed/static/seed/js/controllers/inventory_map_controller.js @@ -541,7 +541,7 @@ angular.module('BE.seed.controller.inventory_map', []) }); }; - function updateApplicableLabels() { + function updateApplicableLabels () { var inventoryIds; if ($scope.inventory_type === 'properties') { inventoryIds = _.map($scope.data, 'property_view_id').sort(); diff --git a/seed/static/seed/js/controllers/inventory_reports_controller.js b/seed/static/seed/js/controllers/inventory_reports_controller.js index 2ab0c5d0ac..44329d4093 100644 --- a/seed/static/seed/js/controllers/inventory_reports_controller.js +++ b/seed/static/seed/js/controllers/inventory_reports_controller.js @@ -318,7 +318,7 @@ angular.module('BE.seed.controller.inventory_reports', []) xVar: $scope.chartData.xAxisVarName, xLabel: $scope.chartData.xAxisTitle, yVar: $scope.chartData.yAxisVarName, - yLabel: $scope.chartData.yAxisTitle, + yLabel: $scope.chartData.yAxisTitle }; }, cycle_start: function () { @@ -326,7 +326,7 @@ angular.module('BE.seed.controller.inventory_reports', []) }, cycle_end: function () { return $scope.toCycle.selected_cycle.end; - }, + } } }); }; @@ -405,7 +405,7 @@ angular.module('BE.seed.controller.inventory_reports', []) $scope.toCycle.selected_cycle.end ).then(function (data) { data = data.aggregated_data; - console.log(data); + $log.log(data); $scope.aggPropertyCounts = data.property_counts; var propertyCounts = data.property_counts; var colorsArr = mapColors(propertyCounts); @@ -440,7 +440,7 @@ angular.module('BE.seed.controller.inventory_reports', []) localStorage.setItem(localStorageFromCycleKey, JSON.stringify($scope.fromCycle.selected_cycle)); localStorage.setItem(localStorageToCycleKey, JSON.stringify($scope.toCycle.selected_cycle)); - }; + } /* Generate an array of color objects to be used as part of chart configuration Each color object should have the following properties: diff --git a/seed/static/seed/js/controllers/mapping_controller.js b/seed/static/seed/js/controllers/mapping_controller.js index ebed94d107..86dea1a800 100644 --- a/seed/static/seed/js/controllers/mapping_controller.js +++ b/seed/static/seed/js/controllers/mapping_controller.js @@ -62,10 +62,10 @@ angular.module('BE.seed.controller.mapping', []) Notification, COLUMN_MAPPING_PRESET_TYPE_NORMAL, COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM ) { $scope.presets = [ - {id: 0, mappings: [], name: ""} + {id: 0, mappings: [], name: ''} ].concat(column_mapping_presets_payload); // $scope.selected_preset = $scope.applied_preset = $scope.mock_presets[0]; @@ -82,15 +82,15 @@ angular.module('BE.seed.controller.mapping', []) $scope.mappings_change_possible = true; }; - $scope.is_buildingsync_and_preset_not_ok = function() { + $scope.is_buildingsync_and_preset_not_ok = function () { if (!$scope.mappingBuildingSync) { - return false + return false; } // BuildingSync requires a saved preset to be applied return $scope.current_preset.id === 0 - || $scope.preset_change_possible - || $scope.mappings_change_possible - } + || $scope.preset_change_possible + || $scope.mappings_change_possible; + }; var analyze_chosen_inventory_types = function () { var chosenTypes = _.uniq(_.map($scope.mappings, 'suggestion_table_name')); @@ -129,54 +129,57 @@ angular.module('BE.seed.controller.mapping', []) var preset_mappings_from_working_mappings = function () { // for to_field, try DB col name, if not use col display name return _.reduce($scope.mappings, function (preset_mapping_data, mapping) { - const this_mapping = { + var this_mapping = { from_field: mapping.name, from_units: mapping.from_units, to_field: mapping.suggestion_column_name || mapping.suggestion || '', to_table_name: mapping.suggestion_table_name }; - const isBuildingSyncPreset = $scope.current_preset.preset_type !== undefined + var isBuildingSyncPreset = $scope.current_preset.preset_type !== undefined && [ - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM - ].includes($scope.current_preset.preset_type) + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ].includes($scope.current_preset.preset_type); if (isBuildingSyncPreset) { - this_mapping.from_field_value = mapping.from_field_value + this_mapping.from_field_value = mapping.from_field_value; } - preset_mapping_data.push(this_mapping) + preset_mapping_data.push(this_mapping); return preset_mapping_data; }, []); }; $scope.new_preset = function () { - let preset_mapping_data = preset_mappings_from_working_mappings(); + var preset_mapping_data = preset_mappings_from_working_mappings(); - let presetType; + var presetType; if (!$scope.mappingBuildingSync) { - presetType = COLUMN_MAPPING_PRESET_TYPE_NORMAL + presetType = COLUMN_MAPPING_PRESET_TYPE_NORMAL; } else { - presetType = COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + presetType = COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM; // make sure the new preset mapping data has the required data - const currentPresetForBuildingSync = + var currentPresetForBuildingSync = $scope.current_preset.preset_type !== undefined && [ - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM - ].includes($scope.current_preset.preset_type) + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ].includes($scope.current_preset.preset_type); if (!currentPresetForBuildingSync) { // we need to add mapping data, from_field_value, using the default mapping - const defaultPreset = $scope.presets.find(preset => preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT) - preset_mapping_data = preset_mapping_data.map(mapping => { + var defaultPreset = $scope.presets.find(function (preset) { + return preset.preset_type === COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT; + }); + preset_mapping_data = preset_mapping_data.map(function (mapping) { // find the corresponding mapping in the default preset - const defaultMapping = defaultPreset.mappings.find(defaultMapping => defaultMapping.from_field === mapping.from_field) - return { - ...mapping, + var defaultMapping = defaultPreset.mappings.find(function (defaultMapping) { + return defaultMapping.from_field === mapping.from_field; + }); + return _.merge({}, mapping, { from_field_value: defaultMapping.from_field_value - } - }) + }); + }); } } @@ -250,7 +253,7 @@ angular.module('BE.seed.controller.mapping', []) $scope.isValidCycle = Boolean(_.find(cycles.cycles, {id: $scope.import_file.cycle})); - $scope.mappingBuildingSync = $scope.import_file.source_type === 'BuildingSync Raw' + $scope.mappingBuildingSync = $scope.import_file.source_type === 'BuildingSync Raw'; matching_criteria_columns_payload.PropertyState = _.map(matching_criteria_columns_payload.PropertyState, function (column_name) { var display_name = _.find($scope.mappable_property_columns, {column_name: column_name}).display_name; @@ -279,7 +282,7 @@ angular.module('BE.seed.controller.mapping', []) $scope.setAllFieldsOptions.push({ name: 'Tax Lot', value: 'TaxLotState' - }) + }); } var eui_columns = _.filter($scope.mappable_property_columns, {data_type: 'eui'}); @@ -389,9 +392,9 @@ angular.module('BE.seed.controller.mapping', []) $scope.duplicates_present = duplicates_present; }; - const get_col_from_suggestion = (name) => { + var get_col_from_suggestion = function (name) { - const suggestion = _.find($scope.current_preset.mappings, {from_field: name}) || {}; + var suggestion = _.find($scope.current_preset.mappings, {from_field: name}) || {}; return { from_units: suggestion.from_units, @@ -402,7 +405,7 @@ angular.module('BE.seed.controller.mapping', []) suggestion_column_name: suggestion.to_field, suggestion_table_name: suggestion.to_table_name }; - } + }; /** * initialize_mappings: prototypical inheritance for all the raw columns @@ -411,7 +414,7 @@ angular.module('BE.seed.controller.mapping', []) $scope.initialize_mappings = function () { $scope.mappings = []; _.forEach($scope.raw_columns, function (name) { - let col = get_col_from_suggestion(name); + var col = get_col_from_suggestion(name); var match; if (col.suggestion_table_name === 'PropertyState') { @@ -808,7 +811,9 @@ angular.module('BE.seed.controller.mapping', []) var init = function () { $scope.initialize_mappings(); - if ($scope.import_file.matching_done) { display_cached_column_mappings(); } + if ($scope.import_file.matching_done) { + display_cached_column_mappings(); + } $scope.updateColDuplicateStatus(); $scope.updateInventoryTypeDropdown(); diff --git a/seed/static/seed/js/controllers/members_controller.js b/seed/static/seed/js/controllers/members_controller.js index 2a79bf20ce..63680fe0a5 100644 --- a/seed/static/seed/js/controllers/members_controller.js +++ b/seed/static/seed/js/controllers/members_controller.js @@ -63,7 +63,7 @@ angular.module('BE.seed.controller.members', []) } }; - function confirm_remove_user(user) { + function confirm_remove_user (user) { organization_service.remove_user(user.user_id, $scope.org.id).then(function () { organization_service.get_organization_users({org_id: $scope.org.id}).then(function (data) { $scope.users = data.users; diff --git a/seed/static/seed/js/controllers/menu_controller.js b/seed/static/seed/js/controllers/menu_controller.js index d3e668eeb6..9d1c7d4037 100644 --- a/seed/static/seed/js/controllers/menu_controller.js +++ b/seed/static/seed/js/controllers/menu_controller.js @@ -88,14 +88,14 @@ angular.module('BE.seed.controller.menu', []) }; //Sets initial expanded/collapse state of sidebar menu - function init_menu() { + function init_menu () { //Default to false but use cookie value if one has been set var isNavExpanded = $cookies.seed_nav_is_expanded === 'true'; $scope.expanded_controller = isNavExpanded; $scope.collapsed_controller = !isNavExpanded; $scope.narrow_controller = isNavExpanded; $scope.wide_controller = !isNavExpanded; - }; + } // returns true if menu toggle has never been clicked, i.e. first run, else returns false $scope.menu_toggle_has_never_been_clicked = function () { diff --git a/seed/static/seed/js/controllers/merge_modal_controller.js b/seed/static/seed/js/controllers/merge_modal_controller.js index eaf90f9fcf..e874c77123 100644 --- a/seed/static/seed/js/controllers/merge_modal_controller.js +++ b/seed/static/seed/js/controllers/merge_modal_controller.js @@ -111,15 +111,15 @@ angular.module('BE.seed.controller.merge_modal', []) var plural = ($scope.inventory_type === 'properties' ? ' properties' : ' tax lots'); // The term "subsequent" below implies not including itself var merged_count = Math.max(result.match_merged_count - 1, 0); - var link_count = result.match_link_count; + var link_count = result.match_link_count; Notification.info({ message: (merged_count + ' subsequent ' + (merged_count === 1 ? singular : plural) + ' merged'), - delay: 10000, + delay: 10000 }); Notification.info({ message: ('Resulting ' + singular + ' has ' + link_count + ' cross-cycle link' + (link_count === 1 ? '' : 's')), - delay: 10000, + delay: 10000 }); }; @@ -136,8 +136,8 @@ angular.module('BE.seed.controller.merge_modal', []) }, headers: function () { return { - properties: "The resulting property will be further merged & linked with any matching properties.", - taxlots: "The resulting tax lot will be further merged & linked with any matching tax lots.", + properties: 'The resulting property will be further merged & linked with any matching properties.', + taxlots: 'The resulting tax lot will be further merged & linked with any matching tax lots.' }; } } @@ -155,7 +155,7 @@ angular.module('BE.seed.controller.merge_modal', []) state_ids = _.map($scope.data, 'property_state_id').reverse(); return matching_service.mergeProperties(state_ids).then(function (data) { Notification.success('Successfully merged ' + state_ids.length + ' properties'); - notify_merges_and_links(data) + notify_merges_and_links(data); $scope.close(); }, function (err) { $log.error(err); @@ -167,7 +167,7 @@ angular.module('BE.seed.controller.merge_modal', []) state_ids = _.map($scope.data, 'taxlot_state_id').reverse(); return matching_service.mergeTaxlots(state_ids).then(function (data) { Notification.success('Successfully merged ' + state_ids.length + ' tax lots'); - notify_merges_and_links(data) + notify_merges_and_links(data); $scope.close(); }, function (err) { $log.error(err); diff --git a/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js b/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js index 455cb86119..acd72b28e1 100644 --- a/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js +++ b/seed/static/seed/js/controllers/record_match_merge_link_modal_controller.js @@ -33,33 +33,33 @@ angular.module('BE.seed.controller.record_match_merge_link_modal', []) $scope.helpBtnText = 'Expand Help'; - $scope.changeHelpBtnText = function(helpBtnText) { - if(helpBtnText === 'Collapse Help'){ - $scope.helpBtnText = 'Expand Help' ; + $scope.changeHelpBtnText = function (helpBtnText) { + if (helpBtnText === 'Collapse Help') { + $scope.helpBtnText = 'Expand Help'; } else { $scope.helpBtnText = 'Collapse Help'; } }; - promises = [ - organization_service.matching_criteria_columns(organization_id), + var promises = [ + organization_service.matching_criteria_columns(organization_id) ]; if (inventory_type === 'properties') { - promises.unshift(inventory_service.get_property_columns()) + promises.unshift(inventory_service.get_property_columns()); } else if (inventory_type === 'taxlots') { - promises.unshift(inventory_service.get_taxlot_columns()) + promises.unshift(inventory_service.get_taxlot_columns()); } - $scope.matching_criteria_columns = ["Loading..."]; + $scope.matching_criteria_columns = ['Loading...']; - $q.all(promises).then(function(results) { - var inventory_columns = _.filter(results[0], { table_name: $scope.table_name }); + $q.all(promises).then(function (results) { + var inventory_columns = _.filter(results[0], {table_name: $scope.table_name}); var raw_column_names = results[1][$scope.table_name]; // Use display names to identify matching criteria columns. - $scope.matching_criteria_columns = _.map(raw_column_names, function(col_name) { - return _.find(inventory_columns, { column_name: col_name }).displayName; + $scope.matching_criteria_columns = _.map(raw_column_names, function (col_name) { + return _.find(inventory_columns, {column_name: col_name}).displayName; }); }); diff --git a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js index 0198b76717..e6b87b7990 100644 --- a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js +++ b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js @@ -58,7 +58,7 @@ angular.module('BE.seed.controller.show_populated_columns_modal', []) var relatedCols = _.filter($scope.columns, 'related'); // console.log('relatedCols', relatedCols); - var col_key = provided_inventory ? "column_name" : "name"; + var col_key = provided_inventory ? 'column_name' : 'name'; _.forEach(inventory, function (record, index) { // console.log(cols.length + ' remaining cols to check'); diff --git a/seed/static/seed/js/seed.js b/seed/static/seed/js/seed.js index ddad6a7e08..78d5ffa60e 100644 --- a/seed/static/seed/js/seed.js +++ b/seed/static/seed/js/seed.js @@ -154,7 +154,7 @@ var SEED_app = angular.module('BE.seed', [ 'BE.seed.services', 'BE.seed.controllers', 'BE.seed.utilities', - 'BE.seed.constants', + 'BE.seed.constants' ], ['$interpolateProvider', '$qProvider', function ($interpolateProvider, $qProvider) { $interpolateProvider.startSymbol('{$'); $interpolateProvider.endSymbol('$}'); @@ -241,7 +241,6 @@ SEED_app.run([ // Revert the url when the transition was triggered by a sidebar link (options.source === 'url') if (transition.options().source === 'url') { - var $state = transition.router.stateService; var $urlRouter = transition.router.urlRouter; $urlRouter.push($state.$current.navigable.url, $state.params, {replace: true}); @@ -545,21 +544,23 @@ SEED_app.config(['stateHelperProvider', '$urlRouterProvider', '$locationProvider COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, import_file_payload) { - let filter_preset_types - if (import_file_payload.import_file.source_type === "BuildingSync Raw") { - filter_preset_types = [ - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, - COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM, - ] - } else { - filter_preset_types = [COLUMN_MAPPING_PRESET_TYPE_NORMAL] - } - var organization_id = user_service.get_organization().id; - return column_mappings_service.get_column_mapping_presets_for_org( - organization_id, - filter_preset_types - ).then(response => response.data) - }], + var filter_preset_types; + if (import_file_payload.import_file.source_type === 'BuildingSync Raw') { + filter_preset_types = [ + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_DEFAULT, + COLUMN_MAPPING_PRESET_TYPE_BUILDINGSYNC_CUSTOM + ]; + } else { + filter_preset_types = [COLUMN_MAPPING_PRESET_TYPE_NORMAL]; + } + var organization_id = user_service.get_organization().id; + return column_mappings_service.get_column_mapping_presets_for_org( + organization_id, + filter_preset_types + ).then(function (response) { + return response.data; + }); + }], import_file_payload: ['dataset_service', '$stateParams', function (dataset_service, $stateParams) { var importfile_id = $stateParams.importfile_id; return dataset_service.get_import_file(importfile_id); diff --git a/seed/static/seed/js/services/column_mappings_service.js b/seed/static/seed/js/services/column_mappings_service.js index 9e75dad426..d41ba178a7 100644 --- a/seed/static/seed/js/services/column_mappings_service.js +++ b/seed/static/seed/js/services/column_mappings_service.js @@ -29,14 +29,14 @@ angular.module('BE.seed.service.column_mappings', []).factory('column_mappings_s }; column_mappings_factory.get_column_mapping_presets_for_org = function (org_id, filter_preset_types) { - const params = { - organization_id: org_id, - } + var params = { + organization_id: org_id + }; if (filter_preset_types != null) { - params.preset_type = filter_preset_types + params.preset_type = filter_preset_types; } return $http.get('/api/v2/column_mapping_presets/', { - params + params: params }).then(function (response) { return response.data; }); @@ -58,10 +58,10 @@ angular.module('BE.seed.service.column_mappings', []).factory('column_mappings_s column_mappings_factory.get_header_suggestions_for_org = function (org_id, headers) { return $http.post('/api/v2/column_mapping_presets/suggestions/', { - headers: headers, + headers: headers }, { params: { - organization_id: org_id, + organization_id: org_id } }).then(function (response) { return response.data; diff --git a/seed/static/seed/js/services/inventory_reports_service.js b/seed/static/seed/js/services/inventory_reports_service.js index a82b3c65f6..f659ce091d 100644 --- a/seed/static/seed/js/services/inventory_reports_service.js +++ b/seed/static/seed/js/services/inventory_reports_service.js @@ -44,7 +44,7 @@ angular.module('BE.seed.service.inventory_reports', function get_report_data (xVar, yVar, start, end) { // Error checks - if (_.isNil(xVar) || _.isNil(yVar) || _.isNil(start) || _.isNil(end)) { + if (_.some([xVar, yVar, start, end], _.isNil)) { $log.error('#inventory_reports_service.get_report_data(): null parameter'); throw new Error('Invalid Parameter'); } @@ -97,7 +97,7 @@ angular.module('BE.seed.service.inventory_reports', function get_aggregated_report_data (xVar, yVar, start, end) { // Error checks - if (_.isNil(xVar) || _.isNil(yVar) || _.isNil(start) || _.isNil(end)) { + if (_.some([xVar, yVar, start, end], _.isNil)) { $log.error('#inventory_reports_service.get_aggregated_report_data(): null parameter'); throw new Error('Invalid Parameter'); } @@ -123,8 +123,8 @@ angular.module('BE.seed.service.inventory_reports', var xLabel = axes_data.xLabel; var yVar = axes_data.yVar; var yLabel = axes_data.yLabel; - // Error checks - if (_.isNil(xVar) || _.isNil(xLabel) || _.isNil(yVar) || _.isNil(yLabel) || _.isNil(start) || _.isNil(end)) { + // Error checks + if (_.some([xVar, xLabel, yVar, yLabel, start, end], _.isNil)) { $log.error('#inventory_reports_service.get_aggregated_report_data(): null parameter'); throw new Error('Invalid Parameter'); } @@ -137,11 +137,11 @@ angular.module('BE.seed.service.inventory_reports', y_var: yVar, y_label: yLabel, start: start, - end: end, + end: end }, responseType: 'arraybuffer' }).then(function (response) { - return response + return response; }); } @@ -158,7 +158,7 @@ angular.module('BE.seed.service.inventory_reports', //get_summary_data : get_summary_data, get_report_data: get_report_data, get_aggregated_report_data: get_aggregated_report_data, - export_reports_data: export_reports_data, + export_reports_data: export_reports_data }; diff --git a/seed/static/seed/js/services/inventory_service.js b/seed/static/seed/js/services/inventory_service.js index b03953cb45..165a77414d 100644 --- a/seed/static/seed/js/services/inventory_service.js +++ b/seed/static/seed/js/services/inventory_service.js @@ -54,7 +54,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ return $http.post('/api/v2/properties/cycles/', { organization_id: user_service.get_organization().id, profile_id: profile_id, - cycle_ids: cycle_ids, + cycle_ids: cycle_ids }).then(function (response) { return response.data; }); @@ -163,7 +163,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ spinner_utility.show(); return $http.post('/api/v2/properties/' + view_id + '/links/', { - organization_id: user_service.get_organization().id, + organization_id: user_service.get_organization().id }).then(function (response) { return response.data; }).finally(function () { @@ -288,7 +288,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ return $http.post('/api/v2/taxlots/cycles/', { organization_id: user_service.get_organization().id, profile_id: profile_id, - cycle_ids: cycle_ids, + cycle_ids: cycle_ids }).then(function (response) { return response.data; }); @@ -392,7 +392,7 @@ angular.module('BE.seed.service.inventory', []).factory('inventory_service', [ spinner_utility.show(); return $http.post('/api/v2/taxlots/' + view_id + '/links/', { - organization_id: user_service.get_organization().id, + organization_id: user_service.get_organization().id }).then(function (response) { return response.data; }).finally(function () { diff --git a/seed/static/seed/js/services/organization_service.js b/seed/static/seed/js/services/organization_service.js index 30981cd76e..fe5315c24a 100644 --- a/seed/static/seed/js/services/organization_service.js +++ b/seed/static/seed/js/services/organization_service.js @@ -165,7 +165,7 @@ angular.module('BE.seed.service.organization', []).factory('organization_service organization_factory.match_merge_link = function (org_id, inventory_type) { return $http.post('/api/v2/organizations/' + org_id + '/match_merge_link/', { - inventory_type: inventory_type, + inventory_type: inventory_type }).then(function (response) { return response.data; }); @@ -181,7 +181,7 @@ angular.module('BE.seed.service.organization', []).factory('organization_service return $http.post('/api/v2/organizations/' + org_id + '/match_merge_link_preview/', { inventory_type: inventory_type, add: criteria_change_columns.add, - remove: criteria_change_columns.remove, + remove: criteria_change_columns.remove }).then(function (response) { return response.data; }); From 9d7850613c53a393bf2212749f0910c7984fd413 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Thu, 4 Jun 2020 16:27:22 -0600 Subject: [PATCH 080/135] Clarify Column's _column_fields_to_columns dedup raw col objects logic --- seed/models/columns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/seed/models/columns.py b/seed/models/columns.py index 505cfb6189..611c60c004 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -982,9 +982,11 @@ def select_col_obj(column_name, table_name, organization_column): organization=organization, table_name__in=[None, ''], column_name=field['from_field'], - is_extra_data=False # data from header rows in the files are NEVER extra data + is_extra_data=False # Column objects representing raw/header rows are NEVER extra data ) except Column.MultipleObjectsReturned: + # We want to avoid the ambiguity of having multiple Column objects for a specific raw column. + # To do that, delete all multiples along with any associated ColumnMapping objects. _log.debug( "More than one from_column found for {}.{}".format(field['to_table_name'], field['to_field'])) @@ -999,11 +1001,11 @@ def select_col_obj(column_name, table_name, organization_column): ColumnMapping.objects.filter(column_raw__id__in=models.Subquery(all_from_cols.values('id'))).delete() all_from_cols.delete() - from_org_col, _ = Column.objects.get_or_create( + from_org_col = Column.objects.create( organization=organization, table_name__in=[None, ''], column_name=field['from_field'], - is_extra_data=False # data from header rows in the files are NEVER extra data + is_extra_data=False # Column objects representing raw/header rows are NEVER extra data ) _log.debug("Creating a new from_column") From a7721dd95e16d9873e3e82d328d5bfc19a9cda73 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 11:58:42 -0400 Subject: [PATCH 081/135] refactor(api_schema): convert some methods to class or static --- seed/utils/api_schema.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index ef72c49fad..c7cfc4e8e8 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -22,7 +22,8 @@ class AutoSchemaHelper(SwaggerAutoSchema): ) } - def base_field(self, name, location_attr, description, required, type): + @staticmethod + def base_field(name, location_attr, description, required, type): """ Created to avoid needing to directly access openapi within ViewSets. Ideally, the cases below will be used instead of this one. @@ -35,7 +36,8 @@ def base_field(self, name, location_attr, description, required, type): type=type ) - def org_id_field(self, required=True): + @staticmethod + def org_id_field(required=True): return openapi.Parameter( 'organization_id', openapi.IN_QUERY, @@ -44,7 +46,8 @@ def org_id_field(self, required=True): type=openapi.TYPE_INTEGER ) - def query_integer_field(self, name, required, description): + @staticmethod + def query_integer_field(name, required, description): return openapi.Parameter( name, openapi.IN_QUERY, @@ -53,7 +56,8 @@ def query_integer_field(self, name, required, description): type=openapi.TYPE_INTEGER ) - def query_string_field(self, name, required, description): + @staticmethod + def query_string_field(name, required, description): return openapi.Parameter( name, openapi.IN_QUERY, @@ -62,7 +66,8 @@ def query_string_field(self, name, required, description): type=openapi.TYPE_STRING ) - def query_boolean_field(self, name, required, description): + @staticmethod + def query_boolean_field(name, required, description): return openapi.Parameter( name, openapi.IN_QUERY, @@ -71,7 +76,8 @@ def query_boolean_field(self, name, required, description): type=openapi.TYPE_BOOLEAN ) - def path_id_field(self, description): + @staticmethod + def path_id_field(description): return openapi.Parameter( 'id', openapi.IN_PATH, @@ -80,20 +86,22 @@ def path_id_field(self, description): type=openapi.TYPE_INTEGER ) - def body_field(self, required, description, name='body', params_to_formats={}): + @classmethod + def body_field(cls, required, description, name='body', params_to_formats={}): return openapi.Parameter( name, openapi.IN_BODY, description=description, required=required, - schema=self._build_body_schema(params_to_formats) + schema=cls._build_body_schema(params_to_formats) ) - def _build_body_schema(self, params_to_formats): + @classmethod + def _build_body_schema(cls, params_to_formats): return openapi.Schema( type=openapi.TYPE_OBJECT, properties={ - k: self.body_parameter_formats.get(format_name, "") + k: cls.body_parameter_formats.get(format_name, "") for k, format_name in params_to_formats.items() } From 4f42d8b2bb69dbe0b09bdee3d98394f668a24d26 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 13:54:24 -0400 Subject: [PATCH 082/135] refactor: fix typo --- seed/utils/api_schema.py | 2 +- seed/views/v3/data_quality.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index c7cfc4e8e8..6eca408141 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -10,7 +10,7 @@ class AutoSchemaHelper(SwaggerAutoSchema): # Used to easily build out example values displayed on Swagger page. body_parameter_formats = { - 'interger_array': openapi.Schema( + 'integer_array': openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_INTEGER) ), diff --git a/seed/views/v3/data_quality.py b/seed/views/v3/data_quality.py index 8fb930b2fa..f7837da44a 100644 --- a/seed/views/v3/data_quality.py +++ b/seed/views/v3/data_quality.py @@ -105,8 +105,8 @@ def __init__(self, *args): required=True, description="An object containing IDs of the records to perform data quality checks on. Should contain two keys- property_state_ids and taxlot_state_ids, each of which is an array of appropriate IDs.", params_to_formats={ - 'property_state_ids': 'interger_array', - 'taxlot_state_ids': 'interger_array' + 'property_state_ids': 'integer_array', + 'taxlot_state_ids': 'integer_array' } ), ], From 115c2d2b80f216ad67fa34f7ac5c0844f0729c1f Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 14:13:50 -0400 Subject: [PATCH 083/135] refactor(api_schema): make method public schema_factory --- seed/utils/api_schema.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 6eca408141..261995629c 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -93,15 +93,20 @@ def body_field(cls, required, description, name='body', params_to_formats={}): openapi.IN_BODY, description=description, required=required, - schema=cls._build_body_schema(params_to_formats) + schema=cls.schema_factory(params_to_formats) ) @classmethod - def _build_body_schema(cls, params_to_formats): + def schema_factory(cls, params_to_formats): + """Translates simple dictionary into an openapi Schema instance + + :param params_to_formats: dict[str, str] + :return: drf_yasg.openapi.Schema + """ return openapi.Schema( type=openapi.TYPE_OBJECT, properties={ - k: cls.body_parameter_formats.get(format_name, "") + k: cls.body_parameter_formats[format_name] for k, format_name in params_to_formats.items() } From 72fd92cb89e31e18a95cf6ec4a9775dbe5e37ce4 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 14:15:55 -0400 Subject: [PATCH 084/135] refactor: create api v3 for import_file endpoints --- seed/api/v3/urls.py | 2 + seed/views/v3/import_files.py | 1052 +++++++++++++++++++++++++++++++++ 2 files changed, 1054 insertions(+) create mode 100644 seed/views/v3/import_files.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 5478aa163d..4b85d46f37 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -7,6 +7,7 @@ from seed.views.v3.cycles import CycleViewSet from seed.views.v3.datasets import DatasetViewSet from seed.views.v3.data_quality import DataQualityViews +from seed.views.v3.import_files import ImportFileViewSet from seed.views.v3.organizations import OrganizationViewSet from seed.views.v3.users import UserViewSet @@ -15,6 +16,7 @@ api_v3_router.register(r'cycles', CycleViewSet, base_name='cycles') api_v3_router.register(r'datasets', DatasetViewSet, base_name='datasets') api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') +api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files') api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations') api_v3_router.register(r'users', UserViewSet, base_name='user') diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py new file mode 100644 index 0000000000..cb94b0f2f0 --- /dev/null +++ b/seed/views/v3/import_files.py @@ -0,0 +1,1052 @@ +# !/usr/bin/env python +# encoding: utf-8 +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" +import logging + +from django.contrib.postgres.fields import JSONField +from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponse, JsonResponse +from rest_framework import serializers, status, viewsets +from rest_framework.decorators import ( + action, + permission_classes, +) + +from seed.data_importer.models import ( + ImportFile, + ImportRecord +) +from seed.data_importer.models import ROW_DELIMITER +from seed.data_importer.tasks import do_checks +from seed.data_importer.tasks import ( + map_data, + geocode_buildings_task as task_geocode_buildings, + map_additional_models as task_map_additional_models, + match_buildings as task_match_buildings, + save_raw_data as task_save_raw, +) +from seed.decorators import ajax_request_class +from seed.decorators import get_prog_key +from seed.lib.mappings import mapper as simple_mapper +from seed.lib.mcm import mapper +from seed.lib.xml_mapping import mapper as xml_mapper +from seed.lib.superperms.orgs.decorators import has_perm_class +from seed.lib.superperms.orgs.models import ( + Organization, +) +from seed.lib.superperms.orgs.models import OrganizationUser +from seed.lib.superperms.orgs.permissions import SEEDOrgPermissions +from seed.models import ( + get_column_mapping, +) +from seed.models import ( + obj_to_dict, + PropertyState, + TaxLotState, + DATA_STATE_MAPPING, + DATA_STATE_MATCHING, + MERGE_STATE_UNKNOWN, + MERGE_STATE_NEW, + MERGE_STATE_MERGED, + Cycle, + Column, + PropertyAuditLog, + TaxLotAuditLog, + AUDIT_USER_EDIT, + TaxLotProperty, +) +from seed.utils.api import api_endpoint_class +from seed.utils.cache import get_cache +from seed.utils.geocode import MapQuestAPIKeyError + +_log = logging.getLogger(__name__) + + +class MappingResultsPayloadSerializer(serializers.Serializer): + q = serializers.CharField(max_length=100) + order_by = serializers.CharField(max_length=100) + filter_params = JSONField() + page = serializers.IntegerField() + number_per_page = serializers.IntegerField() + + +class MappingResultsPropertySerializer(serializers.Serializer): + pm_property_id = serializers.CharField(max_length=100) + address_line_1 = serializers.CharField(max_length=100) + property_name = serializers.CharField(max_length=100) + + +class MappingResultsTaxLotSerializer(serializers.Serializer): + pm_property_id = serializers.CharField(max_length=100) + address_line_1 = serializers.CharField(max_length=100) + jurisdiction_tax_lot_id = serializers.CharField(max_length=100) + + +class MappingResultsResponseSerializer(serializers.Serializer): + status = serializers.CharField(max_length=100) + properties = MappingResultsPropertySerializer(many=True) + tax_lots = MappingResultsTaxLotSerializer(many=True) + + +def convert_first_five_rows_to_list(header, first_five_rows): + """ + Return the first five rows. This is a complicated method because it handles converting the + persisted format of the first five rows into a list of dictionaries. It handles some basic + logic if there are crlf in the fields. Note that this method does not cover all the use cases + and cannot due to the custom delimeter. See the tests in + test_views.py:test_get_first_five_rows_newline_should_work to see the limitation + + :param header: list, ordered list of headers as strings + :param first_five_rows: string, long string with |#*#| delimeter. + :return: list + """ + row_data = [] + rows = [] + number_of_columns = len(header) + split_cells = first_five_rows.split(ROW_DELIMITER) + number_cells = len(split_cells) + # catch the case where there is only one column, therefore no ROW_DELIMITERs + if number_of_columns == 1: + # Note that this does not support having a single column with carriage returns! + rows = first_five_rows.splitlines() + else: + for idx, l in enumerate(split_cells): + crlf_count = l.count('\n') + + if crlf_count == 0: + row_data.append(l) + elif crlf_count >= 1: + # if add this element to row_data equals number_of_columns, then it is a new row + if len(row_data) == number_of_columns - 1: + # check if this is the last columns, if so, then just store the value and move on + if idx == number_cells - 1: + row_data.append(l) + rows.append(row_data) + continue + else: + # split the cell_data. The last cell becomes the beginning of the new + # row, and the other cells stay joined with \n. + cell_data = l.splitlines() + row_data.append('\n'.join(cell_data[:crlf_count])) + rows.append(row_data) + + # initialize the next row_data with the remainder + row_data = [cell_data[-1]] + continue + else: + # this is not the end, so it must be a carriage return in the cell, just save data + row_data.append(l) + + if len(row_data) == number_of_columns: + rows.append(row_data) + + return [dict(zip(header, row)) for row in rows] + + +class ImportFileViewSet(viewsets.ViewSet): + raise_exception = True + queryset = ImportFile.objects.all() + + @api_endpoint_class + @ajax_request_class + def retrieve(self, request, pk=None): + """ + Retrieves details about an ImportFile. + --- + type: + status: + required: true + type: string + description: either success or error + import_file: + type: ImportFile structure + description: full detail of import file + parameter_strategy: replace + parameters: + - name: pk + description: "Primary Key" + required: true + paramType: path + """ + + import_file_id = pk + orgs = request.user.orgs.all() + try: + import_file = ImportFile.objects.get( + pk=import_file_id + ) + d = ImportRecord.objects.filter( + super_organization__in=orgs, pk=import_file.import_record_id + ) + except ObjectDoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'Could not access an import file with this ID' + }, status=status.HTTP_403_FORBIDDEN) + # check if user has access to the import file + if not d.exists(): + return JsonResponse({ + 'status': 'error', + 'message': 'Could not locate import file with this ID', + 'import_file': {}, + }, status=status.HTTP_400_BAD_REQUEST) + + f = obj_to_dict(import_file) + f['name'] = import_file.filename_only + if not import_file.uploaded_filename: + f['uploaded_filename'] = import_file.filename + f['dataset'] = obj_to_dict(import_file.import_record) + + return JsonResponse({ + 'status': 'success', + 'import_file': f, + }) + + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['GET']) + def first_five_rows(self, request, pk=None): + """ + Retrieves the first five rows of an ImportFile. + --- + type: + status: + required: true + type: string + description: either success or error + first_five_rows: + type: array of strings + description: list of strings for each of the first five rows for this import file + parameter_strategy: replace + parameters: + - name: pk + description: "Primary Key" + required: true + paramType: path + """ + import_file = ImportFile.objects.get(pk=pk) + if import_file is None: + return JsonResponse( + {'status': 'error', 'message': 'Could not find import file with pk=' + str( + pk)}, status=status.HTTP_400_BAD_REQUEST) + if import_file.cached_second_to_fifth_row is None: + return JsonResponse({'status': 'error', + 'message': 'Internal problem occurred, import file first five rows not cached'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + ''' + import_file.cached_second_to_fifth_row is a field that contains the first + 5 lines of data from the file, split on newlines, delimited by + ROW_DELIMITER. This becomes an issue when fields have newlines in them, + so the following is to handle newlines in the fields. + In the case of only one data column there will be no ROW_DELIMITER. + ''' + header = import_file.cached_first_row.split(ROW_DELIMITER) + data = import_file.cached_second_to_fifth_row + return JsonResponse({ + 'status': 'success', + 'first_five_rows': convert_first_five_rows_to_list(header, data) + }) + + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['GET']) + def raw_column_names(self, request, pk=None): + """ + Retrieves a list of all column names from an ImportFile. + --- + type: + status: + required: true + type: string + description: either success or error + raw_columns: + type: array of strings + description: list of strings of the header row of the ImportFile + parameter_strategy: replace + parameters: + - name: pk + description: "Primary Key" + required: true + paramType: path + """ + import_file = ImportFile.objects.get(pk=pk) + return JsonResponse({ + 'status': 'success', + 'raw_columns': import_file.first_row_columns + }) + + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['POST'], url_path='filtered_mapping_results') + def filtered_mapping_results(self, request, pk=None): + """ + Retrieves a paginated list of Properties and Tax Lots for an import file after mapping. + --- + parameter_strategy: replace + parameters: + - name: pk + description: Import File ID (Primary key) + type: integer + required: true + paramType: path + response_serializer: MappingResultsResponseSerializer + """ + import_file_id = pk + org_id = request.query_params.get('organization_id', False) + + # get the field names that were in the mapping + import_file = ImportFile.objects.get(id=import_file_id) + + # List of the only fields to show + field_names = import_file.get_cached_mapped_columns + + # set of fields + fields = { + 'PropertyState': ['id', 'extra_data', 'lot_number'], + 'TaxLotState': ['id', 'extra_data'] + } + columns_from_db = Column.retrieve_all(org_id) + property_column_name_mapping = {} + taxlot_column_name_mapping = {} + for field_name in field_names: + # find the field from the columns in the database + for column in columns_from_db: + if column['table_name'] == 'PropertyState' and \ + field_name[0] == 'PropertyState' and \ + field_name[1] == column['column_name']: + property_column_name_mapping[column['column_name']] = column['name'] + if not column['is_extra_data']: + fields['PropertyState'].append(field_name[1]) # save to the raw db fields + continue + elif column['table_name'] == 'TaxLotState' and \ + field_name[0] == 'TaxLotState' and \ + field_name[1] == column['column_name']: + taxlot_column_name_mapping[column['column_name']] = column['name'] + if not column['is_extra_data']: + fields['TaxLotState'].append(field_name[1]) # save to the raw db fields + continue + + inventory_type = request.data.get('inventory_type', 'all') + + result = { + 'status': 'success' + } + + if inventory_type == 'properties' or inventory_type == 'all': + properties = PropertyState.objects.filter( + import_file_id=import_file_id, + data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], + merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW] + ).only(*fields['PropertyState']).order_by('id') + + property_results = [] + for prop in properties: + prop_dict = TaxLotProperty.model_to_dict_with_mapping( + prop, + property_column_name_mapping, + fields=fields['PropertyState'], + exclude=['extra_data'] + ) + + prop_dict.update( + TaxLotProperty.extra_data_to_dict_with_mapping( + prop.extra_data, + property_column_name_mapping, + fields=prop.extra_data.keys(), + ).items() + ) + property_results.append(prop_dict) + + result['properties'] = property_results + + if inventory_type == 'taxlots' or inventory_type == 'all': + tax_lots = TaxLotState.objects.filter( + import_file_id=import_file_id, + data_state__in=[DATA_STATE_MAPPING, DATA_STATE_MATCHING], + merge_state__in=[MERGE_STATE_UNKNOWN, MERGE_STATE_NEW] + ).only(*fields['TaxLotState']).order_by('id') + + tax_lot_results = [] + for tax_lot in tax_lots: + tax_lot_dict = TaxLotProperty.model_to_dict_with_mapping( + tax_lot, + taxlot_column_name_mapping, + fields=fields['TaxLotState'], + exclude=['extra_data'] + ) + tax_lot_dict.update( + TaxLotProperty.extra_data_to_dict_with_mapping( + tax_lot.extra_data, + taxlot_column_name_mapping, + fields=tax_lot.extra_data.keys(), + ).items() + ) + tax_lot_results.append(tax_lot_dict) + + result['tax_lots'] = tax_lot_results + + return result + + @staticmethod + def has_coparent(state_id, inventory_type, fields=None): + """ + Return the coparent of the current state id based on the inventory type. If fields + are given (as a list), then it will only return the fields specified of the state model + object as a dictionary. + + :param state_id: int, ID of PropertyState or TaxLotState + :param inventory_type: string, either properties | taxlots + :param fields: list, either None or list of fields to return + :return: dict or state object, If fields is not None then will return state_object + """ + state_model = PropertyState if inventory_type == 'properties' else TaxLotState + + # TODO: convert coparent to instance method, not class method + audit_entry, audit_count = state_model.coparent(state_id) + + if audit_count == 0: + return False + + if audit_count > 1: + return JsonResponse( + { + 'status': 'error', + 'message': 'Internal problem occurred, more than one merge record found' + }, + status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + return audit_entry[0] + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def perform_mapping(self, request, pk=None): + """ + Starts a background task to convert imported raw data into + PropertyState and TaxLotState, using user's column mappings. + --- + type: + status: + required: true + type: string + description: either success or error + progress_key: + type: integer + description: ID of background job, for retrieving job progress + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + + body = request.data + + remap = body.get('remap', False) + mark_as_done = body.get('mark_as_done', True) + if not ImportFile.objects.filter(pk=pk).exists(): + return { + 'status': 'error', + 'message': 'ImportFile {} does not exist'.format(pk) + } + + # return remap_data(import_file_id) + return JsonResponse(map_data(pk, remap, mark_as_done)) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def start_system_matching_and_geocoding(self, request, pk=None): + """ + Starts a background task to attempt automatic matching between buildings + in an ImportFile with other existing buildings within the same org. + --- + type: + status: + required: true + type: string + description: either success or error + progress_key: + type: integer + description: ID of background job, for retrieving job progress + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + try: + task_geocode_buildings(pk) + except MapQuestAPIKeyError: + result = JsonResponse({ + 'status': 'error', + 'message': 'MapQuest API key may be invalid or at its limit.' + }, status=status.HTTP_403_FORBIDDEN) + return result + + try: + import_file = ImportFile.objects.get(pk=pk) + except ImportFile.DoesNotExist: + return { + 'status': 'error', + 'message': 'ImportFile {} does not exist'.format(pk) + } + + # if the file is BuildingSync, don't do the merging, but instead finish + # creating it's associated models (scenarios, meters, etc) + if import_file.from_buildingsync: + return task_map_additional_models(pk) + + return task_match_buildings(pk) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def start_data_quality_checks(self, request, pk=None): + """ + Starts a background task to attempt automatic matching between buildings + in an ImportFile with other existing buildings within the same org. + --- + type: + status: + required: true + type: string + description: either success or error + progress_key: + type: integer + description: ID of background job, for retrieving job progress + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + organization = Organization.objects.get(pk=request.query_params['organization_id']) + + return_value = do_checks(organization.id, None, None, pk) + # step 5: create a new model instance + return JsonResponse({ + 'progress_key': return_value['progress_key'], + 'progress': return_value, + }) + + @api_endpoint_class + @ajax_request_class + @action(detail=True, methods=['GET']) + def data_quality_progress(self, request, pk=None): + """ + Return the progress of the data quality check. + --- + type: + status: + required: true + type: string + description: either success or error + progress: + type: integer + description: status of background data quality task + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + + import_file_id = pk + prog_key = get_prog_key('get_progress', import_file_id) + cache = get_cache(prog_key) + return HttpResponse(cache['progress']) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def save_raw_data(self, request, pk=None): + """ + Starts a background task to import raw data from an ImportFile + into PropertyState objects as extra_data. If the cycle_id is set to + year_ending then the cycle ID will be set to the year_ending column for each + record in the uploaded file. Note that the year_ending flag is not yet enabled. + --- + type: + status: + required: true + type: string + description: either success or error + message: + required: false + type: string + description: error message, if any + progress_key: + type: integer + description: ID of background job, for retrieving job progress + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + - name: cycle_id + description: The ID of the cycle or the string "year_ending" + paramType: string + required: true + """ + body = request.data + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must pass file_id of file to save' + }, status=status.HTTP_400_BAD_REQUEST) + + cycle_id = body.get('cycle_id') + if not cycle_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must pass cycle_id of the cycle to save the data' + }, status=status.HTTP_400_BAD_REQUEST) + elif cycle_id == 'year_ending': + _log.error("NOT CONFIGURED FOR YEAR ENDING OPTION AT THE MOMENT") + return JsonResponse({ + 'status': 'error', + 'message': 'SEED is unable to parse year_ending at the moment' + }, status=status.HTTP_400_BAD_REQUEST) + else: + # find the cycle + cycle = Cycle.objects.get(id=cycle_id) + if cycle: + # assign the cycle id to the import file object + import_file = ImportFile.objects.get(id=import_file_id) + import_file.cycle = cycle + import_file.save() + else: + return JsonResponse({ + 'status': 'error', + 'message': 'cycle_id was invalid' + }, status=status.HTTP_400_BAD_REQUEST) + + return JsonResponse(task_save_raw(import_file_id)) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['PUT']) + def mapping_done(self, request, pk=None): + """ + Tell the backend that the mapping is complete. + --- + type: + status: + required: true + type: string + description: either success or error + message: + required: false + type: string + description: error message, if any + parameter_strategy: replace + parameters: + - name: pk + description: Import file ID + required: true + paramType: path + """ + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must pass import_file_id' + }, status=status.HTTP_400_BAD_REQUEST) + + import_file = ImportFile.objects.get(pk=import_file_id) + import_file.mapping_done = True + import_file.save() + + return JsonResponse( + { + 'status': 'success', + 'message': '' + } + ) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @action(detail=True, methods=['POST']) + def save_column_mappings(self, request, pk=None): + """ + Saves the mappings between the raw headers of an ImportFile and the + destination fields in the `to_table_name` model which should be either + PropertyState or TaxLotState + + Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` + + Payload:: + + { + "import_file_id": ID of the ImportFile record, + "mappings": [ + { + 'from_field': 'eui', # raw field in import file + 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional + 'to_field': 'energy_use_intensity', + 'to_field_display_name': 'Energy Use Intensity', + 'to_table_name': 'PropertyState', + }, + { + 'from_field': 'gfa', + 'from_units': 'ft**2', # pint-parsable units, optional + 'to_field': 'gross_floor_area', + 'to_field_display_name': 'Gross Floor Area', + 'to_table_name': 'PropertyState', + } + ] + } + + Returns:: + + {'status': 'success'} + """ + body = request.data + import_file = ImportFile.objects.get(pk=pk) + organization = import_file.import_record.super_organization + mappings = body.get('mappings', []) + result = Column.create_mappings(mappings, organization, request.user, import_file.id) + + if result: + return JsonResponse({'status': 'success'}) + else: + return JsonResponse({'status': 'error'}) + + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @action(detail=True, methods=['GET']) + def matching_and_geocoding_results(self, request, pk=None): + """ + Retrieves the number of matched and unmatched properties & tax lots for + a given ImportFile record. Specifically for new imports + + :GET: Expects import_file_id corresponding to the ImportFile in question. + + Returns:: + + { + 'status': 'success', + 'properties': { + 'matched': Number of PropertyStates that have been matched, + 'unmatched': Number of PropertyStates that are unmatched new imports + }, + 'tax_lots': { + 'matched': Number of TaxLotStates that have been matched, + 'unmatched': Number of TaxLotStates that are unmatched new imports + } + } + + """ + import_file = ImportFile.objects.get(pk=pk) + + # property views associated with this imported file (including merges) + properties_new = [] + properties_matched = list(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_MERGED, + ).values_list('id', flat=True)) + + # Check audit log in case PropertyStates are listed as "new" but were merged into a different property + properties = list(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_NEW, + )) + + for state in properties: + audit_creation_id = PropertyAuditLog.objects.only('id').exclude( + import_filename=None).get( + state_id=state.id, + name='Import Creation' + ) + if PropertyAuditLog.objects.exclude(record_type=AUDIT_USER_EDIT).filter( + parent1_id=audit_creation_id + ).exists(): + properties_matched.append(state.id) + else: + properties_new.append(state.id) + + tax_lots_new = [] + tax_lots_matched = list(TaxLotState.objects.only('id').filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_MERGED, + ).values_list('id', flat=True)) + + # Check audit log in case TaxLotStates are listed as "new" but were merged into a different tax lot + taxlots = list(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + merge_state=MERGE_STATE_NEW, + )) + + for state in taxlots: + audit_creation_id = TaxLotAuditLog.objects.only('id').exclude(import_filename=None).get( + state_id=state.id, + name='Import Creation' + ) + if TaxLotAuditLog.objects.exclude(record_type=AUDIT_USER_EDIT).filter( + parent1_id=audit_creation_id + ).exists(): + tax_lots_matched.append(state.id) + else: + tax_lots_new.append(state.id) + + # Construct Geocode Results + property_geocode_results = { + 'high_confidence': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='High' + )), + 'low_confidence': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='Low' + )), + 'manual': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Manually geocoded (N/A)' + )), + 'missing_address_components': len(PropertyState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Missing address components (N/A)' + )), + } + + tax_lot_geocode_results = { + 'high_confidence': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='High' + )), + 'low_confidence': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence__startswith='Low' + )), + 'manual': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Manually geocoded (N/A)' + )), + 'missing_address_components': len(TaxLotState.objects.filter( + import_file__pk=import_file.pk, + data_state=DATA_STATE_MATCHING, + geocoding_confidence='Missing address components (N/A)' + )), + } + + # merge in any of the matching results from the JSON field + return { + 'status': 'success', + 'import_file_records': import_file.matching_results_data.get('import_file_records', None), + 'properties': { + 'initial_incoming': import_file.matching_results_data.get('property_initial_incoming', None), + 'duplicates_against_existing': import_file.matching_results_data.get('property_duplicates_against_existing', None), + 'duplicates_within_file': import_file.matching_results_data.get('property_duplicates_within_file', None), + 'merges_against_existing': import_file.matching_results_data.get('property_merges_against_existing', None), + 'merges_between_existing': import_file.matching_results_data.get('property_merges_between_existing', None), + 'merges_within_file': import_file.matching_results_data.get('property_merges_within_file', None), + 'new': import_file.matching_results_data.get('property_new', None), + 'geocoded_high_confidence': property_geocode_results.get('high_confidence'), + 'geocoded_low_confidence': property_geocode_results.get('low_confidence'), + 'geocoded_manually': property_geocode_results.get('manual'), + 'geocode_not_possible': property_geocode_results.get('missing_address_components'), + }, + 'tax_lots': { + 'initial_incoming': import_file.matching_results_data.get('tax_lot_initial_incoming', None), + 'duplicates_against_existing': import_file.matching_results_data.get('tax_lot_duplicates_against_existing', None), + 'duplicates_within_file': import_file.matching_results_data.get('tax_lot_duplicates_within_file', None), + 'merges_against_existing': import_file.matching_results_data.get('tax_lot_merges_against_existing', None), + 'merges_between_existing': import_file.matching_results_data.get('tax_lot_merges_between_existing', None), + 'merges_within_file': import_file.matching_results_data.get('tax_lot_merges_within_file', None), + 'new': import_file.matching_results_data.get('tax_lot_new', None), + 'geocoded_high_confidence': tax_lot_geocode_results.get('high_confidence'), + 'geocoded_low_confidence': tax_lot_geocode_results.get('low_confidence'), + 'geocoded_manually': tax_lot_geocode_results.get('manual'), + 'geocode_not_possible': tax_lot_geocode_results.get('missing_address_components'), + } + } + + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @permission_classes((SEEDOrgPermissions,)) + @action(detail=True, methods=['GET']) + def mapping_suggestions(self, request, pk): + """ + Returns suggested mappings from an uploaded file's headers to known + data fields. + --- + type: + status: + required: true + type: string + description: Either success or error + suggested_column_mappings: + required: true + type: dictionary + description: Dictionary where (key, value) = (the column header from the file, + array of tuples (destination column, score)) + building_columns: + required: true + type: array + description: A list of all possible columns + building_column_types: + required: true + type: array + description: A list of column types corresponding to the building_columns array + parameter_strategy: replace + parameters: + - name: pk + description: import_file_id + required: true + paramType: path + - name: organization_id + description: The organization_id for this user's organization + required: true + paramType: query + + """ + organization_id = request.query_params.get('organization_id', None) + + result = {'status': 'success'} + + membership = OrganizationUser.objects.select_related('organization') \ + .get(organization_id=organization_id, user=request.user) + organization = membership.organization + + # For now, each organization holds their own mappings. This is non-ideal, but it is the + # way it is for now. In order to move to parent_org holding, then we need to be able to + # dynamically match columns based on the names and not the db id (or support many-to-many). + # parent_org = organization.get_parent() + + import_file = ImportFile.objects.get( + pk=pk, + import_record__super_organization_id=organization.pk + ) + + # Get a list of the database fields in a list, these are the db columns and the extra_data columns + property_columns = Column.retrieve_mapping_columns(organization.pk, 'property') + taxlot_columns = Column.retrieve_mapping_columns(organization.pk, 'taxlot') + + # If this is a portfolio manager file, then load in the PM mappings and if the column_mappings + # are not in the original mappings, default to PM + if import_file.from_portfolio_manager: + pm_mappings = simple_mapper.get_pm_mapping(import_file.first_row_columns, + resolve_duplicates=True) + suggested_mappings = mapper.build_column_mapping( + import_file.first_row_columns, + Column.retrieve_all_by_tuple(organization_id), + previous_mapping=get_column_mapping, + map_args=[organization], + default_mappings=pm_mappings, + thresh=80 + ) + elif import_file.from_buildingsync: + bsync_mappings = xml_mapper.build_column_mapping() + suggested_mappings = mapper.build_column_mapping( + import_file.first_row_columns, + Column.retrieve_all_by_tuple(organization_id), + previous_mapping=get_column_mapping, + map_args=[organization], + default_mappings=bsync_mappings, + thresh=80 + ) + else: + # All other input types + suggested_mappings = mapper.build_column_mapping( + import_file.first_row_columns, + Column.retrieve_all_by_tuple(organization.pk), + previous_mapping=get_column_mapping, + map_args=[organization], + thresh=80 # percentage match that we require. 80% is random value for now. + ) + # replace None with empty string for column names and PropertyState for tables + # TODO #239: Move this fix to build_column_mapping + for m in suggested_mappings: + table, destination_field, _confidence = suggested_mappings[m] + if destination_field is None: + suggested_mappings[m][1] = '' + + # Fix the table name, eventually move this to the build_column_mapping + for m in suggested_mappings: + table, _destination_field, _confidence = suggested_mappings[m] + # Do not return the campus, created, updated fields... that is force them to be in the property state + if not table or table == 'Property': + suggested_mappings[m][0] = 'PropertyState' + elif table == 'TaxLot': + suggested_mappings[m][0] = 'TaxLotState' + + result['suggested_column_mappings'] = suggested_mappings + result['property_columns'] = property_columns + result['taxlot_columns'] = taxlot_columns + + return JsonResponse(result) + + @api_endpoint_class + @ajax_request_class + @permission_classes((SEEDOrgPermissions,)) + @has_perm_class('requires_member') + def destroy(self, request, pk): + """ + Returns suggested mappings from an uploaded file's headers to known + data fields. + --- + type: + status: + required: true + type: string + description: Either success or error + parameter_strategy: replace + parameters: + - name: pk + description: import_file_id + required: true + paramType: path + - name: organization_id + description: The organization_id for this user's organization + required: true + paramType: query + + """ + organization_id = int(request.query_params.get('organization_id', None)) + import_file = ImportFile.objects.get(pk=pk) + + # check if the import record exists for the file and organization + d = ImportRecord.objects.filter( + super_organization_id=organization_id, + pk=import_file.import_record.pk + ) + + if not d.exists(): + return JsonResponse({ + 'status': 'error', + 'message': 'user does not have permission to delete file', + }, status=status.HTTP_403_FORBIDDEN) + + # This does not actually delete the object because it is a NonDeletableModel + import_file.delete() + return JsonResponse({'status': 'success'}) From 5b5c8371861dd710bd1628ac3a047663ad7ef7b6 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 16:44:39 -0400 Subject: [PATCH 085/135] refactor: improve import files viewset for swagger --- seed/views/v3/import_files.py | 337 +++++++--------------------------- 1 file changed, 65 insertions(+), 272 deletions(-) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index cb94b0f2f0..a11205ab26 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -6,9 +6,9 @@ """ import logging -from django.contrib.postgres.fields import JSONField from django.core.exceptions import ObjectDoesNotExist -from django.http import HttpResponse, JsonResponse +from django.http import JsonResponse +from drf_yasg.utils import swagger_auto_schema from rest_framework import serializers, status, viewsets from rest_framework.decorators import ( action, @@ -29,7 +29,6 @@ save_raw_data as task_save_raw, ) from seed.decorators import ajax_request_class -from seed.decorators import get_prog_key from seed.lib.mappings import mapper as simple_mapper from seed.lib.mcm import mapper from seed.lib.xml_mapping import mapper as xml_mapper @@ -59,20 +58,12 @@ TaxLotProperty, ) from seed.utils.api import api_endpoint_class -from seed.utils.cache import get_cache +from seed.utils.api_schema import AutoSchemaHelper from seed.utils.geocode import MapQuestAPIKeyError _log = logging.getLogger(__name__) -class MappingResultsPayloadSerializer(serializers.Serializer): - q = serializers.CharField(max_length=100) - order_by = serializers.CharField(max_length=100) - filter_params = JSONField() - page = serializers.IntegerField() - number_per_page = serializers.IntegerField() - - class MappingResultsPropertySerializer(serializers.Serializer): pm_property_id = serializers.CharField(max_length=100) address_line_1 = serializers.CharField(max_length=100) @@ -91,6 +82,39 @@ class MappingResultsResponseSerializer(serializers.Serializer): tax_lots = MappingResultsTaxLotSerializer(many=True) +class MappingSerializer(serializers.Serializer): + from_field = serializers.CharField() + from_units = serializers.CharField() + to_field = serializers.CharField() + to_field_display_name = serializers.CharField() + to_table_name = serializers.CharField() + + +class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): + """ + Example: + { + "mappings": [ + { + 'from_field': 'eui', # raw field in import file + 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional + 'to_field': 'energy_use_intensity', + 'to_field_display_name': 'Energy Use Intensity', + 'to_table_name': 'PropertyState', + }, + { + 'from_field': 'gfa', + 'from_units': 'ft**2', # pint-parsable units, optional + 'to_field': 'gross_floor_area', + 'to_field_display_name': 'Gross Floor Area', + 'to_table_name': 'PropertyState', + } + ] + } + """ + mappings = serializers.ListField(child=MappingSerializer()) + + def convert_first_five_rows_to_list(header, first_five_rows): """ Return the first five rows. This is a complicated method because it handles converting the @@ -155,23 +179,7 @@ class ImportFileViewSet(viewsets.ViewSet): def retrieve(self, request, pk=None): """ Retrieves details about an ImportFile. - --- - type: - status: - required: true - type: string - description: either success or error - import_file: - type: ImportFile structure - description: full detail of import file - parameter_strategy: replace - parameters: - - name: pk - description: "Primary Key" - required: true - paramType: path """ - import_file_id = pk orgs = request.user.orgs.all() try: @@ -211,21 +219,6 @@ def retrieve(self, request, pk=None): def first_five_rows(self, request, pk=None): """ Retrieves the first five rows of an ImportFile. - --- - type: - status: - required: true - type: string - description: either success or error - first_five_rows: - type: array of strings - description: list of strings for each of the first five rows for this import file - parameter_strategy: replace - parameters: - - name: pk - description: "Primary Key" - required: true - paramType: path """ import_file = ImportFile.objects.get(pk=pk) if import_file is None: @@ -256,21 +249,6 @@ def first_five_rows(self, request, pk=None): def raw_column_names(self, request, pk=None): """ Retrieves a list of all column names from an ImportFile. - --- - type: - status: - required: true - type: string - description: either success or error - raw_columns: - type: array of strings - description: list of strings of the header row of the ImportFile - parameter_strategy: replace - parameters: - - name: pk - description: "Primary Key" - required: true - paramType: path """ import_file = ImportFile.objects.get(pk=pk) return JsonResponse({ @@ -278,21 +256,20 @@ def raw_column_names(self, request, pk=None): 'raw_columns': import_file.first_row_columns }) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.org_id_field(), + ], + responses={ + 200: MappingResultsResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @action(detail=True, methods=['POST'], url_path='filtered_mapping_results') def filtered_mapping_results(self, request, pk=None): """ Retrieves a paginated list of Properties and Tax Lots for an import file after mapping. - --- - parameter_strategy: replace - parameters: - - name: pk - description: Import File ID (Primary key) - type: integer - required: true - paramType: path - response_serializer: MappingResultsResponseSerializer """ import_file_id = pk org_id = request.query_params.get('organization_id', False) @@ -429,21 +406,6 @@ def perform_mapping(self, request, pk=None): """ Starts a background task to convert imported raw data into PropertyState and TaxLotState, using user's column mappings. - --- - type: - status: - required: true - type: string - description: either success or error - progress_key: - type: integer - description: ID of background job, for retrieving job progress - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID - required: true - paramType: path """ body = request.data @@ -467,21 +429,6 @@ def start_system_matching_and_geocoding(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. - --- - type: - status: - required: true - type: string - description: either success or error - progress_key: - type: integer - description: ID of background job, for retrieving job progress - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID - required: true - paramType: path """ try: task_geocode_buildings(pk) @@ -507,6 +454,7 @@ def start_system_matching_and_geocoding(self, request, pk=None): return task_match_buildings(pk) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @@ -515,21 +463,6 @@ def start_data_quality_checks(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. - --- - type: - status: - required: true - type: string - description: either success or error - progress_key: - type: integer - description: ID of background job, for retrieving job progress - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID - required: true - paramType: path """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -540,34 +473,9 @@ def start_data_quality_checks(self, request, pk=None): 'progress': return_value, }) - @api_endpoint_class - @ajax_request_class - @action(detail=True, methods=['GET']) - def data_quality_progress(self, request, pk=None): - """ - Return the progress of the data quality check. - --- - type: - status: - required: true - type: string - description: either success or error - progress: - type: integer - description: status of background data quality task - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID - required: true - paramType: path - """ - - import_file_id = pk - prog_key = get_prog_key('get_progress', import_file_id) - cache = get_cache(prog_key) - return HttpResponse(cache['progress']) - + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({'cycle_id': 'string'}) + ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @@ -578,29 +486,6 @@ def save_raw_data(self, request, pk=None): into PropertyState objects as extra_data. If the cycle_id is set to year_ending then the cycle ID will be set to the year_ending column for each record in the uploaded file. Note that the year_ending flag is not yet enabled. - --- - type: - status: - required: true - type: string - description: either success or error - message: - required: false - type: string - description: error message, if any - progress_key: - type: integer - description: ID of background job, for retrieving job progress - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID - required: true - paramType: path - - name: cycle_id - description: The ID of the cycle or the string "year_ending" - paramType: string - required: true """ body = request.data import_file_id = pk @@ -645,22 +530,6 @@ def save_raw_data(self, request, pk=None): def mapping_done(self, request, pk=None): """ Tell the backend that the mapping is complete. - --- - type: - status: - required: true - type: string - description: either success or error - message: - required: false - type: string - description: error message, if any - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID - required: true - paramType: path """ import_file_id = pk if not import_file_id: @@ -669,7 +538,14 @@ def mapping_done(self, request, pk=None): 'message': 'must pass import_file_id' }, status=status.HTTP_400_BAD_REQUEST) - import_file = ImportFile.objects.get(pk=import_file_id) + try: + import_file = ImportFile.objects.get(pk=import_file_id) + except ImportFile.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'no import file with given id' + }, status=status.HTTP_404_NOT_FOUND) + import_file.mapping_done = True import_file.save() @@ -680,6 +556,12 @@ def mapping_done(self, request, pk=None): } ) + @swagger_auto_schema( + request_body=SaveColumnMappingsRequestPayloadSerializer, + responses={ + 200: 'success response' + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -691,32 +573,6 @@ def save_column_mappings(self, request, pk=None): PropertyState or TaxLotState Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` - - Payload:: - - { - "import_file_id": ID of the ImportFile record, - "mappings": [ - { - 'from_field': 'eui', # raw field in import file - 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional - 'to_field': 'energy_use_intensity', - 'to_field_display_name': 'Energy Use Intensity', - 'to_table_name': 'PropertyState', - }, - { - 'from_field': 'gfa', - 'from_units': 'ft**2', # pint-parsable units, optional - 'to_field': 'gross_floor_area', - 'to_field_display_name': 'Gross Floor Area', - 'to_table_name': 'PropertyState', - } - ] - } - - Returns:: - - {'status': 'success'} """ body = request.data import_file = ImportFile.objects.get(pk=pk) @@ -737,23 +593,6 @@ def matching_and_geocoding_results(self, request, pk=None): """ Retrieves the number of matched and unmatched properties & tax lots for a given ImportFile record. Specifically for new imports - - :GET: Expects import_file_id corresponding to the ImportFile in question. - - Returns:: - - { - 'status': 'success', - 'properties': { - 'matched': Number of PropertyStates that have been matched, - 'unmatched': Number of PropertyStates that are unmatched new imports - }, - 'tax_lots': { - 'matched': Number of TaxLotStates that have been matched, - 'unmatched': Number of TaxLotStates that are unmatched new imports - } - } - """ import_file = ImportFile.objects.get(pk=pk) @@ -890,6 +729,7 @@ def matching_and_geocoding_results(self, request, pk=None): } } + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -899,36 +739,6 @@ def mapping_suggestions(self, request, pk): """ Returns suggested mappings from an uploaded file's headers to known data fields. - --- - type: - status: - required: true - type: string - description: Either success or error - suggested_column_mappings: - required: true - type: dictionary - description: Dictionary where (key, value) = (the column header from the file, - array of tuples (destination column, score)) - building_columns: - required: true - type: array - description: A list of all possible columns - building_column_types: - required: true - type: array - description: A list of column types corresponding to the building_columns array - parameter_strategy: replace - parameters: - - name: pk - description: import_file_id - required: true - paramType: path - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query - """ organization_id = request.query_params.get('organization_id', None) @@ -1006,31 +816,14 @@ def mapping_suggestions(self, request, pk): return JsonResponse(result) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @api_endpoint_class @ajax_request_class @permission_classes((SEEDOrgPermissions,)) @has_perm_class('requires_member') def destroy(self, request, pk): """ - Returns suggested mappings from an uploaded file's headers to known - data fields. - --- - type: - status: - required: true - type: string - description: Either success or error - parameter_strategy: replace - parameters: - - name: pk - description: import_file_id - required: true - paramType: path - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query - + Deletes an import file """ organization_id = int(request.query_params.get('organization_id', None)) import_file = ImportFile.objects.get(pk=pk) From c1eb3638ef9bbb0b7edf7c8c074bb5cc5df3c53a Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 16:50:01 -0400 Subject: [PATCH 086/135] refactor!: improve names of some endpoints --- seed/views/v3/import_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index a11205ab26..4d263fb33d 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -402,7 +402,7 @@ def has_coparent(state_id, inventory_type, fields=None): @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) - def perform_mapping(self, request, pk=None): + def map(self, request, pk=None): """ Starts a background task to convert imported raw data into PropertyState and TaxLotState, using user's column mappings. @@ -480,7 +480,7 @@ def start_data_quality_checks(self, request, pk=None): @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) - def save_raw_data(self, request, pk=None): + def start_save_data(self, request, pk=None): """ Starts a background task to import raw data from an ImportFile into PropertyState objects as extra_data. If the cycle_id is set to @@ -526,7 +526,7 @@ def save_raw_data(self, request, pk=None): @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') - @action(detail=True, methods=['PUT']) + @action(detail=True, methods=['POST']) def mapping_done(self, request, pk=None): """ Tell the backend that the mapping is complete. From de1bccb335a57096086112766afb39e77aae650c Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Thu, 4 Jun 2020 18:30:38 -0400 Subject: [PATCH 087/135] refactor(apiV3)!: move save_column_mappings endpoint --- seed/views/v3/import_files.py | 62 ---------------------- seed/views/v3/organizations.py | 95 ++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 62 deletions(-) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index 4d263fb33d..06c8fafe3a 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -82,39 +82,6 @@ class MappingResultsResponseSerializer(serializers.Serializer): tax_lots = MappingResultsTaxLotSerializer(many=True) -class MappingSerializer(serializers.Serializer): - from_field = serializers.CharField() - from_units = serializers.CharField() - to_field = serializers.CharField() - to_field_display_name = serializers.CharField() - to_table_name = serializers.CharField() - - -class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): - """ - Example: - { - "mappings": [ - { - 'from_field': 'eui', # raw field in import file - 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional - 'to_field': 'energy_use_intensity', - 'to_field_display_name': 'Energy Use Intensity', - 'to_table_name': 'PropertyState', - }, - { - 'from_field': 'gfa', - 'from_units': 'ft**2', # pint-parsable units, optional - 'to_field': 'gross_floor_area', - 'to_field_display_name': 'Gross Floor Area', - 'to_table_name': 'PropertyState', - } - ] - } - """ - mappings = serializers.ListField(child=MappingSerializer()) - - def convert_first_five_rows_to_list(header, first_five_rows): """ Return the first five rows. This is a complicated method because it handles converting the @@ -556,35 +523,6 @@ def mapping_done(self, request, pk=None): } ) - @swagger_auto_schema( - request_body=SaveColumnMappingsRequestPayloadSerializer, - responses={ - 200: 'success response' - } - ) - @api_endpoint_class - @ajax_request_class - @has_perm_class('requires_member') - @action(detail=True, methods=['POST']) - def save_column_mappings(self, request, pk=None): - """ - Saves the mappings between the raw headers of an ImportFile and the - destination fields in the `to_table_name` model which should be either - PropertyState or TaxLotState - - Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` - """ - body = request.data - import_file = ImportFile.objects.get(pk=pk) - organization = import_file.import_record.super_organization - mappings = body.get('mappings', []) - result = Column.create_mappings(mappings, organization, request.user, import_file.id) - - if result: - return JsonResponse({'status': 'success'}) - else: - return JsonResponse({'status': 'error'}) - @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py index 9e28669998..7caa71b97a 100644 --- a/seed/views/v3/organizations.py +++ b/seed/views/v3/organizations.py @@ -7,6 +7,9 @@ import logging from django.http import JsonResponse +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import serializers from rest_framework import status from rest_framework import viewsets from rest_framework.decorators import action @@ -15,10 +18,46 @@ from seed.lib.superperms.orgs.decorators import has_perm_class from seed.lib.superperms.orgs.models import Organization from seed.models.columns import Column +from seed.models import ImportFile +from seed.utils.api import api_endpoint_class +from seed.utils.api_schema import AutoSchemaHelper _log = logging.getLogger(__name__) +class MappingSerializer(serializers.Serializer): + from_field = serializers.CharField() + from_units = serializers.CharField() + to_field = serializers.CharField() + to_field_display_name = serializers.CharField() + to_table_name = serializers.CharField() + + +class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): + """ + Example: + { + "mappings": [ + { + 'from_field': 'eui', # raw field in import file + 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional + 'to_field': 'energy_use_intensity', + 'to_field_display_name': 'Energy Use Intensity', + 'to_table_name': 'PropertyState', + }, + { + 'from_field': 'gfa', + 'from_units': 'ft**2', # pint-parsable units, optional + 'to_field': 'gross_floor_area', + 'to_field_display_name': 'Gross Floor Area', + 'to_table_name': 'PropertyState', + } + ] + } + """ + mappings = serializers.ListField(child=MappingSerializer()) + + class OrganizationViewSet(viewsets.ViewSet): model = Column @@ -67,3 +106,59 @@ def columns(self, request, pk=None): 'status': 'error', 'message': 'organization with with id {} does not exist'.format(pk) }, status=status.HTTP_404_NOT_FOUND) + + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.query_integer_field( + 'import_file_id', required=True, description='Import file id'), + openapi.Parameter( + 'id', openapi.IN_PATH, type=openapi.TYPE_INTEGER, description='Organization id'), + ], + request_body=SaveColumnMappingsRequestPayloadSerializer, + responses={ + 200: 'success response' + } + ) + @api_endpoint_class + @ajax_request_class + @has_perm_class('requires_member') + @action(detail=True, methods=['POST']) + def column_mappings(self, request, pk=None): + """ + Saves the mappings between the raw headers of an ImportFile and the + destination fields in the `to_table_name` model which should be either + PropertyState or TaxLotState + + Valid source_type values are found in ``seed.models.SEED_DATA_SOURCES`` + """ + import_file_id = request.query_params.get('import_file_id') + if import_file_id is None: + return JsonResponse({ + 'status': 'error', + 'message': 'Query param `import_file_id` is required' + }, status=status.HTTP_400_BAD_REQUEST) + try: + _ = ImportFile.objects.get(pk=import_file_id) + organization = Organization.objects.get(pk=pk) + except ImportFile.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'No import file found' + }, status=status.HTTP_404_NOT_FOUND) + except Organization.DoesNotExist: + return JsonResponse({ + 'status': 'error', + 'message': 'No organization found' + }, status=status.HTTP_404_NOT_FOUND) + + result = Column.create_mappings( + request.data.get('mappings', []), + organization, + request.user, + import_file_id + ) + + if result: + return JsonResponse({'status': 'success'}) + else: + return JsonResponse({'status': 'error'}) From 0bd4bdf584f5d7a3e8dd5e5b18b00d10ee35fd79 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 11:29:48 -0400 Subject: [PATCH 088/135] refactor(api_schema): pass schema_factory kwargs to openapi instance --- seed/utils/api_schema.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 261995629c..efececf6c3 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -97,7 +97,7 @@ def body_field(cls, required, description, name='body', params_to_formats={}): ) @classmethod - def schema_factory(cls, params_to_formats): + def schema_factory(cls, params_to_formats, **kwargs): """Translates simple dictionary into an openapi Schema instance :param params_to_formats: dict[str, str] @@ -109,7 +109,8 @@ def schema_factory(cls, params_to_formats): k: cls.body_parameter_formats[format_name] for k, format_name in params_to_formats.items() - } + }, + **kwargs ) def add_manual_parameters(self, parameters): From 48c55458721230405e9545d4834bb73ff08a0d42 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 12:01:15 -0400 Subject: [PATCH 089/135] refactor(api_schema): schema_factory handles complex types --- seed/utils/api_schema.py | 43 ++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index efececf6c3..7e53152b33 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -19,7 +19,8 @@ class AutoSchemaHelper(SwaggerAutoSchema): 'string_array': openapi.Schema( type=openapi.TYPE_ARRAY, items=openapi.Schema(type=openapi.TYPE_STRING) - ) + ), + 'integer': openapi.Schema(type=openapi.TYPE_INTEGER), } @staticmethod @@ -97,21 +98,37 @@ def body_field(cls, required, description, name='body', params_to_formats={}): ) @classmethod - def schema_factory(cls, params_to_formats, **kwargs): - """Translates simple dictionary into an openapi Schema instance + def schema_factory(cls, obj, **kwargs): + """Translates an object into an openapi Schema - :param params_to_formats: dict[str, str] + :param obj: str, list, dict[str, obj] :return: drf_yasg.openapi.Schema """ - return openapi.Schema( - type=openapi.TYPE_OBJECT, - properties={ - k: cls.body_parameter_formats[format_name] - for k, format_name - in params_to_formats.items() - }, - **kwargs - ) + if type(obj) is str: + if obj not in cls.body_parameter_formats: + raise Exception(f'Invalid type "{obj}"; expected one of {cls.body_parameter_formats.keys()}') + return cls.body_parameter_formats[obj] + + if type(obj) is list: + if len(obj) != 1: + raise Exception(f'List types must have exactly one element to specify the schema of `items`') + return openapi.Schema( + type=openapi.TYPE_ARRAY, + items=cls.schema_factory(obj[0]) + ) + + if type(obj) is dict: + return openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + k: cls.schema_factory(sub_obj) + for k, sub_obj + in obj.items() + }, + **kwargs + ) + + raise Exception(f'Unhandled type "{type(obj)}" for {obj}') def add_manual_parameters(self, parameters): manual_params = self.manual_fields.get((self.method, self.view.action), []) From 67919372df56c304dae96099aa95904295086957 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 12:02:35 -0400 Subject: [PATCH 090/135] chore(v3/data_quality): improve swagger docs --- seed/views/v3/data_quality.py | 243 ++++++++++++---------------------- 1 file changed, 88 insertions(+), 155 deletions(-) diff --git a/seed/views/v3/data_quality.py b/seed/views/v3/data_quality.py index f7837da44a..d8ba265bfc 100644 --- a/seed/views/v3/data_quality.py +++ b/seed/views/v3/data_quality.py @@ -9,6 +9,7 @@ from celery.utils.log import get_task_logger from django.http import JsonResponse, HttpResponse +from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets, serializers, status from rest_framework.decorators import action from unidecode import unidecode @@ -30,6 +31,7 @@ logger = get_task_logger(__name__) +# TODO: Consider moving these serializers into seed/serializers (and maybe actually use them...) class RulesSubSerializer(serializers.Serializer): field = serializers.CharField(max_length=100) severity = serializers.CharField(max_length=100) @@ -55,6 +57,29 @@ class RulesSerializer(serializers.Serializer): data_quality_rules = RulesIntermediateSerializer() +class RulePayloadSerializer(serializers.ModelSerializer): + # currently our requests put this in a different field + label = serializers.IntegerField(source='status_label') + + class Meta: + model = Rule + exclude = ['id', 'status_label'] + + +class DataQualityRulesSerializer(serializers.Serializer): + properties = serializers.ListField(child=RulePayloadSerializer()) + taxlots = serializers.ListField(child=RulePayloadSerializer()) + + +class SaveDataQualityRulesPayloadSerializer(serializers.Serializer): + data_quality_rules = DataQualityRulesSerializer() + + +class DataQualityRulesResponseSerializer(serializers.Serializer): + status = serializers.CharField() + rules = DataQualityRulesSerializer() + + def _get_js_rule_type(data_type): """return the JS friendly data type name for the data data_quality rule @@ -93,79 +118,40 @@ def _get_severity_from_js(severity): return d.get(severity) -class DataQualitySchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('POST', 'create'): [ - self.org_id_field(), - self.body_field( - name='data_quality_ids', - required=True, - description="An object containing IDs of the records to perform data quality checks on. Should contain two keys- property_state_ids and taxlot_state_ids, each of which is an array of appropriate IDs.", - params_to_formats={ - 'property_state_ids': 'integer_array', - 'taxlot_state_ids': 'integer_array' - } - ), - ], - ('GET', 'data_quality_rules'): [self.org_id_field()], - ('PUT', 'reset_all_data_quality_rules'): [self.org_id_field()], - ('PUT', 'reset_default_data_quality_rules'): [self.org_id_field()], - ('POST', 'save_data_quality_rules'): [ - self.org_id_field(), - self.body_field( - name='data_quality_rules', - required=True, - description="Rules information" - ) - ], - ('GET', 'results'): [ - self.org_id_field(), - self.query_integer_field( - name='data_quality_id', - required=True, - description="Task ID created when DataQuality task is created." - ), - ], - ('GET', 'csv'): [ - # This will replace the auto-generated field - adds description. - self.path_id_field(description="Import file ID or cache key") - ], - } - - class DataQualityViews(viewsets.ViewSet): """ Handles Data Quality API operations within Inventory backend. (1) Post, wait, get… (2) Respond with what changed """ - swagger_schema = DataQualitySchema + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.org_id_field(), + ], + request_body=AutoSchemaHelper.schema_factory( + { + 'property_state_ids': 'integer_array', + 'taxlot_state_ids': 'integer_array', + }, + description='An object containing IDs of the records to perform' + ' data quality checks on. Should contain two keys- ' + 'property_state_ids and taxlot_state_ids, each of ' + 'which is an array of appropriate IDs.', + ), + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'num_properties': 'integer', + 'num_taxlots': 'integer', + 'progress_key': 'string', + 'progress': {}, + }) + } + ) def create(self, request): """ This API endpoint will create a new cleansing operation process in the background, on potentially a subset of properties/taxlots, and return back a query key - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - - name: data_quality_ids - description: An object containing IDs of the records to perform data quality checks on. - Should contain two keys- property_state_ids and taxlot_state_ids, each of which is an array - of appropriate IDs. - required: true - paramType: body - type: - status: - type: string - description: success or error - required: true """ # step 0: retrieving the data body = request.data @@ -187,6 +173,11 @@ def create(self, request): 'progress': return_value, }) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.path_id_field(description="Import file ID or cache key") + ] + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -194,13 +185,6 @@ def create(self, request): def csv(self, request, pk): """ Download a csv of the data quality checks by the pk which is the cache_key - --- - parameter_strategy: replace - parameters: - - name: pk - description: Import file ID or cache key - required: true - paramType: path """ data_quality_results = get_cache_raw(DataQualityCheck.cache_key(pk)) response = HttpResponse(content_type='text/csv') @@ -233,6 +217,12 @@ def csv(self, request, pk): return response + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -240,22 +230,6 @@ def csv(self, request, pk): def data_quality_rules(self, request): """ Returns the data_quality rules for an org. - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - type: - status: - type: string - required: true - description: success or error - rules: - type: object - required: true - description: An object containing 'properties' and 'taxlots' arrays of rules """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -288,6 +262,12 @@ def data_quality_rules(self, request): return JsonResponse(result) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -295,30 +275,6 @@ def data_quality_rules(self, request): def reset_all_data_quality_rules(self, request): """ Resets an organization's data data_quality rules - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - type: - status: - type: string - description: success or error - required: true - in_range_checking: - type: array[string] - required: true - description: An array of in-range error rules - missing_matching_field: - type: array[string] - required: true - description: An array of fields to verify existence - missing_values: - type: array[string] - required: true - description: An array of fields to ignore missing values """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -326,6 +282,12 @@ def reset_all_data_quality_rules(self, request): dq.reset_all_rules() return self.data_quality_rules(request) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -333,30 +295,6 @@ def reset_all_data_quality_rules(self, request): def reset_default_data_quality_rules(self, request): """ Resets an organization's data data_quality rules - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - type: - status: - type: string - description: success or error - required: true - in_range_checking: - type: array[string] - required: true - description: An array of in-range error rules - missing_matching_field: - type: array[string] - required: true - description: An array of fields to verify existence - missing_values: - type: array[string] - required: true - description: An array of fields to ignore missing values """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -364,6 +302,13 @@ def reset_default_data_quality_rules(self, request): dq.reset_default_rules() return self.data_quality_rules(request) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + request_body=SaveDataQualityRulesPayloadSerializer, + responses={ + 200: DataQualityRulesResponseSerializer + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_parent_org_owner') @@ -373,28 +318,6 @@ def save_data_quality_rules(self, request, pk=None): Saves an organization's settings: name, query threshold, shared fields. The method passes in all the fields again, so it is okay to remove all the rules in the db, and just recreate them (albeit inefficient) - --- - parameter_strategy: replace - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - paramType: query - - name: body - description: JSON body containing organization rules information - paramType: body - pytype: RulesSerializer - required: true - type: - status: - type: string - description: success or error - required: true - message: - type: string - description: error message, if any - required: true """ organization = Organization.objects.get(pk=request.query_params['organization_id']) @@ -464,6 +387,16 @@ def save_data_quality_rules(self, request, pk=None): return self.data_quality_rules(request) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.org_id_field(), + AutoSchemaHelper.query_integer_field( + name='data_quality_id', + required=True, + description="Task ID created when DataQuality task is created." + ), + ] + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') From 3136685e65f57eb171980f8f61a91035506b9382 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 12:15:56 -0400 Subject: [PATCH 091/135] chore(v3/cycles): improve swagger docs --- seed/views/v3/cycles.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 22b6d03951..487e9aa022 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -7,6 +7,9 @@ All rights reserved. # NOQA :authors Paul Munday Fable Turas """ +from django.utils.decorators import method_decorator +from drf_yasg.utils import swagger_auto_schema + from seed.models import Cycle from seed.serializers.cycles import CycleSerializer @@ -14,19 +17,27 @@ from seed.utils.api_schema import AutoSchemaHelper -class CycleSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('GET', 'list'): [self.org_id_field()], - ('POST', 'create'): [self.org_id_field()], - ('GET', 'retrieve'): [self.org_id_field()], - ('PUT', 'update'): [self.org_id_field()], - ('DELETE', 'destroy'): [self.org_id_field()], - } - - +# adds organization_id query param to a view +org_param_swagger_decorator = swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()] +) + + +@method_decorator( + name='list', + decorator=org_param_swagger_decorator) +@method_decorator( + name='create', + decorator=org_param_swagger_decorator) +@method_decorator( + name='retrieve', + decorator=org_param_swagger_decorator) +@method_decorator( + name='update', + decorator=org_param_swagger_decorator) +@method_decorator( + name='destroy', + decorator=org_param_swagger_decorator) class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): """API endpoint for viewing and creating cycles (time periods). @@ -67,7 +78,6 @@ class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): """ serializer_class = CycleSerializer - swagger_schema = CycleSchema pagination_class = None model = Cycle data_name = 'cycles' From 4f4425d76f375060b35d8e3f414b31061b6ae8d4 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 12:44:20 -0400 Subject: [PATCH 092/135] chore(v3/columns): improve swagger docs chore(v3/columns): remove old swagger class --- seed/views/v3/columns.py | 124 +++++++++++++-------------------------- 1 file changed, 42 insertions(+), 82 deletions(-) diff --git a/seed/views/v3/columns.py b/seed/views/v3/columns.py index 45f88ade47..d04bd11a88 100644 --- a/seed/views/v3/columns.py +++ b/seed/views/v3/columns.py @@ -6,6 +6,8 @@ """ import logging from django.http import JsonResponse +from django.utils.decorators import method_decorator +from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer @@ -21,46 +23,23 @@ _log = logging.getLogger(__name__) -class ColumnSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('GET', 'list'): [ - self.org_id_field(), - self.query_string_field( - name='inventory_type', - required=False, - description='Which inventory type is being matched (for related fields and naming)' - '\nDefault: "property"' - ), - self.query_boolean_field( - name='used_only', - required=False, - description='Determine whether or not to show only the used fields ' - '(i.e. only columns that have been mapped)' - '\nDefault: "false"' - ), - ], - ('POST', 'create'): [self.org_id_field(required=False)], - ('GET', 'retrieve'): [self.org_id_field()], - ('DELETE', 'delete'): [self.org_id_field()], - ('POST', 'rename'): [ - self.body_field( - name='data', - required=True, - description="rename columns", - params_to_formats={ - 'new_column_name': 'string', - 'overwrite': 'boolean' - } - ) - ] - } - - self.overwrite_params.extend([('POST', 'rename')]) +org_param_swagger_decorator = swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()] +) +@method_decorator( + name='create', + decorator=org_param_swagger_decorator +) +@method_decorator( + name='update', + decorator=org_param_swagger_decorator +) +@method_decorator( + name='destroy', + decorator=org_param_swagger_decorator +) class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet, OrgCreateUpdateMixin): """ create: @@ -76,13 +55,30 @@ class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet, Org model = Column pagination_class = None parser_classes = (JSONParser, FormParser) - swagger_schema = ColumnSchema def get_queryset(self): # check if the request is properties or taxlots org_id = self.get_organization(self.request) return Column.objects.filter(organization_id=org_id) + @swagger_auto_schema( + manual_parameters=[ + AutoSchemaHelper.org_id_field(required=False), + AutoSchemaHelper.query_string_field( + name='inventory_type', + required=False, + description='Which inventory type is being matched (for related fields and naming)' + '\nDefault: "property"' + ), + AutoSchemaHelper.query_boolean_field( + name='used_only', + required=False, + description='Determine whether or not to show only the used fields ' + '(i.e. only columns that have been mapped)' + '\nDefault: "false"' + ), + ], + ) @ajax_request_class def list(self, request): """ @@ -90,31 +86,6 @@ def list(self, request): return all the columns across both the Property and Tax Lot tables. The related field will be true if the column came from the other table that is not the 'inventory_type' (which defaults to Property) - ___ - parameters: - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query - - name: inventory_type - description: Which inventory type is being matched (for related fields and naming). - property or taxlot - required: false - paramType: query - - name: used_only - description: Determine whether or not to show only the used fields (i.e. only columns that have been mapped) - type: boolean - required: false - paramType: query - type: - status: - required: true - type: string - description: Either success or error - columns: - required: true - type: array[column] - description: Returns an array where each item is a full column structure. """ organization_id = self.get_organization(self.request) inventory_type = request.query_params.get('inventory_type', 'property') @@ -125,27 +96,11 @@ def list(self, request): 'columns': columns, }) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @ajax_request_class def retrieve(self, request, pk=None): """ This API endpoint retrieves a Column - --- - parameters: - - name: organization_id - description: Organization ID - type: integer - required: true - type: - status: - type: string - description: success or error - required: true - column: - required: true - type: dictionary - description: Returns a dictionary of a full column structure with keys such as - keys ''name'', ''id'', ''is_extra_data'', ''column_name'', - ''table_name'',.. """ organization_id = self.get_organization(self.request) # check if column exists for the organization @@ -168,13 +123,18 @@ def retrieve(self, request, pk=None): 'column': ColumnSerializer(c).data }) + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'new_column_name': 'string', + 'overwrite': 'boolean' + }) + ) @ajax_request_class @has_perm_class('can_modify_data') @action(detail=True, methods=['POST']) def rename(self, request, pk=None): """ This API endpoint renames a Column - --- """ org_id = self.get_organization(request) try: From 6e8b665cc29cb43e83c2871edde9f68da1863869 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jun 2020 18:35:31 +0000 Subject: [PATCH 093/135] Bump django from 2.2.10 to 2.2.13 in /requirements Bumps [django](https://github.com/django/django) from 2.2.10 to 2.2.13. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/2.2.10...2.2.13) Signed-off-by: dependabot[bot] --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index be1837d9d2..318532423a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # Django -django==2.2.10 +django==2.2.13 # NL 12/30/2019 - DJ database url is not longer used? #dj-database-url==0.5.0 From 86b85cd88646e7e1ab9689c6fd870ec26b9baabc Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 14:59:49 -0400 Subject: [PATCH 094/135] chore(v3/datasets): improve swagger docs --- seed/views/v3/datasets.py | 188 +++++++------------------------------- 1 file changed, 32 insertions(+), 156 deletions(-) diff --git a/seed/views/v3/datasets.py b/seed/views/v3/datasets.py index 9ab7e2d472..a402ee8122 100644 --- a/seed/views/v3/datasets.py +++ b/seed/views/v3/datasets.py @@ -8,6 +8,7 @@ from django.http import JsonResponse from django.utils import timezone +from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework import viewsets from rest_framework.decorators import action @@ -25,69 +26,16 @@ _log = logging.getLogger(__name__) -class DatasetSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('GET', 'list'): [self.org_id_field()], - ('POST', 'create'): [ - self.org_id_field(), - self.body_field( - name='name', - required=True, - description="The name of the dataset to be created", - params_to_formats={ - 'name': 'string' - } - ), - ], - ('GET', 'count'): [self.org_id_field()], - ('GET', 'retrieve'): [ - self.org_id_field()], - ('PUT', 'update'): [ - self.org_id_field(), - self.body_field( - name='dataset', - required=True, - description="The new name for this dataset", - params_to_formats={ - 'dataset': 'string' - } - ) - ], - ('DELETE', 'destroy'): [self.org_id_field()] - } - - class DatasetViewSet(viewsets.ViewSet): raise_exception = True - swagger_schema = DatasetSchema - + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()],) @require_organization_id_class @api_endpoint_class @ajax_request_class def list(self, request): """ Retrieves all datasets for the user's organization. - --- - type: - status: - required: true - type: string - description: Either success or error - datasets: - required: true - type: array[dataset] - description: Returns an array where each item is a full dataset structure, including - keys ''name'', ''number_of_buildings'', ''id'', ''updated_at'', - ''last_modified_by'', ''importfiles'', ... - parameters: - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query """ org_id = request.query_params.get('organization_id', None) @@ -107,35 +55,19 @@ def list(self, request): 'datasets': datasets, }) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + {'dataset': 'string'}, + description='The new name for this dataset' + ) + ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') def update(self, request, pk=None): """ Updates the name of a dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: either success or error - message: - type: string - description: error message, if any - parameter_strategy: replace - parameters: - - name: organization_id - description: "The organization_id" - required: true - paramType: query - - name: dataset - description: "The new name for this dataset" - required: true - paramType: string - - name: pk - description: "Primary Key" - required: true - paramType: path """ organization_id = request.query_params.get('organization_id', None) @@ -166,33 +98,12 @@ def update(self, request, pk=None): 'status': 'success', }) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): """ Retrieves a dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: Either success or error - dataset: - required: true - type: dictionary - description: A dictionary of a full dataset structure, including - keys ''name'', ''id'', ''updated_at'', - ''last_modified_by'', ''importfiles'', ... - parameter_strategy: replace - parameters: - - name: pk - description: The ID of the dataset to retrieve - required: true - paramType: path - - name: organization_id - description: The organization_id for this user's organization - required: true - paramType: query """ organization_id = request.query_params.get('organization_id', None) @@ -250,31 +161,13 @@ def retrieve(self, request, pk=None): 'dataset': dataset, }) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') def destroy(self, request, pk=None): """ Deletes a dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: either success or error - message: - type: string - description: error message, if any - parameter_strategy: replace - parameters: - - name: organization_id - description: The organization_id - required: true - paramType: query - - name: pk - description: "Primary Key" - required: true - paramType: path """ organization_id = int(request.query_params.get('organization_id', None)) dataset_id = pk @@ -291,6 +184,19 @@ def destroy(self, request, pk=None): d.delete() return JsonResponse({'status': 'success'}) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + {'name': 'string'}, + description='Name of the dataset to be created' + ), + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'id': 'integer', + 'name': 'string', + }) + } + ) @require_organization_id_class @api_endpoint_class @ajax_request_class @@ -298,29 +204,6 @@ def destroy(self, request, pk=None): def create(self, request): """ Creates a new empty dataset (ImportRecord). - --- - type: - status: - required: true - type: string - description: either success or error - id: - required: true - type: integer - description: primary key for the newly created dataset - name: - required: true - type: string - description: name of the newly created dataset - parameters: - - name: name - description: The name of this dataset - required: true - paramType: string - - name: organization_id - description: The organization_id - required: true - paramType: query """ body = request.data @@ -343,6 +226,14 @@ def create(self, request): return JsonResponse({'status': 'success', 'id': record.pk, 'name': record.name}) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'datasets_count': 'integer', + }) + } + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_viewer') @@ -351,21 +242,6 @@ def create(self, request): def count(self, request): """ Retrieves the number of datasets for an org. - --- - parameters: - - name: organization_id - description: The organization_id - required: true - paramType: query - type: - status: - description: success or error - type: string - required: true - datasets_count: - description: Number of datasets belonging to this org - type: integer - required: true """ org_id = int(request.query_params.get('organization_id', None)) From 61dc5c76d7a8c27f409fa71f87f2d84204bcdf5f Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 16:08:14 -0400 Subject: [PATCH 095/135] chore(v3/users): improve swagger docs --- seed/views/v3/users.py | 426 ++++++++--------------------------------- 1 file changed, 81 insertions(+), 345 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 7dede3e68b..bf9db1c9b7 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -11,6 +11,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.http import JsonResponse +from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets, status, serializers from rest_framework.decorators import action @@ -111,91 +112,18 @@ class ListUsersResponseSerializer(serializers.Serializer): users = EmailAndIDSerializer(many=True) -class UserSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('GET', 'list'): [], - ('POST', 'create'): [ - self.org_id_field(), - self.body_field( - name='New User Fields', - required=True, - description="An object containing meta data for a new user: \n" - "- Required - first_name, last_name, email \n" - "- Optional - role ['viewer'(default), 'member', or 'owner']", - params_to_formats={ - 'first_name': 'string', - 'last_name': 'string', - 'role': 'string', - 'email': 'string' - }) - ], - ('GET', 'current_user_id'): [], - ('GET', 'retrieve'): [self.path_id_field(description="users PK ID")], - ('PUT', 'update'): [ - self.path_id_field(description="Updated Users PK ID"), - self.body_field( - name='Updated user fields', - required=True, - description="An object containing meta data for a updated user: \n" - "- Required - first_name, last_name, email", - params_to_formats={ - 'first_name': 'string', - 'last_name': 'string', - 'email': 'string' - } - )], - ('PUT', 'default_organization'): [self.org_id_field(), - self.path_id_field(description="Updated Users PK ID")], - ('POST', 'is_authorized'): [ - self.org_id_field(), - self.path_id_field(description="Users PK ID"), - self.body_field( - name='Actions', - required=True, - description="A list of actions to check: examples include (requires_parent_org_owner, " - "requires_owner, requires_member, requires_viewer, " - "requires_superuser, can_create_sub_org, can_remove_org)", - params_to_formats={ - 'actions': 'string_array', - }) - ], - ('PUT', 'set_password'): [ - self.body_field( - name='Change Password', - required=True, - description="fill in the current and new matching passwords ", - params_to_formats={ - 'current_password': 'string', - 'password_1': 'string', - 'password_2': 'string' - }) - ], - ('PUT', 'role'): [ - self.org_id_field(), - self.path_id_field(description="Users PK ID"), - self.body_field( - name='role', - required=True, - description="fill in the role to be updated", - params_to_formats={ - 'role': 'string', - - }) - ], - ('PUT', 'deactivate'): [ - self.path_id_field(description="Users PK ID") - ] - } +# this is used for swagger docs for some views below +user_response_schema = AutoSchemaHelper.schema_factory({ + 'first_name': 'string', + 'last_name': 'string', + 'email': 'string', + 'api_key': 'string', +}) class UserViewSet(viewsets.ViewSet): raise_exception = True - swagger_schema = UserSchema - def validate_request_user(self, pk, request): try: user = User.objects.get(pk=pk) @@ -209,6 +137,15 @@ def validate_request_user(self, pk, request): status=status.HTTP_403_FORBIDDEN) return True, user + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + request_body=AutoSchemaHelper.schema_factory({ + 'first_name': 'string', + 'last_name': 'string', + 'role': 'string', + 'email': 'string', + }) + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_owner') @@ -216,57 +153,6 @@ def create(self, request): """ Creates a new SEED user. One of 'organization_id' or 'org_name' is needed. Sends invitation email to the new user. - --- - parameters: - - name: organization_id - description: Organization ID if adding user to an existing organization - required: false - type: integer - - name: org_name - description: New organization name if creating a new organization for this user - required: false - type: string - - name: first_name - description: First name of new user - required: true - type: string - - name: last_name - description: Last name of new user - required: true - type: string - - name: role - description: one of owner, member, or viewer - required: true - type: string - - name: email - description: Email address of the new user - required: true - type: string - type: - status: - description: success or error - required: true - type: string - message: - description: email address of new user - required: true - type: string - org: - description: name of the new org (or existing org) - required: true - type: string - org_created: - description: True if new org created - required: true - type: string - username: - description: Username of new user - required: true - type: string - user_id: - description: User ID (pk) of new user - required: true - type: integer """ body = request.data org_name = body.get('org_name') @@ -335,14 +221,17 @@ def create(self, request): 'user_id': user.id }) + @swagger_auto_schema( + responses={ + 200: ListUsersResponseSerializer, + } + ) @ajax_request_class @has_perm_class('requires_superuser') def list(self, request): """ Retrieves all users' email addresses and IDs. Only usable by superusers. - --- - response_serializer: ListUsersResponseSerializer """ users = [] for user in User.objects.only('id', 'email'): @@ -365,6 +254,15 @@ def current(self, request): """ return JsonResponse({'pk': request.user.id}) + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'role': 'string', + }, + description='new role for user', + ), + ) @api_endpoint_class @ajax_request_class @has_perm_class('requires_owner') @@ -372,31 +270,6 @@ def current(self, request): def role(self, request, pk=None): """ Updates a user's role within an organization. - --- - parameter_strategy: replace - parameters: - - name: pk - description: ID for the user to modify - type: integer - required: true - paramType: path - - name: organization_id - description: The organization ID to update this user within - type: integer - required: true - - name: role - description: one of owner, member, or viewer - type: string - required: true - type: - status: - required: true - description: success or error - type: string - message: - required: false - description: error message, if any - type: string """ body = request.data role = _get_role_from_js(body['role']) @@ -432,41 +305,17 @@ def role(self, request, pk=None): return JsonResponse({'status': 'success'}) + @swagger_auto_schema( + responses={ + 200: user_response_schema, + } + ) @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): """ Retrieves the a user's first_name, last_name, email and api key if exists by user ID (pk). - --- - parameter_strategy: replace - parameters: - - name: pk - description: User ID / primary key - type: integer - required: true - paramType: path - type: - status: - description: success or error - type: string - required: true - first_name: - description: user first name - type: string - required: true - last_name: - description: user last name - type: string - required: true - email: - description: user email - type: string - required: true - api_key: - description: user API key - type: string - required: true """ ok, content = self.validate_request_user(pk, request) @@ -516,52 +365,23 @@ def generate_api_key(self, request, pk=None): 'api_key': User.objects.get(pk=pk).api_key } + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'first_name': 'string', + 'last_name': 'string', + 'email': 'string' + }), + description='An object containing meta data for a updated user: \n' + '- Required - first_name, last_name, email', + responses={ + 200: user_response_schema, + } + ) @api_endpoint_class @ajax_request_class def update(self, request, pk=None): """ Updates the user's first name, last name, and email - --- - parameter_strategy: replace - parameters: - - name: pk - description: User ID / primary key - type: integer - required: true - paramType: path - - name: first_name - description: New first name - type: string - required: true - - name: last_name - description: New last name - type: string - required: true - - name: email - description: New user email - type: string - required: true - type: - status: - description: success or error - type: string - required: true - first_name: - description: user first name - type: string - required: true - last_name: - description: user last name - type: string - required: true - email: - description: user email - type: string - required: true - api_key: - description: user API key - type: string - required: true """ body = request.data ok, content = self.validate_request_user(pk, request) @@ -583,36 +403,19 @@ def update(self, request, pk=None): 'api_key': user.api_key, }) + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'current_password': 'string', + 'password_1': 'string', + 'password_2': 'string', + }), + ) @ajax_request_class @action(detail=True, methods=['PUT']) def set_password(self, request, pk=None): """ sets/updates a user's password, follows the min requirement of django password validation settings in config/settings/common.py - --- - parameter_strategy: replace - parameters: - - name: current_password - description: Users current password - type: string - required: true - - name: password_1 - description: Users new password 1 - type: string - required: true - - name: password_2 - description: Users new password 2 - type: string - required: true - type: - status: - type: string - description: success or error - required: true - message: - type: string - description: error message, if any - required: false """ body = request.data ok, content = self.validate_request_user(pk, request) @@ -646,42 +449,30 @@ def get_actions(self, request): 'actions': list(PERMS.keys()), } + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.org_id_field()], + request_body=AutoSchemaHelper.schema_factory( + { + 'actions': 'string_array', + }, + description='A list of actions to check: examples include (requires_parent_org_owner, ' + 'requires_owner, requires_member, requires_viewer, ' + 'requires_superuser, can_create_sub_org, can_remove_org)' + ), + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'auth': { + 'action_name': 'boolean' + } + }) + } + ) @ajax_request_class @action(detail=True, methods=['POST']) def is_authorized(self, request, pk=None): """ Checks the auth for a given action, if user is the owner of the parent org then True is returned for each action - --- - parameter_strategy: replace - parameters: - - name: pk - description: User ID (primary key) - type: integer - required: true - paramType: path - - name: organization_id - description: ID (primary key) for organization - type: integer - required: true - paramType: query - - name: actions - type: array[string] - required: true - description: a list of actions to check - type: - status: - type: string - description: success or error - required: true - message: - type: string - description: error message, if any - required: false - auth: - type: object - description: a dict of with keys equal to the actions, and values as bool - required: true """ actions, org, error, message = self._parse_is_authenticated_params(request) if error: @@ -763,32 +554,18 @@ def _try_parent_org_auth(self, user, organization, actions): action: PERMS['requires_owner'](ou) for action in actions } + @swagger_auto_schema( + responses={ + 200: AutoSchemaHelper.schema_factory({ + 'show_shared_buildings': 'boolean' + }) + } + ) @ajax_request_class @action(detail=True, methods=['GET']) def shared_buildings(self, request, pk=None): """ Get the request user's ``show_shared_buildings`` attr - --- - parameter_strategy: replace - parameters: - - name: pk - description: User ID (primary key) - type: integer - required: true - paramType: path - type: - status: - type: string - description: success or error - required: true - show_shared_buildings: - type: string - description: the user show shared buildings attribute - required: true - message: - type: string - description: error message, if any - required: false """ ok, content = self.validate_request_user(pk, request) if ok: @@ -801,32 +578,12 @@ def shared_buildings(self, request, pk=None): 'show_shared_buildings': user.show_shared_buildings, }) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) @ajax_request_class @action(detail=True, methods=['PUT']) def default_organization(self, request, pk=None): """ Sets the user's default organization - --- - parameter_strategy: replace - parameters: - - name: pk - description: User ID (primary key) - type: integer - required: true - paramType: path - - name: organization_id - description: The new default organization ID to use for this user - type: integer - required: true - type: - status: - type: string - description: success or error - required: true - message: - type: string - description: error message, if any - required: false """ ok, content = self.validate_request_user(pk, request) if ok: @@ -842,27 +599,6 @@ def default_organization(self, request, pk=None): def deactivate(self, request, pk=None): """ Deactivates a user - --- - parameter_strategy: replace - parameters: - - name: - description: User ID (primary key) - type: integer - required: true - paramType: path - - name: organization_id - description: The new default organization ID to use for this user - type: integer - required: true - type: - status: - type: string - description: success or error - required: true - message: - type: string - description: error message, if any - required: false """ try: user_id = pk From df9e9f2dac945e0a468ba2ca91f61d3e688c0ccf Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Fri, 5 Jun 2020 16:13:38 -0400 Subject: [PATCH 096/135] chore(api_schema): flake8 formatting --- seed/utils/api_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 7e53152b33..1f9415ca2f 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -111,7 +111,7 @@ def schema_factory(cls, obj, **kwargs): if type(obj) is list: if len(obj) != 1: - raise Exception(f'List types must have exactly one element to specify the schema of `items`') + raise Exception('List types must have exactly one element to specify the schema of `items`') return openapi.Schema( type=openapi.TYPE_ARRAY, items=cls.schema_factory(obj[0]) @@ -127,7 +127,7 @@ def schema_factory(cls, obj, **kwargs): }, **kwargs ) - + raise Exception(f'Unhandled type "{type(obj)}" for {obj}') def add_manual_parameters(self, parameters): From 9cef37f28a34964aa2a0caf71257595a941f4953 Mon Sep 17 00:00:00 2001 From: Adrian Lara Date: Fri, 5 Jun 2020 15:03:51 -0600 Subject: [PATCH 097/135] Test for data persistence when remapping w/ and w/o unit-aware fields --- seed/data_importer/tests/test_mapping.py | 99 ++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/seed/data_importer/tests/test_mapping.py b/seed/data_importer/tests/test_mapping.py index 070818401c..3f4f1f7754 100644 --- a/seed/data_importer/tests/test_mapping.py +++ b/seed/data_importer/tests/test_mapping.py @@ -94,6 +94,105 @@ def test_mapping(self): # for p in props: # pp(p) + def test_remapping_with_and_without_unit_aware_columns_doesnt_lose_data(self): + """ + During import, when the initial -State objects are created from the extra_data values, + ColumnMapping objects are used to take the extra_data dictionary values and create the + -State objects, setting the DB-level values as necessary - e.g. taking a raw + "Site EUI (kBtu/ft2)" value and inserting it into the DB field "site_eui". + + Previously, remapping could cause extra Column objects to be created, and subsequently, + this created extra ColumnMapping objects. These extra ColumnMapping objects could cause + raw values to be inserted into the wrong DB field on -State creation. + """ + # Just as in the previous test, build extra_data PropertyState + state = self.property_state_factory.get_property_state_as_extra_data( + import_file_id=self.import_file.id, + source_type=ASSESSED_RAW, + data_state=DATA_STATE_IMPORT, + random_extra=42, + ) + + # Replace the site_eui key-value that gets autogenerated by get_property_state_as_extra_data + del state.extra_data['site_eui'] + state.extra_data['Site EUI (kBtu/ft2)'] = 123 + state.save() + + self.import_file.raw_save_done = True + self.import_file.save() + + # Build 2 sets of mappings - with and without a unit-aware destination site_eui data + suggested_mappings = mapper.build_column_mapping( + list(state.extra_data.keys()), + Column.retrieve_all_by_tuple(self.org), + previous_mapping=get_column_mapping, + map_args=[self.org], + thresh=80 + ) + + ed_site_eui_mappings = [] + unit_aware_site_eui_mappings = [] + for raw_column, suggestion in suggested_mappings.items(): + if raw_column == 'Site EUI (kBtu/ft2)': + # Make this an extra_data field (without from_units) + ed_site_eui_mappings.append({ + "from_field": raw_column, + "from_units": None, + "to_table_name": 'PropertyState', + "to_field": raw_column, + "to_field_display_name": raw_column, + }) + + unit_aware_site_eui_mappings.append({ + "from_field": raw_column, + "from_units": 'kBtu/ft**2/year', + "to_table_name": 'PropertyState', + "to_field": 'site_eui', + "to_field_display_name": 'Site EUI', + }) + else: + other_mapping = { + "from_field": raw_column, + "from_units": None, + "to_table_name": suggestion[0], + "to_field": suggestion[1], + "to_field_display_name": suggestion[1], + } + ed_site_eui_mappings.append(other_mapping) + unit_aware_site_eui_mappings.append(other_mapping) + + # Map and remap the file multiple times with different mappings each time. + # Round 1 - Map site_eui data into Extra Data + Column.create_mappings(ed_site_eui_mappings, self.org, self.user, self.import_file.id) + tasks.map_data(self.import_file.id) + + # There should only be one raw 'Site EUI (kBtu/ft2)' Column object + self.assertEqual(1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) + # The one propertystate should have site eui info in extra_data + prop = self.import_file.find_unmatched_property_states().get() + self.assertIsNone(prop.site_eui) + self.assertIsNotNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) + + # Round 2 - Map site_eui data into the PropertyState attribute "site_eui" + Column.create_mappings(unit_aware_site_eui_mappings, self.org, self.user, self.import_file.id) + tasks.map_data(self.import_file.id, remap=True) + + self.assertEqual(1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) + # The one propertystate should have site eui info in site_eui + prop = self.import_file.find_unmatched_property_states().get() + self.assertIsNotNone(prop.site_eui) + self.assertIsNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) + + # Round 3 - Map site_eui data into Extra Data + Column.create_mappings(ed_site_eui_mappings, self.org, self.user, self.import_file.id) + tasks.map_data(self.import_file.id, remap=True) + + self.assertEqual(1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) + # The one propertystate should have site eui info in extra_data + prop = self.import_file.find_unmatched_property_states().get() + self.assertIsNone(prop.site_eui) + self.assertIsNotNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) + class TestDuplicateFileHeaders(DataMappingBaseTestCase): def setUp(self): From 669108c60086d40cb918169a93f8689afdcb02cb Mon Sep 17 00:00:00 2001 From: Alex Swindler Date: Fri, 5 Jun 2020 17:50:23 -0600 Subject: [PATCH 098/135] WAVA tweaks --- config/settings/common.py | 1 - docker/nginx-seed.conf | 35 ++++++++++++++++++- seed/landing/forms.py | 2 +- .../seed/partials/data_upload_modal.html | 2 +- seed/static/seed/partials/security.html | 6 ++-- seed/utils/nosniff.py | 17 --------- 6 files changed, 39 insertions(+), 24 deletions(-) delete mode 100644 seed/utils/nosniff.py diff --git a/config/settings/common.py b/config/settings/common.py index 4588a4455f..fcbfe20137 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -76,7 +76,6 @@ 'django.middleware.locale.LocaleMiddleware', 'django.middleware.common.BrokenLinkEmailsMiddleware', 'seed.utils.api.APIBypassCSRFMiddleware', - 'seed.utils.nosniff.DisableMIMESniffingMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', diff --git a/docker/nginx-seed.conf b/docker/nginx-seed.conf index 2e5af7db90..c906636e1a 100644 --- a/docker/nginx-seed.conf +++ b/docker/nginx-seed.conf @@ -1,3 +1,36 @@ +# https://gist.github.com/plentz/6737338 +# don't send the nginx version number in error pages and Server header +server_tokens off; + +# config to don't allow the browser to render the page inside an frame or iframe +# and avoid clickjacking http://en.wikipedia.org/wiki/Clickjacking +# if you need to allow [i]frames, you can use SAMEORIGIN or even set an uri with ALLOW-FROM uri +# https://developer.mozilla.org/en-US/docs/HTTP/X-Frame-Options +add_header X-Frame-Options SAMEORIGIN; + +# when serving user-supplied content, include a X-Content-Type-Options: nosniff header along with the Content-Type: header, +# to disable content-type sniffing on some browsers. +# https://www.owasp.org/index.php/List_of_useful_HTTP_headers +# currently suppoorted in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx +# http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx +# 'soon' on Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=471020 +add_header X-Content-Type-Options nosniff; + +# This header enables the Cross-site scripting (XSS) filter built into most recent web browsers. +# It's usually enabled by default anyway, so the role of this header is to re-enable the filter for +# this particular website if it was disabled by the user. +# https://www.owasp.org/index.php/List_of_useful_HTTP_headers +add_header X-XSS-Protection "1; mode=block"; + +# with Content Security Policy (CSP) enabled(and a browser that supports it(http://caniuse.com/#feat=contentsecuritypolicy), +# you can tell the browser that it can only download content from the domains you explicitly allow +# http://www.html5rocks.com/en/tutorials/security/content-security-policy/ +# https://www.owasp.org/index.php/Content_Security_Policy +# I need to change our application code so we can increase security by disabling 'unsafe-inline' 'unsafe-eval' +# directives for css and js(if you have inline css or js, you will need to keep it too). +# more: http://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful +add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://ssl.google-analytics.com https://assets.zendesk.com https://connect.facebook.net; img-src 'self' https://ssl.google-analytics.com https://s-static.ak.facebook.com https://assets.zendesk.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://assets.zendesk.com; font-src 'self' https://themes.googleusercontent.com; frame-src https://assets.zendesk.com https://www.facebook.com https://s-static.ak.facebook.com https://tautt.zendesk.com; object-src 'none'"; + # the upstream component nginx needs to connect to upstream seed_upsteam { server unix:///tmp/uwsgi-seed.sock; @@ -9,7 +42,7 @@ server { server_name localhost; charset utf-8; - # increase the timeouts (large files can take awhile to upload) + # increase the timeouts (large files can take a while to upload) # These are probably not needed, but increasing anyway proxy_connect_timeout 600; proxy_send_timeout 600; diff --git a/seed/landing/forms.py b/seed/landing/forms.py index 4a07f54de6..3fbde8cc24 100644 --- a/seed/landing/forms.py +++ b/seed/landing/forms.py @@ -19,7 +19,7 @@ class LoginForm(forms.Form): password = forms.CharField( label=_("Password"), widget=forms.PasswordInput( - attrs={'class': 'field', 'placeholder': _('Password')} + attrs={'class': 'field', 'placeholder': _('Password'), 'autocomplete': 'off'} ), required=True ) diff --git a/seed/static/seed/partials/data_upload_modal.html b/seed/static/seed/partials/data_upload_modal.html index f0af0ca658..7a92f69f36 100644 --- a/seed/static/seed/partials/data_upload_modal.html +++ b/seed/static/seed/partials/data_upload_modal.html @@ -270,7 +270,7 @@

PM_METER_IMPORT_RESULTS
- +
diff --git a/seed/static/seed/partials/security.html b/seed/static/seed/partials/security.html index 17a5992d18..3819eac60e 100644 --- a/seed/static/seed/partials/security.html +++ b/seed/static/seed/partials/security.html @@ -25,17 +25,17 @@

{$:: 'Change Password' | translate $}

- +

{$ error_message | translate $}

- +

{$ error_message | translate $}

- +

{$ error_message | translate $}

diff --git a/seed/utils/nosniff.py b/seed/utils/nosniff.py deleted file mode 100644 index 4762b48d62..0000000000 --- a/seed/utils/nosniff.py +++ /dev/null @@ -1,17 +0,0 @@ -# !/usr/bin/env python -# encoding: utf-8 -""" -:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA -:author -""" - - -class DisableMIMESniffingMiddleware(object): - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - response = self.get_response(request) - - response['X-Content-Type-Options'] = 'nosniff' - return response From 438b449bbd7ab0d54c1ced31f9611a9ff991e234 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 8 Jun 2020 09:58:06 -0400 Subject: [PATCH 099/135] chore(v3/organizations): rename mapping serializer --- seed/views/v3/organizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py index 7caa71b97a..29aec9ebaa 100644 --- a/seed/views/v3/organizations.py +++ b/seed/views/v3/organizations.py @@ -25,7 +25,7 @@ _log = logging.getLogger(__name__) -class MappingSerializer(serializers.Serializer): +class ImportMappingSerializer(serializers.Serializer): from_field = serializers.CharField() from_units = serializers.CharField() to_field = serializers.CharField() @@ -55,7 +55,7 @@ class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): ] } """ - mappings = serializers.ListField(child=MappingSerializer()) + mappings = serializers.ListField(child=ImportMappingSerializer()) class OrganizationViewSet(viewsets.ViewSet): From 4070f20f10348377e9b679a43f7a8113257f684a Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 8 Jun 2020 09:58:35 -0400 Subject: [PATCH 100/135] chore(v3/import_files): add swagger body for map view --- seed/views/v3/import_files.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index 06c8fafe3a..41433f3ba3 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -365,6 +365,12 @@ def has_coparent(state_id, inventory_type, fields=None): return audit_entry[0] + @swagger_auto_schema( + request_body=AutoSchemaHelper.schema_factory({ + 'remap': 'boolean', + 'mark_as_done': 'boolean', + }) + ) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') From ebd70618f9137667ae1654b6701551ef411b5b7d Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 8 Jun 2020 10:03:38 -0400 Subject: [PATCH 101/135] refactor(v3/import_files)!: rename filtered_mapping_results to mapping_results --- seed/views/v3/import_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index 41433f3ba3..9d9e6a51b4 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -233,8 +233,8 @@ def raw_column_names(self, request, pk=None): ) @api_endpoint_class @ajax_request_class - @action(detail=True, methods=['POST'], url_path='filtered_mapping_results') - def filtered_mapping_results(self, request, pk=None): + @action(detail=True, methods=['POST'], url_path='mapping_results') + def mapping_results(self, request, pk=None): """ Retrieves a paginated list of Properties and Tax Lots for an import file after mapping. """ From f1f37b9fcce6fd5faeec08c341546acf8c618d05 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 8 Jun 2020 10:10:25 -0400 Subject: [PATCH 102/135] refactor(v3/import_files): move validate_use_cases from v2 api --- seed/views/v3/import_files.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index 9d9e6a51b4..880c9cd5a5 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -27,6 +27,7 @@ map_additional_models as task_map_additional_models, match_buildings as task_match_buildings, save_raw_data as task_save_raw, + validate_use_cases as task_validate_use_cases, ) from seed.decorators import ajax_request_class from seed.lib.mappings import mapper as simple_mapper @@ -446,6 +447,24 @@ def start_data_quality_checks(self, request, pk=None): 'progress': return_value, }) + @api_endpoint_class + @ajax_request_class + @has_perm_class('can_modify_data') + @action(detail=True, methods=['POST']) + def validate_use_cases(self, request, pk=None): + """ + Starts a background task to call BuildingSync's use case validation + tool. + """ + import_file_id = pk + if not import_file_id: + return JsonResponse({ + 'status': 'error', + 'message': 'must include pk of import_file to validate' + }, status=status.HTTP_400_BAD_REQUEST) + + return task_validate_use_cases(import_file_id) + @swagger_auto_schema( request_body=AutoSchemaHelper.schema_factory({'cycle_id': 'string'}) ) From da014fea72147c787a7196f80c9230878dac8721 Mon Sep 17 00:00:00 2001 From: Alex Swindler Date: Mon, 8 Jun 2020 11:43:43 -0600 Subject: [PATCH 103/135] Improved nginx policy --- docker/nginx-seed.conf | 19 +++++++------------ docs/source/docker.rst | 2 +- seed/static/seed/js/services/main_service.js | 2 +- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/docker/nginx-seed.conf b/docker/nginx-seed.conf index c906636e1a..680a4a0bb8 100644 --- a/docker/nginx-seed.conf +++ b/docker/nginx-seed.conf @@ -1,8 +1,5 @@ # https://gist.github.com/plentz/6737338 -# don't send the nginx version number in error pages and Server header -server_tokens off; - -# config to don't allow the browser to render the page inside an frame or iframe +# config to disallow the browser to render the page inside a frame or iframe # and avoid clickjacking http://en.wikipedia.org/wiki/Clickjacking # if you need to allow [i]frames, you can use SAMEORIGIN or even set an uri with ALLOW-FROM uri # https://developer.mozilla.org/en-US/docs/HTTP/X-Frame-Options @@ -11,8 +8,8 @@ add_header X-Frame-Options SAMEORIGIN; # when serving user-supplied content, include a X-Content-Type-Options: nosniff header along with the Content-Type: header, # to disable content-type sniffing on some browsers. # https://www.owasp.org/index.php/List_of_useful_HTTP_headers -# currently suppoorted in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx -# http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx +# currently supported in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx +# https://docs.microsoft.com/en-us/previous-versions/windows/internet-explorer/ie-developer/compatibility/gg622941(v=vs.85) # 'soon' on Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=471020 add_header X-Content-Type-Options nosniff; @@ -22,14 +19,12 @@ add_header X-Content-Type-Options nosniff; # https://www.owasp.org/index.php/List_of_useful_HTTP_headers add_header X-XSS-Protection "1; mode=block"; -# with Content Security Policy (CSP) enabled(and a browser that supports it(http://caniuse.com/#feat=contentsecuritypolicy), +# with Content Security Policy (CSP) enabled (and a browser that supports it (http://caniuse.com/#feat=contentsecuritypolicy)), # you can tell the browser that it can only download content from the domains you explicitly allow -# http://www.html5rocks.com/en/tutorials/security/content-security-policy/ +# https://www.html5rocks.com/en/tutorials/security/content-security-policy/ # https://www.owasp.org/index.php/Content_Security_Policy -# I need to change our application code so we can increase security by disabling 'unsafe-inline' 'unsafe-eval' -# directives for css and js(if you have inline css or js, you will need to keep it too). -# more: http://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful -add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://ssl.google-analytics.com https://assets.zendesk.com https://connect.facebook.net; img-src 'self' https://ssl.google-analytics.com https://s-static.ak.facebook.com https://assets.zendesk.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://assets.zendesk.com; font-src 'self' https://themes.googleusercontent.com; frame-src https://assets.zendesk.com https://www.facebook.com https://s-static.ak.facebook.com https://tautt.zendesk.com; object-src 'none'"; +# https://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful +add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; img-src 'self' data: https://stamen-tiles-a.a.ssl.fastly.net https://stamen-tiles-b.a.ssl.fastly.net https://stamen-tiles-c.a.ssl.fastly.net https://stamen-tiles-d.a.ssl.fastly.net; style-src 'self' 'unsafe-inline'; frame-src 'self'; object-src 'none'"; # the upstream component nginx needs to connect to upstream seed_upsteam { diff --git a/docs/source/docker.rst b/docs/source/docker.rst index e2aa676334..697955f697 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -79,7 +79,7 @@ Ubuntu server 18.04 or newer with a m5ad.xlarge (if using in Production instance # The admin user is only valid only until the database is restored export SEED_ADMIN_USER=user@seed-platform.org - export SEED_ADMIN_PASSWORD=7FeBWal38*&k3jlfa92lakj8ih4 + export SEED_ADMIN_PASSWORD="7FeBWal38*&k3jlfa92lakj8ih4" export SEED_ADMIN_ORG=default # For SES diff --git a/seed/static/seed/js/services/main_service.js b/seed/static/seed/js/services/main_service.js index 241b6c5835..57ae150b4f 100644 --- a/seed/static/seed/js/services/main_service.js +++ b/seed/static/seed/js/services/main_service.js @@ -9,7 +9,7 @@ angular.module('BE.seed.service.main', []).factory('main_service', [ var main_factory = {}; main_factory.version = function () { - return $http.get('/api/v2/version').then(function (response) { + return $http.get('/api/v2/version/').then(function (response) { return response.data; }); }; From aa95f5b345dd68624359b1fb42c57191677d139b Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Mon, 8 Jun 2020 15:21:48 -0400 Subject: [PATCH 104/135] refactor(v3/organizations): move serializers to serializers/column_mappings.py --- seed/serializers/column_mappings.py | 35 +++++++++++++++++++++++++++++ seed/views/v3/organizations.py | 35 +---------------------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/seed/serializers/column_mappings.py b/seed/serializers/column_mappings.py index 0dfb5c9104..cbee425c22 100644 --- a/seed/serializers/column_mappings.py +++ b/seed/serializers/column_mappings.py @@ -33,3 +33,38 @@ def to_representation(self, obj): result['column_mapped'] = ColumnSerializer(obj.column_mapped.first()).data return result + + +class ImportMappingSerializer(serializers.Serializer): + from_field = serializers.CharField() + from_units = serializers.CharField() + to_field = serializers.CharField() + to_field_display_name = serializers.CharField() + to_table_name = serializers.CharField() + + +class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): + """ + Note that this is _not_ a model serializer, but used only for saving mappings + + Example: + { + "mappings": [ + { + 'from_field': 'eui', # raw field in import file + 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional + 'to_field': 'energy_use_intensity', + 'to_field_display_name': 'Energy Use Intensity', + 'to_table_name': 'PropertyState', + }, + { + 'from_field': 'gfa', + 'from_units': 'ft**2', # pint-parsable units, optional + 'to_field': 'gross_floor_area', + 'to_field_display_name': 'Gross Floor Area', + 'to_table_name': 'PropertyState', + } + ] + } + """ + mappings = serializers.ListField(child=ImportMappingSerializer()) diff --git a/seed/views/v3/organizations.py b/seed/views/v3/organizations.py index 29aec9ebaa..b5cf9292c3 100644 --- a/seed/views/v3/organizations.py +++ b/seed/views/v3/organizations.py @@ -9,7 +9,6 @@ from django.http import JsonResponse from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema -from rest_framework import serializers from rest_framework import status from rest_framework import viewsets from rest_framework.decorators import action @@ -19,45 +18,13 @@ from seed.lib.superperms.orgs.models import Organization from seed.models.columns import Column from seed.models import ImportFile +from seed.serializers.column_mappings import SaveColumnMappingsRequestPayloadSerializer from seed.utils.api import api_endpoint_class from seed.utils.api_schema import AutoSchemaHelper _log = logging.getLogger(__name__) -class ImportMappingSerializer(serializers.Serializer): - from_field = serializers.CharField() - from_units = serializers.CharField() - to_field = serializers.CharField() - to_field_display_name = serializers.CharField() - to_table_name = serializers.CharField() - - -class SaveColumnMappingsRequestPayloadSerializer(serializers.Serializer): - """ - Example: - { - "mappings": [ - { - 'from_field': 'eui', # raw field in import file - 'from_units': 'kBtu/ft**2/year', # pint-parsable units, optional - 'to_field': 'energy_use_intensity', - 'to_field_display_name': 'Energy Use Intensity', - 'to_table_name': 'PropertyState', - }, - { - 'from_field': 'gfa', - 'from_units': 'ft**2', # pint-parsable units, optional - 'to_field': 'gross_floor_area', - 'to_field_display_name': 'Gross Floor Area', - 'to_table_name': 'PropertyState', - } - ] - } - """ - mappings = serializers.ListField(child=ImportMappingSerializer()) - - class OrganizationViewSet(viewsets.ViewSet): model = Column From fd37d80dc3ae1dea8d4ed7208ec6d94b615b5e1b Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 8 Jun 2020 14:47:35 -0600 Subject: [PATCH 105/135] refactor and added labels.py in utils to DRY up code --- seed/filters.py | 4 +- seed/utils/labels.py | 32 ++++++++++++++ seed/views/v3/labels.py | 63 +++++++++------------------- seed/views/v3/properties.py | 80 +++++++++-------------------------- seed/views/v3/taxlots.py | 84 +++++++------------------------------ 5 files changed, 88 insertions(+), 175 deletions(-) create mode 100644 seed/utils/labels.py diff --git a/seed/filters.py b/seed/filters.py index 7989279c55..83f66f4143 100644 --- a/seed/filters.py +++ b/seed/filters.py @@ -51,12 +51,12 @@ class InventoryFilterBackend(filters.BaseFilterBackend): PropertyViewSet """ - def filter_queryset(self, request): + def filter_queryset(self, request, inv_type): params = request.query_params.dict() # Since this is being passed in as a query string, the object ends up # coming through as a string. params['filter_params'] = json.loads(params.get('filter_params', '{}')) - inventory_type = params.pop('inventory_type', None) + inventory_type = inv_type params = search.process_search_params( params=params, user=request.user, diff --git a/seed/utils/labels.py b/seed/utils/labels.py new file mode 100644 index 0000000000..6189bd3df2 --- /dev/null +++ b/seed/utils/labels.py @@ -0,0 +1,32 @@ +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, +through Lawrence Berkeley National Laboratory (subject to receipt of any +required approvals from the U.S. Department of Energy) and contributors. +All rights reserved. # NOQA +:author +""" +from seed.filters import ( + InventoryFilterBackend, +) +from seed.serializers.labels import ( + LabelSerializer, +) +from rest_framework import ( + response, + status, +) + + +def _get_labels(request, qs, super_organization, inv_type): + inventory = InventoryFilterBackend().filter_queryset( + request=request, inv_type=inv_type + ) + results = [ + LabelSerializer( + q, + super_organization=super_organization, + inventory=inventory + ).data for q in qs + ] + status_code = status.HTTP_200_OK + return response.Response(results, status=status_code) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 36f1e2abaa..75d007c107 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -1,5 +1,4 @@ - # !/usr/bin/env python # encoding: utf-8 """ @@ -7,11 +6,6 @@ :author 'Piper Merriam ' """ from collections import namedtuple -from django.core.exceptions import ObjectDoesNotExist -from rest_framework import ( - response, - status, -) from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer @@ -27,11 +21,22 @@ LabelSerializer, ) from seed.utils.api import drf_api_endpoint +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import _get_labels from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet ErrorState = namedtuple('ErrorState', ['status_code', 'message']) +class LabelsSchema(AutoSchemaHelper): + def __init__(self, *args): + super().__init__(*args) + + self.manual_fields = { + ('POST', 'create'): [self.org_id_field()] + } + + class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateModelViewSet): """ retrieve: @@ -49,6 +54,7 @@ class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateMo update: Update a label record. """ + swagger_schema = LabelsSchema serializer_class = LabelSerializer renderer_classes = (JSONRenderer,) parser_classes = (JSONParser, FormParser) @@ -57,56 +63,25 @@ class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateMo pagination_class = None _organization = None - def get_parent_organization(self): - org = self.get_organization() - if org.is_parent: - return org - else: - return org.parent_org - - def get_organization(self): - if self._organization is None: - try: - self._organization = self.request.user.orgs.get( - pk=self.request.query_params["organization_id"], - ) - except (KeyError, ObjectDoesNotExist): - self._organization = self.request.user.orgs.all()[0] - return self._organization - def get_queryset(self): - labels = Label.objects.filter( - super_organization=self.get_parent_organization() + super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() return labels def get_serializer(self, *args, **kwargs): - kwargs['super_organization'] = self.get_organization() + kwargs['super_organization'] = self.get_organization(self.request) inventory = InventoryFilterBackend().filter_queryset( - request=self.request, + request=self.request, inv_type=None ) kwargs['inventory'] = inventory return super().get_serializer(*args, **kwargs) - def _get_labels(self, request): - qs = self.get_queryset() - super_organization = self.get_organization() - inventory = InventoryFilterBackend().filter_queryset( - request=self.request, - ) - results = [ - LabelSerializer( - q, - super_organization=super_organization, - inventory=inventory - ).data for q in qs - ] - status_code = status.HTTP_200_OK - return response.Response(results, status=status_code) - def list(self, request): """ Returns a list of all labels """ - return self._get_labels(request) + inv_type = None + qs = self.get_queryset() + super_organization = self.get_organization(request) + return _get_labels(request, qs, super_organization, inv_type) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 55bee385d3..673397326c 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -1,17 +1,19 @@ +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" from rest_framework import viewsets -from rest_framework.decorators import action -from seed.utils.api_schema import AutoSchemaHelper from collections import namedtuple -from django.core.exceptions import ObjectDoesNotExist -from rest_framework import ( - response, - status, -) + +from rest_framework.decorators import action from rest_framework.parsers import JSONParser from rest_framework.renderers import JSONRenderer -from seed.filters import InventoryFilterBackend -from seed.models import StatusLabel as Label -from seed.serializers.labels import LabelSerializer +from seed.models import ( + StatusLabel as Label, +) +from seed.utils.api import OrgMixin +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import _get_labels ErrorState = namedtuple('ErrorState', ['status_code', 'message']) @@ -24,67 +26,25 @@ def __init__(self, *args): ('POST', 'labels'): [self.org_id_field()] } -class PropertyLabelsViewSet(viewsets.ViewSet): + +class PropertyLabelsViewSet(viewsets.ViewSet, OrgMixin): swagger_schema = PropertiesSchema renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) _organization = None - def get_parent_organization(self): - org = self.get_organization() - if org.is_parent: - return org - else: - return org.parent_org - - def get_organization(self): - if self._organization is None: - try: - self._organization = self.request.user.orgs.get( - pk=self.request.query_params["organization_id"], - ) - except (KeyError, ObjectDoesNotExist): - self._organization = self.request.user.orgs.all()[0] - return self._organization - def get_queryset(self): - labels = Label.objects.filter( - super_organization=self.get_parent_organization() + super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() return labels - def get_serializer(self, *args, **kwargs): - kwargs['super_organization'] = self.get_organization() - inventory = InventoryFilterBackend().filter_queryset( - request=self.request, - ) - kwargs['inventory'] = inventory - return super().get_serializer(*args, **kwargs) - - def _get_labels(self, request): - qs = self.get_queryset() - super_organization = self.get_organization() - inventory = InventoryFilterBackend().filter_queryset( - request=self.request, - ) - results = [ - LabelSerializer( - q, - super_organization=super_organization, - inventory=inventory - ).data for q in qs - ] - status_code = status.HTTP_200_OK - return response.Response(results, status=status_code) - @action(detail=False, methods=['POST']) def labels(self, request): """ - Api endpoint to list only labels applied to property inventory type - ___ + Returns a list of all labels """ - request.query_params._mutable = True - request.query_params['inventory_type'] = 'property_view' - request.query_params._mutable = False - return self._get_labels(request) + inv_type = 'property_view' + qs = self.get_queryset() + super_organization = self.get_organization(request) + return _get_labels(request, qs, super_organization, inv_type) diff --git a/seed/views/v3/taxlots.py b/seed/views/v3/taxlots.py index b9e5ef20a2..912d90c053 100644 --- a/seed/views/v3/taxlots.py +++ b/seed/views/v3/taxlots.py @@ -1,17 +1,17 @@ +""" +:copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA +:author +""" from rest_framework import viewsets -from rest_framework.decorators import action from collections import namedtuple -from django.core.exceptions import ObjectDoesNotExist -from rest_framework import ( - response, - status, -) + +from rest_framework.decorators import action from rest_framework.parsers import JSONParser from rest_framework.renderers import JSONRenderer -from seed.filters import InventoryFilterBackend from seed.models import StatusLabel as Label -from seed.serializers.labels import LabelSerializer +from seed.utils.api import OrgMixin from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import _get_labels ErrorState = namedtuple('ErrorState', ['status_code', 'message']) @@ -25,78 +25,24 @@ def __init__(self, *args): } -class TaxlotLabelsViewSet(viewsets.ViewSet): +class TaxlotLabelsViewSet(viewsets.ViewSet, OrgMixin): swagger_schema = TaxlotsSchema renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) _organization = None - errors = { - 'disjoint': ErrorState( - status.HTTP_422_UNPROCESSABLE_ENTITY, - 'add_label_ids and remove_label_ids cannot contain elements in common' - ), - 'missing_org': ErrorState( - status.HTTP_422_UNPROCESSABLE_ENTITY, - 'missing organization_id' - ) - } - - def get_parent_organization(self): - org = self.get_organization() - if org.is_parent: - return org - else: - return org.parent_org - - def get_organization(self): - if self._organization is None: - try: - self._organization = self.request.user.orgs.get( - pk=self.request.query_params["organization_id"], - ) - except (KeyError, ObjectDoesNotExist): - self._organization = self.request.user.orgs.all()[0] - return self._organization - def get_queryset(self): - labels = Label.objects.filter( - super_organization=self.get_parent_organization() + super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() return labels - def get_serializer(self, *args, **kwargs): - kwargs['super_organization'] = self.get_organization() - inventory = InventoryFilterBackend().filter_queryset( - request=self.request, - ) - kwargs['inventory'] = inventory - return super().get_serializer(*args, **kwargs) - - def _get_labels(self, request): - qs = self.get_queryset() - super_organization = self.get_organization() - inventory = InventoryFilterBackend().filter_queryset( - request=self.request, - ) - results = [ - LabelSerializer( - q, - super_organization=super_organization, - inventory=inventory - ).data for q in qs - ] - status_code = status.HTTP_200_OK - return response.Response(results, status=status_code) - @action(detail=False, methods=['POST']) def labels(self, request): """ - Api endpoint to list only labels applied to taxlot inventory type - ___ + Returns a list of all labels """ - request.query_params._mutable = True - request.query_params['inventory_type'] = 'taxlot_view' - request.query_params._mutable = False - return self._get_labels(request) + inv_type = 'taxlot_view' + qs = self.get_queryset() + super_organization = self.get_organization(request) + return _get_labels(request, qs, super_organization, inv_type) From eea600a069d3a5b4aae4398064c0f758c7fbd318 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 8 Jun 2020 17:37:24 -0600 Subject: [PATCH 106/135] created a new class as to not change the functionality of api v2 --- seed/filters.py | 30 ++++++++++++++++++++++++++++++ seed/utils/labels.py | 4 ++-- seed/views/v3/labels.py | 6 +++--- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/seed/filters.py b/seed/filters.py index 83f66f4143..88aad861c1 100644 --- a/seed/filters.py +++ b/seed/filters.py @@ -51,6 +51,36 @@ class InventoryFilterBackend(filters.BaseFilterBackend): PropertyViewSet """ + def filter_queryset(self, request, inv_type): + params = request.query_params.dict() + # Since this is being passed in as a query string, the object ends up + # coming through as a string. + params['filter_params'] = json.loads(params.get('filter_params', '{}')) + inventory_type = params.pop('inventory_type', None) + params = search.process_search_params( + params=params, + user=request.user, + is_api_request=True, + ) + queryset = search.inventory_search_filter_sort( + inventory_type, + params=params, + user=request.user, + ) + if 'selected' in request.data: + # Return labels limited to the 'selected' list. Otherwise, if selected is empty, return all + if request.data['selected']: + return queryset.filter( + id__in=request.data['selected'], + ) + return queryset + + +class InventoryFilterBackendWithInvType(filters.BaseFilterBackend): + """ + A permutation of the InventoryFilterBackend class that uses inventory type as a method parameter + """ + def filter_queryset(self, request, inv_type): params = request.query_params.dict() # Since this is being passed in as a query string, the object ends up diff --git a/seed/utils/labels.py b/seed/utils/labels.py index 6189bd3df2..65c82460f9 100644 --- a/seed/utils/labels.py +++ b/seed/utils/labels.py @@ -6,7 +6,7 @@ :author """ from seed.filters import ( - InventoryFilterBackend, + InventoryFilterBackendWithInvType, ) from seed.serializers.labels import ( LabelSerializer, @@ -18,7 +18,7 @@ def _get_labels(request, qs, super_organization, inv_type): - inventory = InventoryFilterBackend().filter_queryset( + inventory = InventoryFilterBackendWithInvType().filter_queryset( request=request, inv_type=inv_type ) results = [ diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 75d007c107..7f566db842 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -12,7 +12,7 @@ from seed.decorators import DecoratorMixin from seed.filters import ( LabelFilterBackend, - InventoryFilterBackend, + InventoryFilterBackendWithInvType, ) from seed.models import ( StatusLabel as Label, @@ -33,7 +33,7 @@ def __init__(self, *args): super().__init__(*args) self.manual_fields = { - ('POST', 'create'): [self.org_id_field()] + ('GET', 'list'): [self.org_id_field()] } @@ -71,7 +71,7 @@ def get_queryset(self): def get_serializer(self, *args, **kwargs): kwargs['super_organization'] = self.get_organization(self.request) - inventory = InventoryFilterBackend().filter_queryset( + inventory = InventoryFilterBackendWithInvType().filter_queryset( request=self.request, inv_type=None ) kwargs['inventory'] = inventory From fa8fd280c30c3cf9fd373c05f1c4cb60321973b0 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 8 Jun 2020 19:25:38 -0600 Subject: [PATCH 107/135] removed extra method param accidently added to v2 --- seed/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/filters.py b/seed/filters.py index 88aad861c1..c834e3fa1c 100644 --- a/seed/filters.py +++ b/seed/filters.py @@ -51,7 +51,7 @@ class InventoryFilterBackend(filters.BaseFilterBackend): PropertyViewSet """ - def filter_queryset(self, request, inv_type): + def filter_queryset(self, request): params = request.query_params.dict() # Since this is being passed in as a query string, the object ends up # coming through as a string. From acd63ab9c60e5ad26a8222f821f82fc499b32b1f Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 8 Jun 2020 19:56:26 -0600 Subject: [PATCH 108/135] renamed new filter method to not cause collisions --- seed/filters.py | 2 +- seed/utils/labels.py | 2 +- seed/views/v3/labels.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seed/filters.py b/seed/filters.py index c834e3fa1c..0bb2d0c012 100644 --- a/seed/filters.py +++ b/seed/filters.py @@ -81,7 +81,7 @@ class InventoryFilterBackendWithInvType(filters.BaseFilterBackend): A permutation of the InventoryFilterBackend class that uses inventory type as a method parameter """ - def filter_queryset(self, request, inv_type): + def filter_queryset_with_inv(self, request, inv_type): params = request.query_params.dict() # Since this is being passed in as a query string, the object ends up # coming through as a string. diff --git a/seed/utils/labels.py b/seed/utils/labels.py index 65c82460f9..56317ca0b1 100644 --- a/seed/utils/labels.py +++ b/seed/utils/labels.py @@ -18,7 +18,7 @@ def _get_labels(request, qs, super_organization, inv_type): - inventory = InventoryFilterBackendWithInvType().filter_queryset( + inventory = InventoryFilterBackendWithInvType().filter_queryset_with_inv( request=request, inv_type=inv_type ) results = [ diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 7f566db842..3869eb16ee 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -71,7 +71,7 @@ def get_queryset(self): def get_serializer(self, *args, **kwargs): kwargs['super_organization'] = self.get_organization(self.request) - inventory = InventoryFilterBackendWithInvType().filter_queryset( + inventory = InventoryFilterBackendWithInvType().filter_queryset_with_inv( request=self.request, inv_type=None ) kwargs['inventory'] = inventory From 66fada6122b797b36cf7cc1aeb707d79bf580f8a Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Tue, 9 Jun 2020 07:55:39 -0600 Subject: [PATCH 109/135] better descriptions for label endpoints --- seed/views/v3/properties.py | 3 ++- seed/views/v3/taxlots.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 673397326c..2ecfba0f5a 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -42,7 +42,8 @@ def get_queryset(self): @action(detail=False, methods=['POST']) def labels(self, request): """ - Returns a list of all labels + Returns a list of all labels where the is_applied field + in the response pertains to the labels applied to property_view """ inv_type = 'property_view' qs = self.get_queryset() diff --git a/seed/views/v3/taxlots.py b/seed/views/v3/taxlots.py index 912d90c053..2cfdd9ca48 100644 --- a/seed/views/v3/taxlots.py +++ b/seed/views/v3/taxlots.py @@ -40,7 +40,8 @@ def get_queryset(self): @action(detail=False, methods=['POST']) def labels(self, request): """ - Returns a list of all labels + Returns a list of all labels where the is_applied field + in the response pertains to the labels applied to taxlot_view """ inv_type = 'taxlot_view' qs = self.get_queryset() From 41915b480c5d969d8927e5b08e10640d2f4ab72b Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 13:45:12 -0600 Subject: [PATCH 110/135] refactor(api_schema): get rid of class' array param formats --- seed/utils/api_schema.py | 8 -------- seed/views/v3/data_quality.py | 4 ++-- seed/views/v3/users.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 1f9415ca2f..a56d189add 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -10,16 +10,8 @@ class AutoSchemaHelper(SwaggerAutoSchema): # Used to easily build out example values displayed on Swagger page. body_parameter_formats = { - 'integer_array': openapi.Schema( - type=openapi.TYPE_ARRAY, - items=openapi.Schema(type=openapi.TYPE_INTEGER) - ), 'string': openapi.Schema(type=openapi.TYPE_STRING), 'boolean': openapi.Schema(type=openapi.TYPE_BOOLEAN), - 'string_array': openapi.Schema( - type=openapi.TYPE_ARRAY, - items=openapi.Schema(type=openapi.TYPE_STRING) - ), 'integer': openapi.Schema(type=openapi.TYPE_INTEGER), } diff --git a/seed/views/v3/data_quality.py b/seed/views/v3/data_quality.py index d8ba265bfc..1207fa43fd 100644 --- a/seed/views/v3/data_quality.py +++ b/seed/views/v3/data_quality.py @@ -131,8 +131,8 @@ class DataQualityViews(viewsets.ViewSet): ], request_body=AutoSchemaHelper.schema_factory( { - 'property_state_ids': 'integer_array', - 'taxlot_state_ids': 'integer_array', + 'property_state_ids': ['integer'], + 'taxlot_state_ids': ['integer'], }, description='An object containing IDs of the records to perform' ' data quality checks on. Should contain two keys- ' diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index bf9db1c9b7..9247672147 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -453,7 +453,7 @@ def get_actions(self, request): manual_parameters=[AutoSchemaHelper.org_id_field()], request_body=AutoSchemaHelper.schema_factory( { - 'actions': 'string_array', + 'actions': ['string'], }, description='A list of actions to check: examples include (requires_parent_org_owner, ' 'requires_owner, requires_member, requires_viewer, ' From e708c54a23e15b5ab9e7be53e2949723931a2b55 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 13:47:53 -0600 Subject: [PATCH 111/135] refactor(api_schema): rename org_id_field to query_org_id_field --- seed/utils/api_schema.py | 2 +- seed/views/v3/columns.py | 6 +++--- seed/views/v3/cycles.py | 2 +- seed/views/v3/data_quality.py | 12 ++++++------ seed/views/v3/datasets.py | 12 ++++++------ seed/views/v3/import_files.py | 8 ++++---- seed/views/v3/users.py | 8 ++++---- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index a56d189add..9b4acd1fbe 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -30,7 +30,7 @@ def base_field(name, location_attr, description, required, type): ) @staticmethod - def org_id_field(required=True): + def query_org_id_field(required=True): return openapi.Parameter( 'organization_id', openapi.IN_QUERY, diff --git a/seed/views/v3/columns.py b/seed/views/v3/columns.py index d04bd11a88..6c40053303 100644 --- a/seed/views/v3/columns.py +++ b/seed/views/v3/columns.py @@ -24,7 +24,7 @@ org_param_swagger_decorator = swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()] + manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) @@ -63,7 +63,7 @@ def get_queryset(self): @swagger_auto_schema( manual_parameters=[ - AutoSchemaHelper.org_id_field(required=False), + AutoSchemaHelper.query_org_id_field(required=False), AutoSchemaHelper.query_string_field( name='inventory_type', required=False, @@ -96,7 +96,7 @@ def list(self, request): 'columns': columns, }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @ajax_request_class def retrieve(self, request, pk=None): """ diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 487e9aa022..4b7f3c5865 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -19,7 +19,7 @@ # adds organization_id query param to a view org_param_swagger_decorator = swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()] + manual_parameters=[AutoSchemaHelper.query_org_id_field()] ) diff --git a/seed/views/v3/data_quality.py b/seed/views/v3/data_quality.py index 1207fa43fd..7912215943 100644 --- a/seed/views/v3/data_quality.py +++ b/seed/views/v3/data_quality.py @@ -127,7 +127,7 @@ class DataQualityViews(viewsets.ViewSet): @swagger_auto_schema( manual_parameters=[ - AutoSchemaHelper.org_id_field(), + AutoSchemaHelper.query_org_id_field(), ], request_body=AutoSchemaHelper.schema_factory( { @@ -218,7 +218,7 @@ def csv(self, request, pk): return response @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], responses={ 200: DataQualityRulesResponseSerializer } @@ -263,7 +263,7 @@ def data_quality_rules(self, request): return JsonResponse(result) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], responses={ 200: DataQualityRulesResponseSerializer } @@ -283,7 +283,7 @@ def reset_all_data_quality_rules(self, request): return self.data_quality_rules(request) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], responses={ 200: DataQualityRulesResponseSerializer } @@ -303,7 +303,7 @@ def reset_default_data_quality_rules(self, request): return self.data_quality_rules(request) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=SaveDataQualityRulesPayloadSerializer, responses={ 200: DataQualityRulesResponseSerializer @@ -389,7 +389,7 @@ def save_data_quality_rules(self, request, pk=None): @swagger_auto_schema( manual_parameters=[ - AutoSchemaHelper.org_id_field(), + AutoSchemaHelper.query_org_id_field(), AutoSchemaHelper.query_integer_field( name='data_quality_id', required=True, diff --git a/seed/views/v3/datasets.py b/seed/views/v3/datasets.py index a402ee8122..2fcf1e8c8d 100644 --- a/seed/views/v3/datasets.py +++ b/seed/views/v3/datasets.py @@ -29,7 +29,7 @@ class DatasetViewSet(viewsets.ViewSet): raise_exception = True - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()],) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()],) @require_organization_id_class @api_endpoint_class @ajax_request_class @@ -56,7 +56,7 @@ def list(self, request): }) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( {'dataset': 'string'}, description='The new name for this dataset' @@ -98,7 +98,7 @@ def update(self, request, pk=None): 'status': 'success', }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): @@ -161,7 +161,7 @@ def retrieve(self, request, pk=None): 'dataset': dataset, }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -185,7 +185,7 @@ def destroy(self, request, pk=None): return JsonResponse({'status': 'success'}) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( {'name': 'string'}, description='Name of the dataset to be created' @@ -227,7 +227,7 @@ def create(self, request): return JsonResponse({'status': 'success', 'id': record.pk, 'name': record.name}) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], responses={ 200: AutoSchemaHelper.schema_factory({ 'datasets_count': 'integer', diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index 880c9cd5a5..f30e2e4c68 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -226,7 +226,7 @@ def raw_column_names(self, request, pk=None): @swagger_auto_schema( manual_parameters=[ - AutoSchemaHelper.org_id_field(), + AutoSchemaHelper.query_org_id_field(), ], responses={ 200: MappingResultsResponseSerializer @@ -428,7 +428,7 @@ def start_system_matching_and_geocoding(self, request, pk=None): return task_match_buildings(pk) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @@ -692,7 +692,7 @@ def matching_and_geocoding_results(self, request, pk=None): } } - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -779,7 +779,7 @@ def mapping_suggestions(self, request, pk): return JsonResponse(result) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @api_endpoint_class @ajax_request_class @permission_classes((SEEDOrgPermissions,)) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 9247672147..6972892c39 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -138,7 +138,7 @@ def validate_request_user(self, pk, request): return True, user @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory({ 'first_name': 'string', 'last_name': 'string', @@ -255,7 +255,7 @@ def current(self, request): return JsonResponse({'pk': request.user.id}) @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'role': 'string', @@ -450,7 +450,7 @@ def get_actions(self, request): } @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.org_id_field()], + manual_parameters=[AutoSchemaHelper.query_org_id_field()], request_body=AutoSchemaHelper.schema_factory( { 'actions': ['string'], @@ -578,7 +578,7 @@ def shared_buildings(self, request, pk=None): 'show_shared_buildings': user.show_shared_buildings, }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.org_id_field()]) + @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @ajax_request_class @action(detail=True, methods=['PUT']) def default_organization(self, request, pk=None): From 0890a2f539fbf13b3b57559074b370b11598a122 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 13:53:21 -0600 Subject: [PATCH 112/135] chore(api_schema): add better docs to schema_factory --- seed/utils/api_schema.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 9b4acd1fbe..68c06960e3 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -93,6 +93,17 @@ def body_field(cls, required, description, name='body', params_to_formats={}): def schema_factory(cls, obj, **kwargs): """Translates an object into an openapi Schema + This can handle nested objects (lists, dicts), and "types" defined as strings + For example: + { + 'hello': ['string'], + 'world': 'integer', + 'fruits': { + 'orange': 'boolean', + 'apple': 'boolean' + } + } + :param obj: str, list, dict[str, obj] :return: drf_yasg.openapi.Schema """ From 62f6051f4c941114cb4114dd12133279140639b7 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 14:00:20 -0600 Subject: [PATCH 113/135] refactor(api_schema): allow kwargs for list and primitive types in schema_factory --- seed/utils/api_schema.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 68c06960e3..706a6c0bfb 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -9,10 +9,10 @@ class AutoSchemaHelper(SwaggerAutoSchema): overwrite_params = [] # Used to easily build out example values displayed on Swagger page. - body_parameter_formats = { - 'string': openapi.Schema(type=openapi.TYPE_STRING), - 'boolean': openapi.Schema(type=openapi.TYPE_BOOLEAN), - 'integer': openapi.Schema(type=openapi.TYPE_INTEGER), + openapi_primitives = { + 'string': openapi.TYPE_STRING, + 'boolean': openapi.TYPE_BOOLEAN, + 'integer': openapi.TYPE_INTEGER, } @staticmethod @@ -108,16 +108,20 @@ def schema_factory(cls, obj, **kwargs): :return: drf_yasg.openapi.Schema """ if type(obj) is str: - if obj not in cls.body_parameter_formats: - raise Exception(f'Invalid type "{obj}"; expected one of {cls.body_parameter_formats.keys()}') - return cls.body_parameter_formats[obj] + if obj not in cls.openapi_primitives: + raise Exception(f'Invalid type "{obj}"; expected one of {cls.openapi_primitives.keys()}') + return openapi.Schema( + type=cls.openapi_primitives[obj], + **kwargs + ) if type(obj) is list: if len(obj) != 1: raise Exception('List types must have exactly one element to specify the schema of `items`') return openapi.Schema( type=openapi.TYPE_ARRAY, - items=cls.schema_factory(obj[0]) + items=cls.schema_factory(obj[0]), + **kwargs ) if type(obj) is dict: From 260ea25abe7b97450fb7569e710e4eecf3a2a7f1 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 14:51:19 -0600 Subject: [PATCH 114/135] chore(v3/users): update swagger docs for create endpoint --- seed/views/v3/users.py | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 6972892c39..c37aa6f5c9 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -11,6 +11,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.http import JsonResponse +from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets, status, serializers from rest_framework.decorators import action @@ -139,12 +140,41 @@ def validate_request_user(self, pk, request): @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], - request_body=AutoSchemaHelper.schema_factory({ - 'first_name': 'string', - 'last_name': 'string', - 'role': 'string', - 'email': 'string', - }) + request_body=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + 'org_name': AutoSchemaHelper.schema_factory( + 'string', description='New organization name if creating a new organization for this user'), + 'first_name': AutoSchemaHelper.schema_factory( + 'string', description='First name of new user'), + 'last_name': AutoSchemaHelper.schema_factory( + 'string', description='Last name of new user'), + 'role': AutoSchemaHelper.schema_factory( + 'string', description='one of owner, member, or viewer'), + 'email': AutoSchemaHelper.schema_factory( + 'string', description='Email address of the new user'), + }, + required=['first_name', 'last_name', 'role', 'email'], + ), + responses={ + 200: openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + 'status': AutoSchemaHelper.schema_factory( + 'string', description='success or error'), + 'message': AutoSchemaHelper.schema_factory( + 'string', description='email address of new user'), + 'org': AutoSchemaHelper.schema_factory( + 'string', description='name of new org (or existing org)'), + 'org_created': AutoSchemaHelper.schema_factory( + 'boolean', description='true if new org created'), + 'username': AutoSchemaHelper.schema_factory( + 'string', description='username of new user'), + 'user_id': AutoSchemaHelper.schema_factory( + 'string', description='user id (pk) of new user'), + } + ) + } ) @api_endpoint_class @ajax_request_class From 17e5067a217d4cd70d4712e4a4a485a0f0a48e7e Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 15:44:40 -0600 Subject: [PATCH 115/135] refactor(views/v3): create shared decorator for org query param --- seed/utils/api_schema.py | 7 +++++++ seed/views/v3/columns.py | 15 +++++---------- seed/views/v3/cycles.py | 18 ++++++------------ seed/views/v3/datasets.py | 6 +++--- seed/views/v3/import_files.py | 8 ++++---- seed/views/v3/users.py | 4 ++-- 6 files changed, 27 insertions(+), 31 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 706a6c0bfb..8c510d0ce4 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -2,6 +2,7 @@ # encoding: utf-8 from drf_yasg.inspectors import SwaggerAutoSchema from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema class AutoSchemaHelper(SwaggerAutoSchema): @@ -144,3 +145,9 @@ def add_manual_parameters(self, parameters): return manual_params # I think this should add to existing parameters, but haven't been able to confirm. return parameters + manual_params + + +# this is a commonly used swagger decorator so moved here for DRYness +swagger_auto_schema_org_query_param = swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field()] +) diff --git a/seed/views/v3/columns.py b/seed/views/v3/columns.py index 6c40053303..d41f9e22f7 100644 --- a/seed/views/v3/columns.py +++ b/seed/views/v3/columns.py @@ -18,27 +18,22 @@ from seed.serializers.columns import ColumnSerializer from seed.utils.api import OrgValidateMixin, OrgCreateUpdateMixin from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param _log = logging.getLogger(__name__) -org_param_swagger_decorator = swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field()] -) - - @method_decorator( name='create', - decorator=org_param_swagger_decorator + decorator=swagger_auto_schema_org_query_param ) @method_decorator( name='update', - decorator=org_param_swagger_decorator + decorator=swagger_auto_schema_org_query_param ) @method_decorator( name='destroy', - decorator=org_param_swagger_decorator + decorator=swagger_auto_schema_org_query_param ) class ColumnViewSet(OrgValidateMixin, SEEDOrgNoPatchOrOrgCreateModelViewSet, OrgCreateUpdateMixin): """ @@ -96,7 +91,7 @@ def list(self, request): 'columns': columns, }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @ajax_request_class def retrieve(self, request, pk=None): """ diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index 4b7f3c5865..c7eb1674aa 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -14,30 +14,24 @@ from seed.serializers.cycles import CycleSerializer from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet -from seed.utils.api_schema import AutoSchemaHelper - - -# adds organization_id query param to a view -org_param_swagger_decorator = swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field()] -) +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param @method_decorator( name='list', - decorator=org_param_swagger_decorator) + decorator=swagger_auto_schema_org_query_param) @method_decorator( name='create', - decorator=org_param_swagger_decorator) + decorator=swagger_auto_schema_org_query_param) @method_decorator( name='retrieve', - decorator=org_param_swagger_decorator) + decorator=swagger_auto_schema_org_query_param) @method_decorator( name='update', - decorator=org_param_swagger_decorator) + decorator=swagger_auto_schema_org_query_param) @method_decorator( name='destroy', - decorator=org_param_swagger_decorator) + decorator=swagger_auto_schema_org_query_param) class CycleViewSet(SEEDOrgNoPatchOrOrgCreateModelViewSet): """API endpoint for viewing and creating cycles (time periods). diff --git a/seed/views/v3/datasets.py b/seed/views/v3/datasets.py index 2fcf1e8c8d..124908f553 100644 --- a/seed/views/v3/datasets.py +++ b/seed/views/v3/datasets.py @@ -21,7 +21,7 @@ from seed.utils.api import api_endpoint_class from seed.utils.time import convert_to_js_timestamp -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param _log = logging.getLogger(__name__) @@ -98,7 +98,7 @@ def update(self, request, pk=None): 'status': 'success', }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class def retrieve(self, request, pk=None): @@ -161,7 +161,7 @@ def retrieve(self, request, pk=None): 'dataset': dataset, }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') diff --git a/seed/views/v3/import_files.py b/seed/views/v3/import_files.py index f30e2e4c68..6a08948ac4 100644 --- a/seed/views/v3/import_files.py +++ b/seed/views/v3/import_files.py @@ -59,7 +59,7 @@ TaxLotProperty, ) from seed.utils.api import api_endpoint_class -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param from seed.utils.geocode import MapQuestAPIKeyError _log = logging.getLogger(__name__) @@ -428,7 +428,7 @@ def start_system_matching_and_geocoding(self, request, pk=None): return task_match_buildings(pk) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('can_modify_data') @@ -692,7 +692,7 @@ def matching_and_geocoding_results(self, request, pk=None): } } - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @has_perm_class('requires_member') @@ -779,7 +779,7 @@ def mapping_suggestions(self, request, pk): return JsonResponse(result) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @api_endpoint_class @ajax_request_class @permission_classes((SEEDOrgPermissions,)) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index c37aa6f5c9..ed70ca1ff1 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -32,7 +32,7 @@ invite_to_seed, ) from seed.utils.api import api_endpoint_class -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param from seed.utils.organizations import create_organization from rest_framework.status import HTTP_400_BAD_REQUEST @@ -608,7 +608,7 @@ def shared_buildings(self, request, pk=None): 'show_shared_buildings': user.show_shared_buildings, }) - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) + @swagger_auto_schema_org_query_param @ajax_request_class @action(detail=True, methods=['PUT']) def default_organization(self, request, pk=None): From 9626c7d4cad408d6bff7d3f154e27cae5f6e8163 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 15:52:24 -0600 Subject: [PATCH 116/135] chore(v3/users): add swagger description to set_password --- seed/views/v3/users.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index ed70ca1ff1..475486426c 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -434,11 +434,13 @@ def update(self, request, pk=None): }) @swagger_auto_schema( - request_body=AutoSchemaHelper.schema_factory({ - 'current_password': 'string', - 'password_1': 'string', - 'password_2': 'string', - }), + request_body=AutoSchemaHelper.schema_factory( + { + 'current_password': 'string', + 'password_1': 'string', + 'password_2': 'string', + }, + description='Fill in the current and new matching passwords'), ) @ajax_request_class @action(detail=True, methods=['PUT']) From aad4b408b3fd291eec2e07a72150e721c8d2e131 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 15:59:32 -0600 Subject: [PATCH 117/135] chore: flake8 --- seed/views/v3/cycles.py | 3 +-- seed/views/v3/users.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/seed/views/v3/cycles.py b/seed/views/v3/cycles.py index c7eb1674aa..c25f965c7a 100644 --- a/seed/views/v3/cycles.py +++ b/seed/views/v3/cycles.py @@ -8,13 +8,12 @@ :authors Paul Munday Fable Turas """ from django.utils.decorators import method_decorator -from drf_yasg.utils import swagger_auto_schema from seed.models import Cycle from seed.serializers.cycles import CycleSerializer from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet -from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param +from seed.utils.api_schema import swagger_auto_schema_org_query_param @method_decorator( diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 475486426c..178f7819da 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -440,7 +440,7 @@ def update(self, request, pk=None): 'password_1': 'string', 'password_2': 'string', }, - description='Fill in the current and new matching passwords'), + description='Fill in the current and new matching passwords'), ) @ajax_request_class @action(detail=True, methods=['PUT']) From 5c42ccf53f3b751e069f9a89e4325c5bf408f825 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 16:04:34 -0600 Subject: [PATCH 118/135] refactor(api_schema): rename openapi_primitives to _types --- seed/utils/api_schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 8c510d0ce4..4c8fadea3e 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -10,7 +10,7 @@ class AutoSchemaHelper(SwaggerAutoSchema): overwrite_params = [] # Used to easily build out example values displayed on Swagger page. - openapi_primitives = { + openapi_types = { 'string': openapi.TYPE_STRING, 'boolean': openapi.TYPE_BOOLEAN, 'integer': openapi.TYPE_INTEGER, @@ -109,10 +109,10 @@ def schema_factory(cls, obj, **kwargs): :return: drf_yasg.openapi.Schema """ if type(obj) is str: - if obj not in cls.openapi_primitives: - raise Exception(f'Invalid type "{obj}"; expected one of {cls.openapi_primitives.keys()}') + if obj not in cls.openapi_types: + raise Exception(f'Invalid type "{obj}"; expected one of {cls.openapi_types.keys()}') return openapi.Schema( - type=cls.openapi_primitives[obj], + type=cls.openapi_types[obj], **kwargs ) From 2db61d6afa0cbac35d784652b0b0eab853a8bdd8 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 20:01:52 -0400 Subject: [PATCH 119/135] chore(v3/datasets): update list view decorator Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com> --- seed/views/v3/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/views/v3/datasets.py b/seed/views/v3/datasets.py index 124908f553..306250c285 100644 --- a/seed/views/v3/datasets.py +++ b/seed/views/v3/datasets.py @@ -29,7 +29,7 @@ class DatasetViewSet(viewsets.ViewSet): raise_exception = True - @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()],) + @swagger_auto_schema_org_query_param @require_organization_id_class @api_endpoint_class @ajax_request_class From 26d36e0c70d899122146b53293680ac1bd052895 Mon Sep 17 00:00:00 2001 From: Ted Summer Date: Tue, 9 Jun 2020 18:17:55 -0600 Subject: [PATCH 120/135] chore(v3/users): simplify create view swagger docs --- seed/views/v3/users.py | 57 ++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/seed/views/v3/users.py b/seed/views/v3/users.py index 178f7819da..106a02eefa 100644 --- a/seed/views/v3/users.py +++ b/seed/views/v3/users.py @@ -11,7 +11,6 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.http import JsonResponse -from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets, status, serializers from rest_framework.decorators import action @@ -140,39 +139,37 @@ def validate_request_user(self, pk, request): @swagger_auto_schema( manual_parameters=[AutoSchemaHelper.query_org_id_field()], - request_body=openapi.Schema( - type=openapi.TYPE_OBJECT, - properties={ - 'org_name': AutoSchemaHelper.schema_factory( - 'string', description='New organization name if creating a new organization for this user'), - 'first_name': AutoSchemaHelper.schema_factory( - 'string', description='First name of new user'), - 'last_name': AutoSchemaHelper.schema_factory( - 'string', description='Last name of new user'), - 'role': AutoSchemaHelper.schema_factory( - 'string', description='one of owner, member, or viewer'), - 'email': AutoSchemaHelper.schema_factory( - 'string', description='Email address of the new user'), + request_body=AutoSchemaHelper.schema_factory( + { + 'org_name': 'string', + 'first_name': 'string', + 'last_name': 'string', + 'role': 'string', + 'email': 'string', }, required=['first_name', 'last_name', 'role', 'email'], + description='An object containing meta data for a new user:\n' + '-org_name: New organization name if creating a new organization for this user\n' + '-first_name: First name of new user\n' + '-last_name: Last name of new user\n' + '-role: one of owner, member, or viewer\n' + '-email: Email address of the new user' ), responses={ - 200: openapi.Schema( - type=openapi.TYPE_OBJECT, - properties={ - 'status': AutoSchemaHelper.schema_factory( - 'string', description='success or error'), - 'message': AutoSchemaHelper.schema_factory( - 'string', description='email address of new user'), - 'org': AutoSchemaHelper.schema_factory( - 'string', description='name of new org (or existing org)'), - 'org_created': AutoSchemaHelper.schema_factory( - 'boolean', description='true if new org created'), - 'username': AutoSchemaHelper.schema_factory( - 'string', description='username of new user'), - 'user_id': AutoSchemaHelper.schema_factory( - 'string', description='user id (pk) of new user'), - } + 200: AutoSchemaHelper.schema_factory( + { + 'status': 'string', + 'message': 'string', + 'org': 'string', + 'org_created': 'boolean', + 'username': 'string', + 'user_id': 'string', + }, + description='Properties:\n' + '-org: name of new org (or existing org)\n' + '-org_created: true if new org created\n' + '-username: username of new user\n' + '-user_id: user id (pk) of new user' ) } ) From 1883f97e01b8e081b670c75a89e23d4403236f6e Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 10 Jun 2020 11:10:26 -0600 Subject: [PATCH 121/135] updated with new apiv3 autoschema helper pattern --- seed/views/v3/labels.py | 18 +++++++++--------- seed/views/v3/properties.py | 14 ++------------ seed/views/v3/taxlots.py | 13 ++----------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 3869eb16ee..593a846723 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -21,20 +21,20 @@ LabelSerializer, ) from seed.utils.api import drf_api_endpoint -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import swagger_auto_schema_org_query_param from seed.utils.labels import _get_labels from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet ErrorState = namedtuple('ErrorState', ['status_code', 'message']) -class LabelsSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('GET', 'list'): [self.org_id_field()] - } +# class LabelsSchema(AutoSchemaHelper): +# def __init__(self, *args): +# super().__init__(*args) +# +# self.manual_fields = { +# ('GET', 'list'): [self.org_id_field()] +# } class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateModelViewSet): @@ -54,7 +54,6 @@ class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateMo update: Update a label record. """ - swagger_schema = LabelsSchema serializer_class = LabelSerializer renderer_classes = (JSONRenderer,) parser_classes = (JSONParser, FormParser) @@ -77,6 +76,7 @@ def get_serializer(self, *args, **kwargs): kwargs['inventory'] = inventory return super().get_serializer(*args, **kwargs) + @swagger_auto_schema_org_query_param def list(self, request): """ Returns a list of all labels diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 2ecfba0f5a..91e43a17aa 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -12,23 +12,12 @@ StatusLabel as Label, ) from seed.utils.api import OrgMixin -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import swagger_auto_schema_org_query_param from seed.utils.labels import _get_labels ErrorState = namedtuple('ErrorState', ['status_code', 'message']) - -class PropertiesSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('POST', 'labels'): [self.org_id_field()] - } - - class PropertyLabelsViewSet(viewsets.ViewSet, OrgMixin): - swagger_schema = PropertiesSchema renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) _organization = None @@ -39,6 +28,7 @@ def get_queryset(self): ).order_by("name").distinct() return labels + @swagger_auto_schema_org_query_param @action(detail=False, methods=['POST']) def labels(self, request): """ diff --git a/seed/views/v3/taxlots.py b/seed/views/v3/taxlots.py index 2cfdd9ca48..1d158fc4ca 100644 --- a/seed/views/v3/taxlots.py +++ b/seed/views/v3/taxlots.py @@ -10,23 +10,13 @@ from rest_framework.renderers import JSONRenderer from seed.models import StatusLabel as Label from seed.utils.api import OrgMixin -from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.api_schema import swagger_auto_schema_org_query_param from seed.utils.labels import _get_labels ErrorState = namedtuple('ErrorState', ['status_code', 'message']) -class TaxlotsSchema(AutoSchemaHelper): - def __init__(self, *args): - super().__init__(*args) - - self.manual_fields = { - ('POST', 'labels'): [self.org_id_field()] - } - - class TaxlotLabelsViewSet(viewsets.ViewSet, OrgMixin): - swagger_schema = TaxlotsSchema renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) _organization = None @@ -37,6 +27,7 @@ def get_queryset(self): ).order_by("name").distinct() return labels + @swagger_auto_schema_org_query_param @action(detail=False, methods=['POST']) def labels(self, request): """ From 20d8b377f1cfb9cf6ea1e91baa379e6b843072e9 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Wed, 10 Jun 2020 11:11:29 -0600 Subject: [PATCH 122/135] removed commented out code --- seed/views/v3/labels.py | 9 --------- seed/views/v3/properties.py | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 593a846723..55b5ba2ec4 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -28,15 +28,6 @@ ErrorState = namedtuple('ErrorState', ['status_code', 'message']) -# class LabelsSchema(AutoSchemaHelper): -# def __init__(self, *args): -# super().__init__(*args) -# -# self.manual_fields = { -# ('GET', 'list'): [self.org_id_field()] -# } - - class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateModelViewSet): """ retrieve: diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 91e43a17aa..01912a9693 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -17,6 +17,7 @@ ErrorState = namedtuple('ErrorState', ['status_code', 'message']) + class PropertyLabelsViewSet(viewsets.ViewSet, OrgMixin): renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) From 7860dde3deeb32c1cb7e0ca133045f2e77a34d4c Mon Sep 17 00:00:00 2001 From: Alex Swindler Date: Wed, 10 Jun 2020 12:36:05 -0600 Subject: [PATCH 123/135] Added HSTS header --- docker/nginx-seed.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/nginx-seed.conf b/docker/nginx-seed.conf index 680a4a0bb8..b409dc30b6 100644 --- a/docker/nginx-seed.conf +++ b/docker/nginx-seed.conf @@ -26,6 +26,9 @@ add_header X-XSS-Protection "1; mode=block"; # https://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; img-src 'self' data: https://stamen-tiles-a.a.ssl.fastly.net https://stamen-tiles-b.a.ssl.fastly.net https://stamen-tiles-c.a.ssl.fastly.net https://stamen-tiles-d.a.ssl.fastly.net; style-src 'self' 'unsafe-inline'; frame-src 'self'; object-src 'none'"; +# HSTS +add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; + # the upstream component nginx needs to connect to upstream seed_upsteam { server unix:///tmp/uwsgi-seed.sock; From d0b6bb91540ba8017629d4a5783f75172a5d396a Mon Sep 17 00:00:00 2001 From: Austin <31779424+aviveiros11@users.noreply.github.com> Date: Wed, 10 Jun 2020 15:33:52 -0600 Subject: [PATCH 124/135] Update seed/api/v3/urls.py Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com> --- seed/api/v3/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index b621602317..522f32d5d8 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -23,7 +23,7 @@ api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files') api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations') api_v3_router.register(r'properties', PropertyLabelsViewSet, base_name='properties') -api_v3_router.register(r'taxlots', TaxlotLabelsViewSet, base_name='labels') +api_v3_router.register(r'taxlots', TaxlotLabelsViewSet, base_name='taxlots') api_v3_router.register(r'users', UserViewSet, base_name='user') urlpatterns = [ From 06845907bb4a8fdc0df51cc2ec49aee03440d3be Mon Sep 17 00:00:00 2001 From: Austin <31779424+aviveiros11@users.noreply.github.com> Date: Wed, 10 Jun 2020 15:34:11 -0600 Subject: [PATCH 125/135] Update seed/views/v3/labels.py Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com> --- seed/views/v3/labels.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 55b5ba2ec4..b4dbfa9bab 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -67,7 +67,9 @@ def get_serializer(self, *args, **kwargs): kwargs['inventory'] = inventory return super().get_serializer(*args, **kwargs) - @swagger_auto_schema_org_query_param + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + ) def list(self, request): """ Returns a list of all labels From 73fae207accf0e8422f85700eed5f601c400c7c0 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Thu, 11 Jun 2020 07:51:46 -0600 Subject: [PATCH 126/135] revised label_service to use v2 for labels --- seed/static/seed/js/services/label_service.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/seed/static/seed/js/services/label_service.js b/seed/static/seed/js/services/label_service.js index cea050ab5f..62cc8283e1 100644 --- a/seed/static/seed/js/services/label_service.js +++ b/seed/static/seed/js/services/label_service.js @@ -61,7 +61,7 @@ angular.module('BE.seed.service.label', []) return get_labels_for_org(user_service.get_organization().id, inventory_type, filter_ids); } - function get_labels_for_org (org_id, inventory_type, filter_ids) { + function get_labels_for_org(org_id, inventory_type, filter_ids) { var params = { organization_id: org_id }; @@ -69,19 +69,14 @@ angular.module('BE.seed.service.label', []) if (inventory_type === 'properties') { params.inventory_type = 'property_view'; body = {selected: filter_ids}; - return $http.post('/api/v3/properties/labels/', body, { - params: params - }).then(map_labels); } else if (inventory_type === 'taxlots') { params.inventory_type = 'taxlot_view'; body = {selected: filter_ids}; - return $http.post('/api/v3/taxlots/labels/', body, { - params: params - }).then(map_labels); } - + return $http.post('/api/v2/labels/filter/', body, { + params: params + }).then(map_labels); } - /* Add a label to an organization's list of labels @@ -109,7 +104,6 @@ angular.module('BE.seed.service.label', []) }); } - /* Update an existing a label in an organization @param {object} label A label object with changed properties to update on server. From 499a19f6a8caa9e24b6bad44bacf4f637a6ab16e Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Thu, 11 Jun 2020 15:51:45 -0600 Subject: [PATCH 127/135] making review changes and using the new method decorators --- seed/utils/api_schema.py | 4 +-- seed/views/v3/labels.py | 61 ++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/seed/utils/api_schema.py b/seed/utils/api_schema.py index 4c8fadea3e..309f8ff041 100644 --- a/seed/utils/api_schema.py +++ b/seed/utils/api_schema.py @@ -31,11 +31,11 @@ def base_field(name, location_attr, description, required, type): ) @staticmethod - def query_org_id_field(required=True): + def query_org_id_field(required=True, description="Organization ID"): return openapi.Parameter( 'organization_id', openapi.IN_QUERY, - description='Organization ID', + description=description, required=required, type=openapi.TYPE_INTEGER ) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index b4dbfa9bab..cc2636c23c 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -1,11 +1,10 @@ - # !/usr/bin/env python # encoding: utf-8 """ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author 'Piper Merriam ' """ -from collections import namedtuple +from drf_yasg.utils import swagger_auto_schema from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer @@ -21,13 +20,51 @@ LabelSerializer, ) from seed.utils.api import drf_api_endpoint -from seed.utils.api_schema import swagger_auto_schema_org_query_param -from seed.utils.labels import _get_labels +from django.utils.decorators import method_decorator +from seed.utils.api_schema import AutoSchemaHelper from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet -ErrorState = namedtuple('ErrorState', ['status_code', 'message']) - +@method_decorator( + name='retrieve', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + ), +) +@method_decorator( + name='list', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + ), +) +@method_decorator( + name='create', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + request_body=AutoSchemaHelper.schema_factory( + { + 'name': 'string', + 'color': 'string', + }, + required=['name'], + description='An object containing meta data for a new label' + ) + ), +) +@method_decorator( + name='update', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + request_body=AutoSchemaHelper.schema_factory( + { + 'name': 'string', + 'color': 'string', + }, + required=['name'], + description='An object containing meta data for a new label' + ) + ), +) class LabelViewSet(DecoratorMixin(drf_api_endpoint), SEEDOrgNoPatchOrOrgCreateModelViewSet): """ retrieve: @@ -66,15 +103,3 @@ def get_serializer(self, *args, **kwargs): ) kwargs['inventory'] = inventory return super().get_serializer(*args, **kwargs) - - @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] - ) - def list(self, request): - """ - Returns a list of all labels - """ - inv_type = None - qs = self.get_queryset() - super_organization = self.get_organization(request) - return _get_labels(request, qs, super_organization, inv_type) From b7fa0c8a840086953dc140b05fcae1331ee5ad3a Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Thu, 11 Jun 2020 15:52:43 -0600 Subject: [PATCH 128/135] changed wording in description --- seed/views/v3/labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index cc2636c23c..262289dfce 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -61,7 +61,7 @@ 'color': 'string', }, required=['name'], - description='An object containing meta data for a new label' + description='An object containing meta data for updating a label' ) ), ) From cb40efab4b435b680abda85e7d16014b5479ee6c Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Thu, 11 Jun 2020 16:21:08 -0600 Subject: [PATCH 129/135] Destroy not Delete smh added the decorator --- seed/views/v3/labels.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 262289dfce..084f46bf02 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -51,6 +51,12 @@ ) ), ) +@method_decorator( + name='destroy', + decorator=swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + ), +) @method_decorator( name='update', decorator=swagger_auto_schema( From 32319663d33a6070f7e3d7d03f4d9d89c5f5175e Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Fri, 12 Jun 2020 14:18:09 -0600 Subject: [PATCH 130/135] pair session with Adrian --- seed/api/v3/urls.py | 8 ++++---- seed/filters.py | 30 ---------------------------- seed/utils/labels.py | 39 ++++++++++++++++++++++++++++++++----- seed/views/v3/labels.py | 9 ++++++--- seed/views/v3/properties.py | 30 ++++++++++++++++------------ seed/views/v3/taxlots.py | 30 ++++++++++++++++------------ 6 files changed, 80 insertions(+), 66 deletions(-) diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 522f32d5d8..81020182fa 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -10,8 +10,8 @@ from seed.views.v3.labels import LabelViewSet from seed.views.v3.import_files import ImportFileViewSet from seed.views.v3.organizations import OrganizationViewSet -from seed.views.v3.properties import PropertyLabelsViewSet -from seed.views.v3.taxlots import TaxlotLabelsViewSet +from seed.views.v3.properties import PropertyViewSet +from seed.views.v3.taxlots import TaxlotViewSet from seed.views.v3.users import UserViewSet api_v3_router = routers.DefaultRouter() @@ -22,8 +22,8 @@ api_v3_router.register(r'data_quality_checks', DataQualityViews, base_name='data_quality_checks') api_v3_router.register(r'import_files', ImportFileViewSet, base_name='import_files') api_v3_router.register(r'organizations', OrganizationViewSet, base_name='organizations') -api_v3_router.register(r'properties', PropertyLabelsViewSet, base_name='properties') -api_v3_router.register(r'taxlots', TaxlotLabelsViewSet, base_name='taxlots') +api_v3_router.register(r'properties', PropertyViewSet, base_name='properties') +api_v3_router.register(r'taxlots', TaxlotViewSet, base_name='taxlots') api_v3_router.register(r'users', UserViewSet, base_name='user') urlpatterns = [ diff --git a/seed/filters.py b/seed/filters.py index 0bb2d0c012..7989279c55 100644 --- a/seed/filters.py +++ b/seed/filters.py @@ -74,33 +74,3 @@ def filter_queryset(self, request): id__in=request.data['selected'], ) return queryset - - -class InventoryFilterBackendWithInvType(filters.BaseFilterBackend): - """ - A permutation of the InventoryFilterBackend class that uses inventory type as a method parameter - """ - - def filter_queryset_with_inv(self, request, inv_type): - params = request.query_params.dict() - # Since this is being passed in as a query string, the object ends up - # coming through as a string. - params['filter_params'] = json.loads(params.get('filter_params', '{}')) - inventory_type = inv_type - params = search.process_search_params( - params=params, - user=request.user, - is_api_request=True, - ) - queryset = search.inventory_search_filter_sort( - inventory_type, - params=params, - user=request.user, - ) - if 'selected' in request.data: - # Return labels limited to the 'selected' list. Otherwise, if selected is empty, return all - if request.data['selected']: - return queryset.filter( - id__in=request.data['selected'], - ) - return queryset diff --git a/seed/utils/labels.py b/seed/utils/labels.py index 56317ca0b1..c08db354c5 100644 --- a/seed/utils/labels.py +++ b/seed/utils/labels.py @@ -5,9 +5,9 @@ All rights reserved. # NOQA :author """ -from seed.filters import ( - InventoryFilterBackendWithInvType, -) +import json + +from seed import search from seed.serializers.labels import ( LabelSerializer, ) @@ -17,8 +17,37 @@ ) -def _get_labels(request, qs, super_organization, inv_type): - inventory = InventoryFilterBackendWithInvType().filter_queryset_with_inv( +def filter_labels_for_inv_type(request, inv_type): + """ + Method used to filter labels by inventory type and return is_applied inventory id's + Method was initially built as a class defined function above to handle more parameters. + """ + params = request.query_params.dict() + # Since this is being passed in as a query string, the object ends up + # coming through as a string. + params['filter_params'] = json.loads(params.get('filter_params', '{}')) + inventory_type = inv_type + params = search.process_search_params( + params=params, + user=request.user, + is_api_request=True, + ) + queryset = search.inventory_search_filter_sort( + inventory_type, + params=params, + user=request.user, + ) + if 'selected' in request.data: + # Return labels limited to the 'selected' list. Otherwise, if selected is empty, return all + if request.data['selected']: + return queryset.filter( + id__in=request.data['selected'], + ) + return queryset + + +def get_labels(request, qs, super_organization, inv_type): + inventory = filter_labels_for_inv_type( request=request, inv_type=inv_type ) results = [ diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 084f46bf02..35dfaa373f 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -4,14 +4,16 @@ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author 'Piper Merriam ' """ +from docutils.nodes import status from drf_yasg.utils import swagger_auto_schema +from rest_framework import response, status +from rest_framework.decorators import action from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer from seed.decorators import DecoratorMixin from seed.filters import ( LabelFilterBackend, - InventoryFilterBackendWithInvType, ) from seed.models import ( StatusLabel as Label, @@ -22,6 +24,7 @@ from seed.utils.api import drf_api_endpoint from django.utils.decorators import method_decorator from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import filter_labels_for_inv_type from seed.utils.viewsets import SEEDOrgNoPatchOrOrgCreateModelViewSet @@ -40,7 +43,7 @@ @method_decorator( name='create', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="put in org id description")], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string', @@ -104,7 +107,7 @@ def get_queryset(self): def get_serializer(self, *args, **kwargs): kwargs['super_organization'] = self.get_organization(self.request) - inventory = InventoryFilterBackendWithInvType().filter_queryset_with_inv( + inventory = filter_labels_for_inv_type( request=self.request, inv_type=None ) kwargs['inventory'] = inventory diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index 01912a9693..d6d6096099 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -2,6 +2,7 @@ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author """ +from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets from collections import namedtuple @@ -12,24 +13,26 @@ StatusLabel as Label, ) from seed.utils.api import OrgMixin -from seed.utils.api_schema import swagger_auto_schema_org_query_param -from seed.utils.labels import _get_labels +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import get_labels ErrorState = namedtuple('ErrorState', ['status_code', 'message']) -class PropertyLabelsViewSet(viewsets.ViewSet, OrgMixin): +class PropertyViewSet(viewsets.ViewSet, OrgMixin): renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) _organization = None - def get_queryset(self): - labels = Label.objects.filter( - super_organization=self.get_parent_org(self.request) - ).order_by("name").distinct() - return labels - - @swagger_auto_schema_org_query_param + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + request_body=AutoSchemaHelper.schema_factory( + { + 'selected': ['integer'], + }, + description='An array of property ids to be returned' + ) + ) @action(detail=False, methods=['POST']) def labels(self, request): """ @@ -37,6 +40,9 @@ def labels(self, request): in the response pertains to the labels applied to property_view """ inv_type = 'property_view' - qs = self.get_queryset() + labels = Label.objects.filter( + super_organization=self.get_parent_org(self.request) + ).order_by("name").distinct() super_organization = self.get_organization(request) - return _get_labels(request, qs, super_organization, inv_type) + # TODO: refactor to avoid passing request here + return get_labels(request, labels, super_organization, inv_type) diff --git a/seed/views/v3/taxlots.py b/seed/views/v3/taxlots.py index 1d158fc4ca..87229983f7 100644 --- a/seed/views/v3/taxlots.py +++ b/seed/views/v3/taxlots.py @@ -2,6 +2,7 @@ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author """ +from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets from collections import namedtuple @@ -10,24 +11,26 @@ from rest_framework.renderers import JSONRenderer from seed.models import StatusLabel as Label from seed.utils.api import OrgMixin -from seed.utils.api_schema import swagger_auto_schema_org_query_param -from seed.utils.labels import _get_labels +from seed.utils.api_schema import AutoSchemaHelper +from seed.utils.labels import get_labels ErrorState = namedtuple('ErrorState', ['status_code', 'message']) -class TaxlotLabelsViewSet(viewsets.ViewSet, OrgMixin): +class TaxlotViewSet(viewsets.ViewSet, OrgMixin): renderer_classes = (JSONRenderer,) parser_classes = (JSONParser,) _organization = None - def get_queryset(self): - labels = Label.objects.filter( - super_organization=self.get_parent_org(self.request) - ).order_by("name").distinct() - return labels - - @swagger_auto_schema_org_query_param + @swagger_auto_schema( + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + request_body=AutoSchemaHelper.schema_factory( + { + 'selected': ['integer'], + }, + description='An array of taxlot ids to be returned' + ) + ) @action(detail=False, methods=['POST']) def labels(self, request): """ @@ -35,6 +38,9 @@ def labels(self, request): in the response pertains to the labels applied to taxlot_view """ inv_type = 'taxlot_view' - qs = self.get_queryset() + labels = Label.objects.filter( + super_organization=self.get_parent_org(self.request) + ).order_by("name").distinct() super_organization = self.get_organization(request) - return _get_labels(request, qs, super_organization, inv_type) + # TODO: refactor to avoid passing request here + return get_labels(request, labels, super_organization, inv_type) From 740ff1548b6fa54a4ca67503ad046fdcdf2a0203 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 15 Jun 2020 13:10:42 -0600 Subject: [PATCH 131/135] flake8 --- charts/backend/templates/db-postgres-deployment.yaml | 2 +- seed/views/v3/labels.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/charts/backend/templates/db-postgres-deployment.yaml b/charts/backend/templates/db-postgres-deployment.yaml index b8da9a4571..55d510d8ce 100644 --- a/charts/backend/templates/db-postgres-deployment.yaml +++ b/charts/backend/templates/db-postgres-deployment.yaml @@ -32,7 +32,7 @@ spec: image: seedplatform/postgres-seed:11.2 name: db-postrgres # command: ["tail"] -# args: ['-f', '/dev/null'] +# args: ['-f', '/dev/n ports: - containerPort: 5432 resources: diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 35dfaa373f..10f93289ea 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -4,10 +4,7 @@ :copyright (c) 2014 - 2020, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author 'Piper Merriam ' """ -from docutils.nodes import status from drf_yasg.utils import swagger_auto_schema -from rest_framework import response, status -from rest_framework.decorators import action from rest_framework.parsers import JSONParser, FormParser from rest_framework.renderers import JSONRenderer From f3b03cac38a93095ac47263d4ed0f2c9916d7906 Mon Sep 17 00:00:00 2001 From: Austin <31779424+aviveiros11@users.noreply.github.com> Date: Mon, 15 Jun 2020 13:44:03 -0600 Subject: [PATCH 132/135] Update seed/views/v3/properties.py Co-authored-by: Adrian Lara <30608004+adrian-lara@users.noreply.github.com> --- seed/views/v3/properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index d6d6096099..83416d6329 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -30,7 +30,7 @@ class PropertyViewSet(viewsets.ViewSet, OrgMixin): { 'selected': ['integer'], }, - description='An array of property ids to be returned' + description='IDs for properties to be checked for which labels are applied.' ) ) @action(detail=False, methods=['POST']) From 8ea899777a032a39b2bb3e74985281bb6384606c Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 15 Jun 2020 14:37:37 -0600 Subject: [PATCH 133/135] inventory_type being passed as a string / more cleanup / asdded descriptions --- .../templates/db-postgres-deployment.yaml | 2 +- seed/utils/labels.py | 3 +-- seed/views/v3/labels.py | 22 ++++++++++++++----- seed/views/v3/properties.py | 5 ++--- seed/views/v3/taxlots.py | 7 +++--- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/charts/backend/templates/db-postgres-deployment.yaml b/charts/backend/templates/db-postgres-deployment.yaml index 55d510d8ce..b8da9a4571 100644 --- a/charts/backend/templates/db-postgres-deployment.yaml +++ b/charts/backend/templates/db-postgres-deployment.yaml @@ -32,7 +32,7 @@ spec: image: seedplatform/postgres-seed:11.2 name: db-postrgres # command: ["tail"] -# args: ['-f', '/dev/n +# args: ['-f', '/dev/null'] ports: - containerPort: 5432 resources: diff --git a/seed/utils/labels.py b/seed/utils/labels.py index c08db354c5..d0be14aab3 100644 --- a/seed/utils/labels.py +++ b/seed/utils/labels.py @@ -17,7 +17,7 @@ ) -def filter_labels_for_inv_type(request, inv_type): +def filter_labels_for_inv_type(request, inventory_type=None): """ Method used to filter labels by inventory type and return is_applied inventory id's Method was initially built as a class defined function above to handle more parameters. @@ -26,7 +26,6 @@ def filter_labels_for_inv_type(request, inv_type): # Since this is being passed in as a query string, the object ends up # coming through as a string. params['filter_params'] = json.loads(params.get('filter_params', '{}')) - inventory_type = inv_type params = search.process_search_params( params=params, user=request.user, diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 10f93289ea..77002c5e95 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -28,19 +28,25 @@ @method_decorator( name='retrieve', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " + "overrides the users " + "(default) current org id")] ), ) @method_decorator( name='list', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " + "overrides the users (" + "default) current org id")] ), ) @method_decorator( name='create', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="put in org id description")], + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " + "overrides the users (" + "default) current org id")], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string', @@ -54,13 +60,17 @@ @method_decorator( name='destroy', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)] + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " + "overrides the users (" + "default) current org id")] ), ) @method_decorator( name='update', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " + "overrides the users (" + "default) current org id")], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string', @@ -105,7 +115,7 @@ def get_queryset(self): def get_serializer(self, *args, **kwargs): kwargs['super_organization'] = self.get_organization(self.request) inventory = filter_labels_for_inv_type( - request=self.request, inv_type=None + request=self.request ) kwargs['inventory'] = inventory return super().get_serializer(*args, **kwargs) diff --git a/seed/views/v3/properties.py b/seed/views/v3/properties.py index d6d6096099..c03fe32d24 100644 --- a/seed/views/v3/properties.py +++ b/seed/views/v3/properties.py @@ -25,7 +25,7 @@ class PropertyViewSet(viewsets.ViewSet, OrgMixin): _organization = None @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)], request_body=AutoSchemaHelper.schema_factory( { 'selected': ['integer'], @@ -39,10 +39,9 @@ def labels(self, request): Returns a list of all labels where the is_applied field in the response pertains to the labels applied to property_view """ - inv_type = 'property_view' labels = Label.objects.filter( super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() super_organization = self.get_organization(request) # TODO: refactor to avoid passing request here - return get_labels(request, labels, super_organization, inv_type) + return get_labels(request, labels, super_organization, 'property_view') diff --git a/seed/views/v3/taxlots.py b/seed/views/v3/taxlots.py index 87229983f7..baa5615f4e 100644 --- a/seed/views/v3/taxlots.py +++ b/seed/views/v3/taxlots.py @@ -23,12 +23,12 @@ class TaxlotViewSet(viewsets.ViewSet, OrgMixin): _organization = None @swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False)], + manual_parameters=[AutoSchemaHelper.query_org_id_field(required=True)], request_body=AutoSchemaHelper.schema_factory( { 'selected': ['integer'], }, - description='An array of taxlot ids to be returned' + description='IDs for taxlots to be checked for which labels are applied.' ) ) @action(detail=False, methods=['POST']) @@ -37,10 +37,9 @@ def labels(self, request): Returns a list of all labels where the is_applied field in the response pertains to the labels applied to taxlot_view """ - inv_type = 'taxlot_view' labels = Label.objects.filter( super_organization=self.get_parent_org(self.request) ).order_by("name").distinct() super_organization = self.get_organization(request) # TODO: refactor to avoid passing request here - return get_labels(request, labels, super_organization, inv_type) + return get_labels(request, labels, super_organization, 'taxlot_view') From 3b0fa49af41300673e70a845859736ac9d1b2470 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 15 Jun 2020 14:43:11 -0600 Subject: [PATCH 134/135] changed inv_type to inventory_type --- seed/utils/labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seed/utils/labels.py b/seed/utils/labels.py index d0be14aab3..bbe3bdcd09 100644 --- a/seed/utils/labels.py +++ b/seed/utils/labels.py @@ -47,7 +47,7 @@ def filter_labels_for_inv_type(request, inventory_type=None): def get_labels(request, qs, super_organization, inv_type): inventory = filter_labels_for_inv_type( - request=request, inv_type=inv_type + request=request, inventory_type=inv_type ) results = [ LabelSerializer( From 391515753cf5255568275d73b77ddbedf958ba31 Mon Sep 17 00:00:00 2001 From: aviveiros11 Date: Mon, 15 Jun 2020 15:38:30 -0600 Subject: [PATCH 135/135] fixed formatting --- seed/views/v3/labels.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/seed/views/v3/labels.py b/seed/views/v3/labels.py index 77002c5e95..a3637187ce 100644 --- a/seed/views/v3/labels.py +++ b/seed/views/v3/labels.py @@ -28,25 +28,25 @@ @method_decorator( name='retrieve', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " - "overrides the users " - "(default) current org id")] + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")] ), ) @method_decorator( name='list', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " - "overrides the users (" - "default) current org id")] + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")] ), ) @method_decorator( name='create', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " - "overrides the users (" - "default) current org id")], + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string', @@ -60,17 +60,17 @@ @method_decorator( name='destroy', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " - "overrides the users (" - "default) current org id")] + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")] ), ) @method_decorator( name='update', decorator=swagger_auto_schema( - manual_parameters=[AutoSchemaHelper.query_org_id_field(required=False, description="Optional org id which " - "overrides the users (" - "default) current org id")], + manual_parameters=[AutoSchemaHelper.query_org_id_field( + required=False, + description="Optional org id which overrides the users (default) current org id")], request_body=AutoSchemaHelper.schema_factory( { 'name': 'string',