Merge pull request #1877 from SEED-platform/labels-to-views

Labels to views

docs/source/data_quality.rst

@@ -1,5 +1,9 @@
 Data Quality
 ============
 
-Data quality checks are run after the data are paired or on-demand by selecting rows in the inventory
-page and clicking the action button.
+Data quality checks are run after the data are paired, during import of Properties/TaxLots, or on-demand by selecting rows in the inventory
+page and clicking the action button. This checks whether any default or user-defined Rules are broken or satisfied by Property/TaxLot records.
+
+Notably, in most cases when data quality checks are run, Labels can be applied for any broken Rules that have a Label.
+To elaborate, Rules can have an attached Label. When a data quality check is run, records that break one of these "Labeled Rules"
+are then given that Label. The case where this Label attachment does not happen is during import due to performance reasons.

seed/migrations/0104_auto_20190509_1854.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-09 01:54
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+def reassociate_labels_to_views(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+
+    PropertyView = apps.get_model('seed', 'PropertyView')
+    ThroughModel = PropertyView.labels.through
+    records = []
+    property_views = PropertyView.objects.using(db_alias).all()\
+        .select_related('property').prefetch_related('property__labels')
+    for p_view in property_views:
+        for label in p_view.property.labels.all():
+            records.append(ThroughModel(propertyview=p_view, statuslabel=label))
+    ThroughModel.objects.bulk_create(records)
+    # Free memory
+    del property_views
+
+    TaxLotView = apps.get_model('seed', 'TaxLotView')
+    ThroughModel = TaxLotView.labels.through
+    records = []
+    taxlot_views = TaxLotView.objects.using(db_alias).all()\
+        .select_related('taxlot').prefetch_related('taxlot__labels')
+    for tl_view in taxlot_views:
+        for label in tl_view.taxlot.labels.all():
+            records.append(ThroughModel(taxlotview=tl_view, statuslabel=label))
+    ThroughModel.objects.bulk_create(records)
+    # Free memory
+    del taxlot_views
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('seed', '0103_auto_20190505_0731'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='propertyview',
+            name='labels',
+            field=models.ManyToManyField(to='seed.StatusLabel'),
+        ),
+        migrations.AddField(
+            model_name='taxlotview',
+            name='labels',
+            field=models.ManyToManyField(to='seed.StatusLabel'),
+        ),
+        migrations.RunPython(reassociate_labels_to_views),
+        migrations.RemoveField(
+            model_name='taxlot',
+            name='labels',
+        ),
+        migrations.RemoveField(
+            model_name='property',
+            name='labels',
+        ),
+    ]

seed/models/data_quality.py

@@ -22,9 +22,7 @@
 from seed.models import (
     Column,
     StatusLabel,
-    Property,
     PropertyView,
-    TaxLot,
     TaxLotView
 )
 from seed.models import obj_to_dict
@@ -617,21 +615,21 @@ def _check(self, rules, row):
         # check if the row has any rules applied to it
         model_labels = {'linked_id': None, 'label_ids': []}
         if row.__class__.__name__ == 'PropertyState':
-            label = apps.get_model('seed', 'Property_labels')
+            label = apps.get_model('seed', 'PropertyView_labels')
             if PropertyView.objects.filter(state=row).exists():
-                model_labels['linked_id'] = PropertyView.objects.get(state=row).property_id
+                model_labels['linked_id'] = PropertyView.objects.get(state=row).id
                 model_labels['label_ids'] = list(
-                    label.objects.filter(property_id=model_labels['linked_id']).values_list(
+                    label.objects.filter(propertyview_id=model_labels['linked_id']).values_list(
                         'statuslabel_id', flat=True)
                 )
                 # _log.debug("Property {} has {} labels".format(model_labels['linked_id'],
                 #                                               len(model_labels['label_ids'])))
         elif row.__class__.__name__ == 'TaxLotState':
-            label = apps.get_model('seed', 'TaxLot_labels')
+            label = apps.get_model('seed', 'TaxLotView_labels')
             if TaxLotView.objects.filter(state=row).exists():
-                model_labels['linked_id'] = TaxLotView.objects.get(state=row).taxlot_id
+                model_labels['linked_id'] = TaxLotView.objects.get(state=row).id
                 model_labels['label_ids'] = list(
-                    label.objects.filter(taxlot_id=model_labels['linked_id']).values_list(
+                    label.objects.filter(taxlotview_id=model_labels['linked_id']).values_list(
                         'statuslabel_id', flat=True)
                 )
                 # _log.debug("TaxLot {} has {} labels".format(model_labels['linked_id'],
@@ -917,7 +915,7 @@ def add_invalid_geometry_entry_provided(self, row_id, rule, display_name, value)
     def update_status_label(self, label_class, rule, linked_id):
         """
 
-        :param label_class: statuslabel object, either property label or taxlot label
+        :param label_class: statuslabel object, either propertyview label or taxlotview label
         :param rule: rule object
         :param linked_id: id of propertystate or taxlotstate object
         :return: boolean, if labeled was applied
@@ -927,26 +925,26 @@ def update_status_label(self, label_class, rule, linked_id):
             label_org_id = rule.status_label.super_organization_id
 
             if rule.table_name == 'PropertyState':
-                property_parent_org_id = Property.objects.get(pk=linked_id).organization.get_parent().id
+                property_parent_org_id = PropertyView.objects.get(pk=linked_id).property.organization.get_parent().id
                 if property_parent_org_id == label_org_id:
-                    label_class.objects.get_or_create(property_id=linked_id,
+                    label_class.objects.get_or_create(propertyview_id=linked_id,
                                                       statuslabel_id=rule.status_label_id)
                 else:
                     raise IntegrityError(
-                        'Label with super_organization_id={} cannot be applied to a property with parent '
+                        'Label with super_organization_id={} cannot be applied to a record with parent '
                         'organization_id={}.'.format(
                             label_org_id,
                             property_parent_org_id
                         )
                     )
             else:
-                taxlot_parent_org_id = TaxLot.objects.get(pk=linked_id).organization.get_parent().id
+                taxlot_parent_org_id = TaxLotView.objects.get(pk=linked_id).taxlot.organization.get_parent().id
                 if taxlot_parent_org_id == label_org_id:
-                    label_class.objects.get_or_create(taxlot_id=linked_id,
+                    label_class.objects.get_or_create(taxlotview_id=linked_id,
                                                       statuslabel_id=rule.status_label_id)
                 else:
                     raise IntegrityError(
-                        'Label with super_organization_id={} cannot be applied to a taxlot with parent '
+                        'Label with super_organization_id={} cannot be applied to a record with parent '
                         'organization_id={}.'.format(
                             label_org_id,
                             taxlot_parent_org_id

seed/models/properties.py

@@ -67,7 +67,6 @@ class Property(models.Model):
     # Handle properties that may have multiple properties (e.g. buildings)
     campus = models.BooleanField(default=False)
     parent_property = models.ForeignKey('Property', blank=True, null=True)
-    labels = models.ManyToManyField(StatusLabel)
 
     # Track when the entry was created and when it was updated
     created = models.DateTimeField(auto_now_add=True)
@@ -684,6 +683,8 @@ class PropertyView(models.Model):
     cycle = models.ForeignKey(Cycle, on_delete=models.PROTECT)
     state = models.ForeignKey(PropertyState, on_delete=models.CASCADE)
 
+    labels = models.ManyToManyField(StatusLabel)
+
     # notes has a relationship here -- PropertyViews have notes, not the state, and not the property.
 
     def __str__(self):
@@ -807,4 +808,4 @@ def sync_latitude_longitude_and_long_lat(sender, instance, **kwargs):
             instance.geocoding_confidence = None
 
 
-m2m_changed.connect(compare_orgs_between_label_and_target, sender=Property.labels.through)
+m2m_changed.connect(compare_orgs_between_label_and_target, sender=PropertyView.labels.through)

seed/models/tax_lots.py

@@ -46,7 +46,6 @@ class TaxLot(models.Model):
     # NOTE: we have been calling this the organization. We
     # should stay consistent although I prefer the name organization (!super_org)
     organization = models.ForeignKey(Organization)
-    labels = models.ManyToManyField(StatusLabel)
 
     # Track when the entry was created and when it was updated
     created = models.DateTimeField(auto_now_add=True)
@@ -382,7 +381,7 @@ class TaxLotView(models.Model):
     state = models.ForeignKey(TaxLotState, on_delete=models.CASCADE)
     cycle = models.ForeignKey(Cycle, on_delete=models.PROTECT)
 
-    # labels = models.ManyToManyField(StatusLabel)
+    labels = models.ManyToManyField(StatusLabel)
 
     def __str__(self):
         return 'TaxLot View - %s' % self.pk
@@ -486,4 +485,4 @@ class Meta:
         index_together = [['state', 'name'], ['parent_state1', 'parent_state2']]
 
 
-m2m_changed.connect(compare_orgs_between_label_and_target, sender=TaxLot.labels.through)
+m2m_changed.connect(compare_orgs_between_label_and_target, sender=TaxLotView.labels.through)

seed/serializers/labels.py

@@ -6,9 +6,7 @@
 """
 from rest_framework import serializers
 
-from seed.models import (
-    StatusLabel as Label,
-    Property, TaxLot, PropertyView, TaxLotView)
+from seed.models import StatusLabel as Label
 
 
 class LabelSerializer(serializers.ModelSerializer):
@@ -52,14 +50,6 @@ class Meta:
     def get_is_applied(self, obj):
         filtered_result = []
         if self.inventory:
-            result = self.inventory.filter(
-                labels=obj,
-            ).values_list('id', flat=True)
-
-            # Limit results to ones attached to view
-            if self.inventory.model is Property:
-                filtered_result = PropertyView.objects.filter(property_id__in=result).values_list('pk', flat=True)
-            elif self.inventory.model is TaxLot:
-                filtered_result = TaxLotView.objects.filter(taxlot_id__in=result).values_list('pk', flat=True)
+            filtered_result = self.inventory.prefetch_related('labels').filter(labels__in=[obj]).values_list('id', flat=True)
 
         return filtered_result

seed/serializers/properties.py

@@ -10,7 +10,6 @@
 import json
 from collections import OrderedDict
 
-from django.apps import apps
 from django.db import models
 from django.utils.timezone import make_naive
 from past.builtins import basestring
@@ -39,9 +38,6 @@
 from seed.serializers.scenarios import ScenarioSerializer
 from seed.serializers.taxlots import TaxLotViewSerializer
 
-# expose internal model
-PropertyLabel = apps.get_model('seed', 'Property_labels')
-
 CYCLE_FIELDS = ['id', 'name', 'start', 'end', 'created']
 
 # Need to reevaluate this list of fields that are being removed.
@@ -100,23 +96,14 @@ def to_representation(self, data):
             iterable = data.all().prefetch_related('parent_property')
         else:
             iterable = data
-        property_ids = [item.id for item in iterable]
-        labels = PropertyLabel.objects.filter(property_id__in=property_ids)
-        labelset = {}
-        for label in labels:
-            record = labelset.setdefault(label.property_id, [])
-            record.append(label.statuslabel_id)
         result = []
         for item in iterable:
             representation = self.child.to_representation(item)
-            representation['labels'] = labelset.get(item.id, None)
             result.append(representation)
         return result
 
 
 class PropertySerializer(serializers.ModelSerializer):
-    # list of status labels (rather than the join field)
-    labels = PropertyLabelsField(read_only=True, many=True)
 
     class Meta:
         model = Property
@@ -265,12 +252,15 @@ def to_representation(self, data):
 
 
 class PropertyViewSerializer(serializers.ModelSerializer):
+    # list of status labels (rather than the join field)
+    labels = PropertyLabelsField(read_only=True, many=True)
+
     state = PropertyStateSerializer()
 
     class Meta:
         model = PropertyView
         depth = 1
-        fields = ('id', 'cycle', 'state', 'property')
+        fields = ('id', 'cycle', 'state', 'property', 'labels')
 
 
 class PropertyViewListSerializer(serializers.ListSerializer):

seed/serializers/taxlots.py

@@ -17,9 +17,6 @@ def to_representation(self, value):
 
 
 class TaxLotSerializer(serializers.ModelSerializer):
-    # list of status labels (rather than the join field)
-    labels = TaxLotLabelsField(read_only=True, many=True)
-
     class Meta:
         model = TaxLot
         fields = '__all__'
@@ -62,6 +59,9 @@ def to_representation(self, data):
 
 
 class TaxLotViewSerializer(serializers.ModelSerializer):
+    # list of status labels (rather than the join field)
+    labels = TaxLotLabelsField(read_only=True, many=True)
+
     state = TaxLotStateSerializer()
 
     class Meta:

seed/static/seed/js/controllers/inventory_detail_controller.js

@@ -322,15 +322,15 @@ angular.module('BE.seed.controller.inventory_detail', [])
           controller: 'update_item_labels_modal_controller',
           resolve: {
             inventory_ids: function () {
-              return [$scope.item_parent.id];
+              return [$scope.inventory.view_id];
             },
             inventory_type: function () {
               return $scope.inventory_type;
             }
           }
         });
         modalInstance.result.then(function () {
-          label_service.get_labels([$scope.item_parent.id], {
+          label_service.get_labels([$scope.inventory.view_id], {
             inventory_type: $scope.inventory_type
           }).then(function (labels) {
             $scope.labels = _.filter(labels, function (label) {

seed/static/seed/js/controllers/inventory_list_controller.js

@@ -265,7 +265,8 @@ angular.module('BE.seed.controller.inventory_list', [])
           controller: 'update_item_labels_modal_controller',
           resolve: {
             inventory_ids: function () {
-              return _.map(_.filter($scope.gridApi.selection.getSelectedRows(), {$$treeLevel: 0}), 'id');
+              var view_id_prop = ($scope.inventory_type === 'taxlots') ? 'taxlot_view_id' : 'property_view_id';
+              return _.map(_.filter($scope.gridApi.selection.getSelectedRows(), {$$treeLevel: 0}), view_id_prop);
             },
             inventory_type: function () {
               return $scope.inventory_type;

seed/static/seed/js/seed.js

@@ -1179,13 +1179,7 @@ SEED_app.config(['stateHelperProvider', '$urlRouterProvider', '$locationProvider
             return currentProfile;
           }],
           labels_payload: ['$stateParams', 'label_service', 'inventory_payload', function ($stateParams, label_service, inventory_payload) {
-            var inventory_id;
-            if ($stateParams.inventory_type === 'properties') {
-              inventory_id = inventory_payload.property.id;
-            } else {
-              inventory_id = inventory_payload.taxlot.id;
-            }
-            return label_service.get_labels([inventory_id], {
+            return label_service.get_labels([$stateParams.view_id], {
               inventory_type: $stateParams.inventory_type
             });
           }]

seed/static/seed/js/services/label_service.js

@@ -63,8 +63,9 @@ angular.module('BE.seed.service.label', [])
           organization_id: org_id
         }, search_params);
 
-        // If no inventory_type specified use 'property' just to get the list of all labels
-        searchArgs.inventory_type = (searchArgs.inventory_type === 'taxlots') ? 'taxlot' : 'property';
+        // If no inventory_type specified use 'property_view' just to get the list of all labels
+        // Used to specify which Label-connected Model to use in the the back-end.
+        searchArgs.inventory_type = (searchArgs.inventory_type === 'taxlots') ? 'taxlot_view' : 'property_view';
 
         return $http.post('/api/v2/labels/filter/', {
           selected: selected
@@ -95,7 +96,7 @@ angular.module('BE.seed.service.label', [])
       function create_label_for_org (org_id, label) {
         return $http.post('/api/v2/labels/', label, {
           params: {
-            inventory_type: 'property',
+            inventory_type: 'property_view',
             organization_id: org_id
           }
         }).then(function (response) {

seed/test_helpers/fake.py

@@ -601,20 +601,13 @@ def __init__(self, organization=None):
         self.organization = organization
         self.label_factory = FakeStatusLabelFactory(organization=organization)
 
-    def get_taxlot(self, organization=None, labels=None):
+    def get_taxlot(self, organization=None):
         """Get taxlot instance."""
         organization = self._get_attr('organization', organization)
         taxlot_details = {
             'organization': organization
         }
         taxlot = TaxLot.objects.create(**taxlot_details)
-        if labels:
-            for label in labels:
-                taxlot.labels.add(label)
-        else:
-            taxlot.labels.add(
-                self.label_factory.get_statuslabel(organization=organization)
-            )
         return taxlot