Merge branch 'develop' into 417_cache_report_settings

locale/en_US/LC_MESSAGES/django.po

@@ -1125,6 +1125,9 @@ msgstr "kWh/m²/year"
 msgid "Label"
 msgstr "Label"
 
+msgid "Label must be assigned when using Valid Data Severity."
+msgstr "Label must be assigned when using Valid Data Severity."
+
 msgid "Label Name"
 msgstr "Label Name"
 

locale/fr_CA/LC_MESSAGES/django.po

@@ -1130,6 +1130,9 @@ msgstr "kWh/m²/année"
 msgid "Label"
 msgstr "Étiquette"
 
+msgid "Label must be assigned when using Valid Data Severity."
+msgstr "L'étiquette doit être attribuée lors de l'utilisation de la gravité de données valide."
+
 msgid "Label Name"
 msgstr "Nom de l'étiquette"
 

seed/migrations/0114_auto_20191211_0958.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-12-11 17:58
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('seed', '0113_column_geocoding_order'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='rule',
+            name='severity',
+            field=models.IntegerField(choices=[(0, 'error'), (1, 'warning'), (2, 'valid')], default=0),
+        ),
+    ]

seed/models/data_quality.py

@@ -48,6 +48,10 @@ class UnitMismatchError(Exception):
     pass
 
 
+class MissingLabelError(Exception):
+    pass
+
+
 def format_pint_violation(rule, source_value):
     """
     Format a pint min, max violation for human readability.
@@ -110,9 +114,11 @@ class Rule(models.Model):
 
     SEVERITY_ERROR = 0
     SEVERITY_WARNING = 1
+    SEVERITY_VALID = 2
     SEVERITY = [
         (SEVERITY_ERROR, 'error'),
-        (SEVERITY_WARNING, 'warning')
+        (SEVERITY_WARNING, 'warning'),
+        (SEVERITY_VALID, 'valid'),
     ]
 
     DEFAULT_RULES = [
@@ -714,9 +720,10 @@ def _check(self, rules, row):
                     # check the min and max values
                     try:
                         if not rule.minimum_valid(value):
-                            s_min, s_max, s_value = rule.format_strings(value)
-                            self.add_result_min_error(row.id, rule, display_name, s_value, s_min)
-                            label_applied = self.update_status_label(label, rule, linked_id)
+                            if rule.severity == Rule.SEVERITY_ERROR or rule.severity == Rule.SEVERITY_WARNING:
+                                s_min, s_max, s_value = rule.format_strings(value)
+                                self.add_result_min_error(row.id, rule, display_name, s_value, s_min)
+                                label_applied = self.update_status_label(label, rule, linked_id)
                     except ComparisonError:
                         s_min, s_max, s_value = rule.format_strings(value)
                         self.add_result_comparison_error(row.id, rule, display_name, s_value, s_min)
@@ -731,9 +738,10 @@ def _check(self, rules, row):
 
                     try:
                         if not rule.maximum_valid(value):
-                            s_min, s_max, s_value = rule.format_strings(value)
-                            self.add_result_max_error(row.id, rule, display_name, s_value, s_max)
-                            label_applied = self.update_status_label(label, rule, linked_id)
+                            if rule.severity == Rule.SEVERITY_ERROR or rule.severity == Rule.SEVERITY_WARNING:
+                                s_min, s_max, s_value = rule.format_strings(value)
+                                self.add_result_max_error(row.id, rule, display_name, s_value, s_max)
+                                label_applied = self.update_status_label(label, rule, linked_id)
                     except ComparisonError:
                         s_min, s_max, s_value = rule.format_strings(value)
                         self.add_result_comparison_error(row.id, rule, display_name, s_value, s_max)
@@ -746,6 +754,29 @@ def _check(self, rules, row):
                         self.add_result_dimension_error(row.id, rule, display_name, value)
                         continue
 
+                    # Check for mandatory label for valid data:
+                    try:
+                        if rule.minimum_valid(value) and rule.maximum_valid(value):
+                            if rule.severity == Rule.SEVERITY_VALID:
+                                '''
+                                s_min, s_max, s_value = rule.format_strings(value)
+                                self.results[row.id]['data_quality_results'].append(
+                                    {
+                                        'field': rule.field,
+                                        'formatted_field': display_name,
+                                        'value': s_value,
+                                        'table_name': rule.table_name,
+                                        'message': display_name + ' is valid',
+                                        'detailed_message': display_name + ' [' + s_value + '] is valid data',
+                                        'severity': rule.get_severity_display(),
+                                    }
+                                )
+                                '''
+                                label_applied = self.update_status_label(label, rule, linked_id)
+                    except MissingLabelError:
+                        self.add_result_missing_label(row.id, rule, display_name, value)
+                        continue
+
                 if not label_applied and rule.status_label_id in model_labels['label_ids']:
                     self.remove_status_label(label, rule, linked_id)
 
@@ -927,6 +958,18 @@ def add_result_missing_req(self, row_id, rule, display_name, value):
             'severity': rule.get_severity_display(),
         })
 
+    def add_result_missing_label(self, row_id, rule, display_name, value):
+        if rule.severity == Rule.SEVERITY_VALID:
+            self.results[row_id]['data_quality_results'].append({
+                'field': rule.field,
+                'formatted_field': rule.field,
+                'value': value,
+                'table_name': rule.table_name,
+                'message': rule.status_label + ' is missing',
+                'detailed_message': rule.status_label + ' is required and missing',
+                'severity': rule.get_severity_display(),
+            })
+
     def add_result_missing_and_none(self, row_id, rule, display_name, value):
         self.results[row_id]['data_quality_results'].append({
             'field': rule.field,
@@ -974,7 +1017,6 @@ def update_status_label(self, label_class, rule, linked_id):
         :param linked_id: id of propertystate or taxlotstate object
         :return: boolean, if labeled was applied
         """
-
         if rule.status_label_id is not None and linked_id is not None:
             label_org_id = rule.status_label.super_organization_id
 

seed/static/seed/js/controllers/admin_controller.js

@@ -69,8 +69,8 @@ angular.module('BE.seed.controller.admin', [])
           });
           update_alert(true, 'Organization ' + org.name + ' created');
 
-        }, function (data) {
-          update_alert(false, 'error creating organization: ' + data.message);
+        }).catch(function (response) {
+          update_alert(false, 'error creating organization: ' + response.data.message);
         });
       };
       $scope.user_form.add = function (user) {
@@ -89,12 +89,19 @@ angular.module('BE.seed.controller.admin', [])
           get_organizations();
           $scope.user_form.reset();
 
-        }, function (response) {
+        }).catch(function (response) {
           update_alert(false, 'error creating user: ' + response.data.message);
         });
       };
       $scope.org_form.not_ready = function () {
-        return _.isUndefined($scope.org.email);
+        return _.isUndefined($scope.org.email) || organization_exists($scope.org.name);
+      };
+
+      var organization_exists = function (name) {
+        var orgs = _.map($scope.org_user.organizations, function (org) {
+          return org.name.toLowerCase();
+        });
+        return _.includes(orgs, name.toLowerCase());
       };
 
       $scope.user_form.reset = function () {
@@ -130,7 +137,7 @@ angular.module('BE.seed.controller.admin', [])
       $scope.get_organizations_users = function (org) {
         organization_service.get_organization_users(org).then(function (data) {
           $scope.org_user.users = data.users;
-        }, function (response) {
+        }).catch(function (response) {
           $log.log({message: 'error from data call', status: response.status, data: response.data});
           update_alert(false, 'error getting organizations: ' + response.data.message);
         });
@@ -143,7 +150,7 @@ angular.module('BE.seed.controller.admin', [])
           });
           $scope.get_organizations_users($scope.org_user.organization);
           update_alert(true, 'user ' + $scope.org_user.user.email + ' added to organization ' + $scope.org_user.organization.name);
-        }, function (response) {
+        }).catch(function (response) {
           $log.log({message: 'error from data call', status: response.status, data: response.data});
           update_alert(false, 'error adding user to organization: ' + response.data.message);
         });
@@ -153,7 +160,7 @@ angular.module('BE.seed.controller.admin', [])
         organization_service.remove_user(user_id, org_id).then(function () {
           $scope.get_organizations_users($scope.org_user.organization);
           update_alert(true, 'user removed organization');
-        }, function (response) {
+        }).catch(function (response) {
           $log.log({message: 'error from data call', status: response.status, data: response.data});
           update_alert(false, 'error removing user from organization: ' + response.data.message);
         });

seed/static/seed/js/controllers/data_quality_admin_controller.js

@@ -197,7 +197,9 @@ angular.module('BE.seed.controller.data_quality_admin', [])
         data_quality_service.save_data_quality_rules($scope.org.org_id, rules).then(function (rules) {
           loadRules(rules);
           $scope.rules_updated = true;
-        }, function (data) {
+        }).then(function (data) {
+          $scope.$emit('app_success', data);
+        }).catch(function (data) {
           $scope.$emit('app_error', data);
         }).finally(function () {
           spinner_utility.hide();

seed/static/seed/js/controllers/menu_controller.js

@@ -56,8 +56,12 @@ angular.module('BE.seed.controller.menu', [])
         if (!$rootScope.route_load_error) {
           $rootScope.route_load_error = true;
           $scope.menu.error_message = data.message;
+          $rootScope.load_error_message = data.data.message;
         }
       });
+      $scope.$on('app_success', function () {
+        $rootScope.route_load_error = false;
+      })
       $scope.$on('organization_list_updated', function () {
         init();
       });

seed/static/seed/locales/en_US.json

@@ -355,6 +355,7 @@
     "kBtu\/sq. ft.\/year": "kBtu\/sq. ft.\/year",
     "kWh\/m²\/year": "kWh\/m²\/year",
     "Label": "Label",
+    "Label must be assigned when using Valid Data Severity.": "Label must be assigned when using Valid Data Severity.",
     "Label Name": "Label Name",
     "Label updated.": "Label updated",
     "Labels": "Labels",

seed/static/seed/locales/fr_CA.json

@@ -355,6 +355,7 @@
     "kBtu\/sq. ft.\/year": "kBtu\/pi. ca.\/année",
     "kWh\/m²\/year": "kWh\/m²\/année",
     "Label": "Étiquette",
+    "Label must be assigned when using Valid Data Severity.": "L'étiquette doit être attribuée lors de l'utilisation de la gravité de données valide.",
     "Label Name": "Nom de l'étiquette",
     "Label updated.": "Étiquette mise à jour",
     "Labels": "Étiquettes",

seed/static/seed/partials/admin.html

@@ -33,7 +33,7 @@ <h2><i class="fa fa-users"></i> {$:: 'Create an organization' | translate $}</h2
                         <div class="form-group">
                             <label class="col-sm-2 control-label" for="user_emails" translate>User E-mail:</label>
                             <div class="col-sm-4">
-                                <select class="form-control" name="user_emails" id="user_emails" ng-options="u as u.email for u in org.users | orderBy:'email'" ng-model="org.email"> </select>
+                                <select class="form-control" name="user_emails" ng-options="u as u.email for u in org.users | orderBy:'email'" ng-model="org.email"></select>
                             </div>
                         </div>
                         <div class="form-group">
@@ -76,7 +76,7 @@ <h2><i class="fa fa-user"></i> {$:: 'Create a user' | translate $}</h2>
                         <div class="form-group">
                             <label class="col-sm-2 control-label" for="new_user_org_select" translate>Choose Existing Organization:</label>
                             <div class="col-sm-4">
-                                <select name="new_user_orgs" class="form-control" id="orgs" ng-options="o as o.name for o in org_user.organizations" ng-model="user.organization"></select>
+                                <select name="new_user_orgs" class="form-control" ng-options="o as o.name for o in org_user.organizations" ng-model="user.organization"></select>
                             </div>
                         </div>
                         <div class="form-group">
@@ -109,14 +109,14 @@ <h2><i class="fa fa-plus-circle"></i> {$:: 'Add/Remove a user to an organization
                             <div class="form-group">
                                 <label class="col-sm-4 control-label" for="orgs">{$:: 'Organizations' | translate $}:</label>
                                 <div class="col-sm-8">
-                                    <select name="orgs" class="form-control" id="orgs" ng-options="o as o.name for o in org_user.organizations" ng-model="org_user.organization" ng-change="get_organizations_users(org_user.organization)"></select>
+                                    <select name="orgs" class="form-control" ng-options="o as o.name for o in org_user.organizations" ng-model="org_user.organization" ng-change="get_organizations_users(org_user.organization)"></select>
                                 </div>
                             </div>
 
                             <div class="form-group">
                                 <label class="col-sm-4 control-label" for="add_emails">{$:: 'Add user to organization' | translate $}:</label>
                                 <div class="col-sm-8">
-                                    <select name="user_emails" class="form-control" id="user_emails" ng-options="u as u.email for u in org.users | orderBy:'email'" ng-model="org_user.user"></select>
+                                    <select name="user_emails" class="form-control" ng-options="u as u.email for u in org.users | orderBy:'email'" ng-model="org_user.user"></select>
                                 </div>
                             </div>
                             <div class="form-group">

seed/static/seed/partials/data_quality_admin.html

@@ -60,7 +60,7 @@ <h3 translate>Modifying Data Quality Rules</h3>
                             <th translate>Minimum</th>
                             <th translate>Maximum</th>
                             <th translate>Units</th>
-                            <th style="min-width: 100px; width: 100px;" translate>Severity</th>
+                            <th style="min-width: 100px; width: 100px;" translate>Severity Level</th>
                             <th translate>Label</th>
                             <th style="min-width: 54px; width: 54px;" translate>Delete</th>
                         </tr>
@@ -97,8 +97,9 @@ <h3 translate>Modifying Data Quality Rules</h3>
                                 <select class="form-control input-sm" ng-options="unit.id as unit.label for unit in units" ng-model="rule.units" ng-change="rule.rule_type = 1"></select>
                             </td>
                             <td>
-                                <select class="form-control input-sm" ng-model="rule.severity" ng-change="rule.rule_type = 1" ng-class="{'error-bg': rule.severity === 'error', 'warning-bg': rule.severity === 'warning'}">
+                                <select class="form-control input-sm" ng-model="rule.severity" ng-change="rule.rule_type = 1" ng-class="{'valid-bg': rule.severity === 'valid', 'error-bg': rule.severity === 'error', 'warning-bg': rule.severity === 'warning'}">
                                     <option value="error" translate>Error</option>
+                                    <option value="valid" translate>Valid Data</option>
                                     <option value="warning" translate>Warning</option>
                                 </select>
                             </td>

seed/static/seed/scss/style.scss

@@ -1974,6 +1974,10 @@ select {
     background-color: #FDF4BF;
   }
 
+  &.valid-bg {
+    background-color: #A4D1A2;
+  }
+
   > option {
     background-color: #F7F7F7;
   }

seed/views/data_quality.py

@@ -402,6 +402,12 @@ def save_data_quality_rules(self, request, pk=None):
         dq = DataQualityCheck.retrieve(organization.id)
         dq.remove_all_rules()
         for rule in updated_rules:
+            if rule['severity'] == Rule.SEVERITY_VALID and rule['status_label_id'] is None:
+                return JsonResponse({
+                    'status': 'error',
+                    'message': 'Label must be assigned when using Valid Data Severity.'
+                }, status=status.HTTP_400_BAD_REQUEST)
+
             try:
                 dq.add_rule(rule)
             except TypeError as e: