Add button to match and merge cycle properties

seed/data_importer/match.py

@@ -546,7 +546,7 @@ def states_to_views(unmatched_state_ids, org, access_level_instance, cycle, Stat
     """
     table_name = StateClass.__name__
 
-    sub_progress_data = update_sub_progress_total(100, sub_progress_key)
+    # sub_progress_data = update_sub_progress_total(100, sub_progress_key)
 
     if table_name == "PropertyState":
         ViewClass = PropertyView
@@ -584,6 +584,58 @@ def states_to_views(unmatched_state_ids, org, access_level_instance, cycle, Stat
     # If one match is found, pass that along.
     # If multiple matches are found, merge them together, pass along the resulting record.
     # Otherwise, add current -State to be promoted as is.
+
+    (
+        promoted_state_ids,
+        merged_state_ids,
+        merged_between_existing_count,
+        merged_views,
+        errored_merged_states,
+        new_views,
+        errored_new_states,
+    ) = merge_unmatched_states(
+        org,
+        cycle,
+        unmatched_states,
+        promote_states,
+        column_names,
+        ViewClass,
+        StateClass,
+        table_name,
+        existing_cycle_views,
+        access_level_instance,
+        sub_progress_key,
+    )
+
+    # update merge_state while excluding any states that were a product of a previous, file-inclusive merge
+    StateClass.objects.filter(pk__in=promoted_state_ids).exclude(merge_state=MERGE_STATE_MERGED).update(merge_state=MERGE_STATE_NEW)
+    StateClass.objects.filter(pk__in=merged_state_ids).update(data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED)
+
+    return (
+        merged_between_existing_count,
+        duplicate_count,
+        list(set(merged_views)),
+        errored_merged_states,
+        new_views,
+        errored_new_states,
+    )
+
+
+def merge_unmatched_states(
+    org,
+    cycle,
+    unmatched_states,
+    promote_states,
+    column_names,
+    ViewClass,  # noqa: N803
+    StateClass,  # noqa: N803
+    table_name,
+    existing_cycle_views,
+    access_level_instance,
+    sub_progress_key,
+):
+    sub_progress_data = update_sub_progress_total(100, sub_progress_key)
+
     merged_between_existing_count = 0
     merge_state_pairs = []
     batch_size = math.ceil(len(unmatched_states) / 100)
@@ -691,9 +743,10 @@ def states_to_views(unmatched_state_ids, org, access_level_instance, cycle, Stat
     StateClass.objects.filter(pk__in=merged_state_ids).update(data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED)
 
     return (
+        promoted_state_ids,
+        merged_state_ids,
         merged_between_existing_count,
-        duplicate_count,
-        list(set(merged_views)),  # so no dupes, I think?
+        merged_views,
         errored_merged_states,
         new_views,
         errored_new_states,

seed/data_importer/tasks.py

@@ -39,7 +39,7 @@
 from seed.building_sync.building_sync import BuildingSync
 from seed.data_importer.access_level_instances_parser import AccessLevelInstancesParser
 from seed.data_importer.equivalence_partitioner import EquivalencePartitioner
-from seed.data_importer.match import match_and_link_incoming_properties_and_taxlots
+from seed.data_importer.match import match_and_link_incoming_properties_and_taxlots, merge_unmatched_states
 from seed.data_importer.meters_parser import MetersParser
 from seed.data_importer.models import STATUS_READY_TO_MERGE, ImportFile, ImportRecord
 from seed.data_importer.sensor_readings_parser import SensorsReadingsParser
@@ -86,7 +86,7 @@
 from seed.models.data_quality import DataQualityCheck, Rule
 from seed.utils.buildings import get_source_type
 from seed.utils.geocode import MapQuestAPIKeyError, create_geocoded_additional_columns, geocode_buildings
-from seed.utils.match import update_sub_progress_total
+from seed.utils.match import matching_criteria_column_names, update_sub_progress_total
 from seed.utils.ubid import decode_unique_ids
 
 _log = get_task_logger(__name__)
@@ -2007,3 +2007,42 @@ def validate_use_cases(file_pk):
     _validate_use_cases.s(file_pk, progress_data.key).apply_async()
     _log.debug(progress_data.result())
     return progress_data.result()
+
+
+@shared_task
+def match_merge_cycle_inventory(org_id, cycle_id, ali_id, state_name, sub_progress_key):
+    import logging
+
+    logging.error(">>> doing it right")
+    org = Organization.objects.filter(pk=org_id).first()
+    cycle = Cycle.objects.filter(pk=cycle_id).first()
+    access_level_instance = AccessLevelInstance.objects.filter(pk=ali_id).first()
+
+    if state_name == "TaxLotState":
+        view_klass = TaxLotView
+        state_klass = TaxLotState
+        existing_cycle_views = TaxLotView.objects.filter(cycle=cycle)
+        unmatched_states = TaxLotState.objects.filter(taxlotview__in=existing_cycle_views)
+        promote_states = TaxLotState.objects.none()
+    else:
+        view_klass = PropertyView
+        state_klass = PropertyState
+        existing_cycle_views = PropertyView.objects.filter(cycle=cycle)
+        unmatched_states = PropertyState.objects.filter(propertyview__in=existing_cycle_views)
+        promote_states = PropertyState.objects.none()
+
+    column_names = matching_criteria_column_names(org.id, "PropertyState")
+
+    merge_unmatched_states(
+        org,
+        cycle,
+        unmatched_states,
+        promote_states,
+        column_names,
+        view_klass,
+        state_klass,
+        state_name,
+        existing_cycle_views,
+        access_level_instance,
+        sub_progress_key,
+    )

seed/static/seed/js/controllers/match_merge_modal_controller.js

@@ -0,0 +1,71 @@
+/**
+ * SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
+ * See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
+ */
+angular.module('BE.seed.controller.match_merge_modal', []).controller('match_merge_modal_controller', [
+  '$scope',
+  '$uibModalInstance',
+  'spinner_utility',
+  'dataset_service',
+  'uploader_service',
+  'org',
+  'property_ubid_matching',
+  'taxlot_ubid_matching',
+  'Notification',
+  // eslint-disable-next-line func-names
+  function (
+    $scope,
+    $uibModalInstance,
+    spinner_utility,
+    dataset_service,
+    uploader_service,
+    org,
+    property_ubid_matching,
+    taxlot_ubid_matching,
+    Notification
+  ) {
+    $scope.org = org;
+    $scope.cycles = org.cycles;
+    $scope.selected_cycle = {};
+    $scope.property_ubid_matching = property_ubid_matching;
+    $scope.taxlot_ubid_matching = taxlot_ubid_matching;
+
+    $scope.uploader = {
+      in_progress: false,
+      progress: 0,
+      complete: false,
+      status_message: ''
+    };
+
+    spinner_utility.hide();
+    $scope.trigger_match_merge = () => {
+      dataset_service.match_merge_inventory($scope.selected_cycle.cycle_id).then((data) => {
+        $scope.uploader.in_progress = true;
+        uploader_service.check_progress_loop(
+          data.progress_key,
+          0,
+          1,
+          () => {
+            Notification.success('Matched and merged cycle inventory');
+            $scope.close();
+          },
+          () => {
+            console.log('failure');
+          },
+          $scope.uploader
+        );
+      });
+    };
+
+    $scope.cycle_change = () => {
+      console.log($scope.selected_cycle);
+    };
+    $scope.close = () => {
+      $uibModalInstance.close();
+    };
+
+    $scope.cancel = () => {
+      $uibModalInstance.dismiss();
+    };
+  }
+]);

seed/static/seed/js/controllers/organization_settings_controller.js

@@ -9,11 +9,12 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
   'organization_payload',
   'auth_payload',
   'analyses_service',
+  'dataset_service',
   'organization_service',
   'salesforce_mapping_service',
   'salesforce_config_service',
-  'property_column_names',
-  'taxlot_column_names',
+  'property_columns',
+  'taxlot_columns',
   'labels_payload',
   'salesforce_mappings_payload',
   'salesforce_configs_payload',
@@ -28,11 +29,12 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
     organization_payload,
     auth_payload,
     analyses_service,
+    dataset_service,
     organization_service,
     salesforce_mapping_service,
     salesforce_config_service,
-    property_column_names,
-    taxlot_column_names,
+    property_columns,
+    taxlot_columns,
     labels_payload,
     salesforce_mappings_payload,
     salesforce_configs_payload,
@@ -50,8 +52,8 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
     $scope.timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
 
     $scope.auth = auth_payload.auth;
-    $scope.property_column_names = property_column_names;
-    $scope.taxlot_column_names = taxlot_column_names;
+    $scope.property_columns = property_columns;
+    $scope.taxlot_columns = taxlot_columns;
     $scope.salesforce_mappings = salesforce_mappings_payload;
     $scope.org_static = angular.copy($scope.org);
     $scope.token_validity = { message: 'Verify Token' };
@@ -125,6 +127,12 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
       unit: null
     };
 
+    const property_ubid = $scope.property_columns.find((c) => c.column_name === 'ubid');
+    const taxlot_ubid = $scope.taxlot_columns.find((c) => c.column_name === 'ubid');
+    $scope.property_ubid_matching = property_ubid ? property_ubid.is_matching_criteria : false;
+    $scope.taxlot_ubid_matching = taxlot_ubid ? taxlot_ubid.is_matching_criteria : false;
+    $scope.ubid_matching = $scope.property_ubid_matching || $scope.taxlot_ubid_matching;
+
     // Energy type option executed within this method in order to repeat on organization update
     const get_energy_type_options = () => {
       $scope.energy_type_options = _.map($scope.org.display_meter_units, (unit, type) => ({
@@ -441,6 +449,19 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
       }
     };
 
+    $scope.open_match_merge_modal = () => {
+      $uibModal.open({
+        templateUrl: `${urls.static_url}seed/partials/match_merge_modal.html`,
+        controller: 'match_merge_modal_controller',
+        backdrop: 'static',
+        resolve: {
+          org: $scope.org,
+          property_ubid_matching: $scope.property_ubid_matching,
+          taxlot_ubid_matching: $scope.taxlot_ubid_matching
+        }
+      });
+    };
+
     /**
      * reset the last update date (to null)
      */

seed/static/seed/js/seed.js

@@ -100,6 +100,7 @@ angular.module('BE.seed.controllers', [
   'BE.seed.controller.inventory_summary',
   'BE.seed.controller.label_admin',
   'BE.seed.controller.mapping',
+  'BE.seed.controller.match_merge_modal',
   'BE.seed.controller.members',
   'BE.seed.controller.menu',
   'BE.seed.controller.merge_modal',
@@ -1137,21 +1138,37 @@ SEED_app.config([
               return organization_service.get_organization(organization_id);
             }
           ],
-          property_column_names: [
+          // property_column_names: [
+          //   '$stateParams',
+          //   'inventory_service',
+          //   ($stateParams, inventory_service) => {
+          //     const { organization_id } = $stateParams;
+          //     return inventory_service.get_property_column_names_and_ids_for_org(organization_id);
+          //   }
+          // ],
+          // taxlot_column_names: [
+          //   '$stateParams',
+          //   'inventory_service',
+          //   ($stateParams, inventory_service) => {
+          //     const { organization_id } = $stateParams;
+          //     return inventory_service.get_taxlot_column_names_for_org(organization_id);
+          //   }
+          // ],
+          property_columns: [
             '$stateParams',
             'inventory_service',
-            ($stateParams, inventory_service) => {
-              const { organization_id } = $stateParams;
-              return inventory_service.get_property_column_names_and_ids_for_org(organization_id);
-            }
+            ($stateParams, inventory_service) => inventory_service.get_property_columns().then((columns) => {
+              columns = _.reject(columns, 'related');
+              return _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related']));
+            })
           ],
-          taxlot_column_names: [
+          taxlot_columns: [
             '$stateParams',
             'inventory_service',
-            ($stateParams, inventory_service) => {
-              const { organization_id } = $stateParams;
-              return inventory_service.get_taxlot_column_names_for_org(organization_id);
-            }
+            ($stateParams, inventory_service) => inventory_service.get_taxlot_columns().then((columns) => {
+              columns = _.reject(columns, 'related');
+              return _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related']));
+            })
           ],
           labels_payload: [
             'label_service',

seed/static/seed/js/services/dataset_service.js

@@ -94,6 +94,15 @@ angular.module('BE.seed.service.dataset', []).factory('dataset_service', [
       })
       .then((response) => response.data);
 
+    dataset_service.match_merge_inventory = (cycle_id) => $http
+      .post('/api/v3/import_files/match_merge_inventory/', {
+        organization_id: user_service.get_organization().id,
+        cycle_id
+      }).then((response) => {
+        console.log('dataset service', response.data);
+        return response.data;
+      });
+
     return dataset_service;
   }
 ]);

seed/static/seed/partials/match_merge_modal.html

@@ -0,0 +1,36 @@
+<div class="modal-header">
+  <h3 class="modal-title" translate>Match and Merge Inventory</h3>
+</div>
+<div class="modal-body">
+  <div>
+    Select a cycle to re-trigger matching and merging with <strong>ubid threshold: {$ org.ubid_threshold $}</strong>. This process will match and merge all records within the selected cycle based on
+    the matching criteria. This process may take several minutes.
+  </div>
+  <ul style="margin-top: 20px">
+    <li>Property UBID Mathcing Criteria: <strong>{$ property_ubid_matching $}</strong></li>
+    <li>TaxLot UBID Matching Criteria: <strong>{$ taxlot_ubid_matching $}</strong></li>
+  </ul>
+
+  <div class="form-group cycle-select-container" ng-show="!uploader.in_progress">
+    <label for="cycle-select">Select Cycle</label>
+    <select
+      id="cycle-select"
+      class="form-control"
+      ng-model="selected_cycle"
+      ng-options="cycle.name for cycle in cycles"
+      ng-change="cycle_change()"
+      placeholder="{$:: 'Cycle Name' | translate $}"
+    ></select>
+  </div>
+  <div ng-show="uploader.in_progress" style="margin: 20px 0">Selected Cycle: <strong>{$ selected_cycle.name $}</strong></div>
+
+  <div class="progress_bar_container" ng-if="uploader.in_progress">
+    <div class="progress_bar_copy_top">Progress:</div>
+    <uib-progressbar class="progress-striped active" value="uploader.progress" type="success"></uib-progressbar>
+    <div class="progress_bar_copy_bottom">{$ uploader.progress | number:0 $}% {$:: 'Complete' | translate $} {$ uploader.status_message ? ': ' + uploader.status_message : '' $}</div>
+  </div>
+</div>
+<div class="modal-footer">
+  <button type="button" class="btn btn-primary" ng-click="close()" autofocus ng-disabled="uploader.in_progress" translate>Close</button>
+  <button type="button" class="btn btn-warning" ng-click="trigger_match_merge()" ng-disabled="uploader.in_progress" translate>Match and Merge</button>
+</div>