Skip to content

Commit

Permalink
merge
Browse files Browse the repository at this point in the history
  • Loading branch information
Ross Perry authored and Ross Perry committed Jul 2, 2024
2 parents 9a5d6c3 + b727d91 commit 6c7daad
Show file tree
Hide file tree
Showing 12 changed files with 313 additions and 26 deletions.
59 changes: 56 additions & 3 deletions seed/data_importer/match.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ def states_to_views(unmatched_state_ids, org, access_level_instance, cycle, Stat
"""
table_name = StateClass.__name__

sub_progress_data = update_sub_progress_total(100, sub_progress_key)
# sub_progress_data = update_sub_progress_total(100, sub_progress_key)

if table_name == "PropertyState":
ViewClass = PropertyView
Expand Down Expand Up @@ -584,6 +584,58 @@ def states_to_views(unmatched_state_ids, org, access_level_instance, cycle, Stat
# If one match is found, pass that along.
# If multiple matches are found, merge them together, pass along the resulting record.
# Otherwise, add current -State to be promoted as is.

(
promoted_state_ids,
merged_state_ids,
merged_between_existing_count,
merged_views,
errored_merged_states,
new_views,
errored_new_states,
) = merge_unmatched_states(
org,
cycle,
unmatched_states,
promote_states,
column_names,
ViewClass,
StateClass,
table_name,
existing_cycle_views,
access_level_instance,
sub_progress_key,
)

# update merge_state while excluding any states that were a product of a previous, file-inclusive merge
StateClass.objects.filter(pk__in=promoted_state_ids).exclude(merge_state=MERGE_STATE_MERGED).update(merge_state=MERGE_STATE_NEW)
StateClass.objects.filter(pk__in=merged_state_ids).update(data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED)

return (
merged_between_existing_count,
duplicate_count,
list(set(merged_views)),
errored_merged_states,
new_views,
errored_new_states,
)


def merge_unmatched_states(
org,
cycle,
unmatched_states,
promote_states,
column_names,
ViewClass, # noqa: N803
StateClass, # noqa: N803
table_name,
existing_cycle_views,
access_level_instance,
sub_progress_key,
):
sub_progress_data = update_sub_progress_total(100, sub_progress_key)

merged_between_existing_count = 0
merge_state_pairs = []
batch_size = math.ceil(len(unmatched_states) / 100)
Expand Down Expand Up @@ -691,9 +743,10 @@ def states_to_views(unmatched_state_ids, org, access_level_instance, cycle, Stat
StateClass.objects.filter(pk__in=merged_state_ids).update(data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_MERGED)

return (
promoted_state_ids,
merged_state_ids,
merged_between_existing_count,
duplicate_count,
list(set(merged_views)), # so no dupes, I think?
merged_views,
errored_merged_states,
new_views,
errored_new_states,
Expand Down
43 changes: 41 additions & 2 deletions seed/data_importer/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
from seed.building_sync.building_sync import BuildingSync
from seed.data_importer.access_level_instances_parser import AccessLevelInstancesParser
from seed.data_importer.equivalence_partitioner import EquivalencePartitioner
from seed.data_importer.match import match_and_link_incoming_properties_and_taxlots
from seed.data_importer.match import match_and_link_incoming_properties_and_taxlots, merge_unmatched_states
from seed.data_importer.meters_parser import MetersParser
from seed.data_importer.models import STATUS_READY_TO_MERGE, ImportFile, ImportRecord
from seed.data_importer.sensor_readings_parser import SensorsReadingsParser
Expand Down Expand Up @@ -86,7 +86,7 @@
from seed.models.data_quality import DataQualityCheck, Rule
from seed.utils.buildings import get_source_type
from seed.utils.geocode import MapQuestAPIKeyError, create_geocoded_additional_columns, geocode_buildings
from seed.utils.match import update_sub_progress_total
from seed.utils.match import matching_criteria_column_names, update_sub_progress_total
from seed.utils.ubid import decode_unique_ids

_log = get_task_logger(__name__)
Expand Down Expand Up @@ -2007,3 +2007,42 @@ def validate_use_cases(file_pk):
_validate_use_cases.s(file_pk, progress_data.key).apply_async()
_log.debug(progress_data.result())
return progress_data.result()


@shared_task
def match_merge_cycle_inventory(org_id, cycle_id, ali_id, state_name, sub_progress_key):
import logging

logging.error(">>> doing it right")
org = Organization.objects.filter(pk=org_id).first()
cycle = Cycle.objects.filter(pk=cycle_id).first()
access_level_instance = AccessLevelInstance.objects.filter(pk=ali_id).first()

if state_name == "TaxLotState":
view_klass = TaxLotView
state_klass = TaxLotState
existing_cycle_views = TaxLotView.objects.filter(cycle=cycle)
unmatched_states = TaxLotState.objects.filter(taxlotview__in=existing_cycle_views)
promote_states = TaxLotState.objects.none()
else:
view_klass = PropertyView
state_klass = PropertyState
existing_cycle_views = PropertyView.objects.filter(cycle=cycle)
unmatched_states = PropertyState.objects.filter(propertyview__in=existing_cycle_views)
promote_states = PropertyState.objects.none()

column_names = matching_criteria_column_names(org.id, "PropertyState")

merge_unmatched_states(
org,
cycle,
unmatched_states,
promote_states,
column_names,
view_klass,
state_klass,
state_name,
existing_cycle_views,
access_level_instance,
sub_progress_key,
)
71 changes: 71 additions & 0 deletions seed/static/seed/js/controllers/match_merge_modal_controller.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/**
* SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
* See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
*/
angular.module('BE.seed.controller.match_merge_modal', []).controller('match_merge_modal_controller', [
'$scope',
'$uibModalInstance',
'spinner_utility',
'dataset_service',
'uploader_service',
'org',
'property_ubid_matching',
'taxlot_ubid_matching',
'Notification',
// eslint-disable-next-line func-names
function (
$scope,
$uibModalInstance,
spinner_utility,
dataset_service,
uploader_service,
org,
property_ubid_matching,
taxlot_ubid_matching,
Notification
) {
$scope.org = org;
$scope.cycles = org.cycles;
$scope.selected_cycle = {};
$scope.property_ubid_matching = property_ubid_matching;
$scope.taxlot_ubid_matching = taxlot_ubid_matching;

$scope.uploader = {
in_progress: false,
progress: 0,
complete: false,
status_message: ''
};

spinner_utility.hide();
$scope.trigger_match_merge = () => {
dataset_service.match_merge_inventory($scope.selected_cycle.cycle_id).then((data) => {
$scope.uploader.in_progress = true;
uploader_service.check_progress_loop(
data.progress_key,
0,
1,
() => {
Notification.success('Matched and merged cycle inventory');
$scope.close();
},
() => {
console.log('failure');
},
$scope.uploader
);
});
};

$scope.cycle_change = () => {
console.log($scope.selected_cycle);
};
$scope.close = () => {
$uibModalInstance.close();
};

$scope.cancel = () => {
$uibModalInstance.dismiss();
};
}
]);
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
'organization_payload',
'auth_payload',
'analyses_service',
'dataset_service',
'organization_service',
'salesforce_mapping_service',
'salesforce_config_service',
'property_column_names',
'taxlot_column_names',
'property_columns',
'taxlot_columns',
'labels_payload',
'salesforce_mappings_payload',
'salesforce_configs_payload',
Expand All @@ -28,11 +29,12 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
organization_payload,
auth_payload,
analyses_service,
dataset_service,
organization_service,
salesforce_mapping_service,
salesforce_config_service,
property_column_names,
taxlot_column_names,
property_columns,
taxlot_columns,
labels_payload,
salesforce_mappings_payload,
salesforce_configs_payload,
Expand All @@ -50,8 +52,8 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
$scope.timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;

$scope.auth = auth_payload.auth;
$scope.property_column_names = property_column_names;
$scope.taxlot_column_names = taxlot_column_names;
$scope.property_columns = property_columns;
$scope.taxlot_columns = taxlot_columns;
$scope.salesforce_mappings = salesforce_mappings_payload;
$scope.org_static = angular.copy($scope.org);
$scope.token_validity = { message: 'Verify Token' };
Expand Down Expand Up @@ -125,6 +127,12 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
unit: null
};

const property_ubid = $scope.property_columns.find((c) => c.column_name === 'ubid');
const taxlot_ubid = $scope.taxlot_columns.find((c) => c.column_name === 'ubid');
$scope.property_ubid_matching = property_ubid ? property_ubid.is_matching_criteria : false;
$scope.taxlot_ubid_matching = taxlot_ubid ? taxlot_ubid.is_matching_criteria : false;
$scope.ubid_matching = $scope.property_ubid_matching || $scope.taxlot_ubid_matching;

// Energy type option executed within this method in order to repeat on organization update
const get_energy_type_options = () => {
$scope.energy_type_options = _.map($scope.org.display_meter_units, (unit, type) => ({
Expand Down Expand Up @@ -441,6 +449,19 @@ angular.module('BE.seed.controller.organization_settings', []).controller('organ
}
};

$scope.open_match_merge_modal = () => {
$uibModal.open({
templateUrl: `${urls.static_url}seed/partials/match_merge_modal.html`,
controller: 'match_merge_modal_controller',
backdrop: 'static',
resolve: {
org: $scope.org,
property_ubid_matching: $scope.property_ubid_matching,
taxlot_ubid_matching: $scope.taxlot_ubid_matching
}
});
};

/**
* reset the last update date (to null)
*/
Expand Down
37 changes: 27 additions & 10 deletions seed/static/seed/js/seed.js
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ angular.module('BE.seed.controllers', [
'BE.seed.controller.inventory_summary',
'BE.seed.controller.label_admin',
'BE.seed.controller.mapping',
'BE.seed.controller.match_merge_modal',
'BE.seed.controller.members',
'BE.seed.controller.menu',
'BE.seed.controller.merge_modal',
Expand Down Expand Up @@ -1150,21 +1151,37 @@ SEED_app.config([
return organization_service.get_organization(organization_id);
}
],
property_column_names: [
// property_column_names: [
// '$stateParams',
// 'inventory_service',
// ($stateParams, inventory_service) => {
// const { organization_id } = $stateParams;
// return inventory_service.get_property_column_names_and_ids_for_org(organization_id);
// }
// ],
// taxlot_column_names: [
// '$stateParams',
// 'inventory_service',
// ($stateParams, inventory_service) => {
// const { organization_id } = $stateParams;
// return inventory_service.get_taxlot_column_names_for_org(organization_id);
// }
// ],
property_columns: [
'$stateParams',
'inventory_service',
($stateParams, inventory_service) => {
const { organization_id } = $stateParams;
return inventory_service.get_property_column_names_and_ids_for_org(organization_id);
}
($stateParams, inventory_service) => inventory_service.get_property_columns().then((columns) => {
columns = _.reject(columns, 'related');
return _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related']));
})
],
taxlot_column_names: [
taxlot_columns: [
'$stateParams',
'inventory_service',
($stateParams, inventory_service) => {
const { organization_id } = $stateParams;
return inventory_service.get_taxlot_column_names_for_org(organization_id);
}
($stateParams, inventory_service) => inventory_service.get_taxlot_columns().then((columns) => {
columns = _.reject(columns, 'related');
return _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related']));
})
],
labels_payload: [
'label_service',
Expand Down
9 changes: 9 additions & 0 deletions seed/static/seed/js/services/dataset_service.js
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,15 @@ angular.module('BE.seed.service.dataset', []).factory('dataset_service', [
})
.then((response) => response.data);

dataset_service.match_merge_inventory = (cycle_id) => $http
.post('/api/v3/import_files/match_merge_inventory/', {
organization_id: user_service.get_organization().id,
cycle_id
}).then((response) => {
console.log('dataset service', response.data);
return response.data;
});

return dataset_service;
}
]);
36 changes: 36 additions & 0 deletions seed/static/seed/partials/match_merge_modal.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
<div class="modal-header">
<h3 class="modal-title" translate>Match and Merge Inventory</h3>
</div>
<div class="modal-body">
<div>
Select a cycle to re-trigger matching and merging with <strong>ubid threshold: {$ org.ubid_threshold $}</strong>. This process will match and merge all records within the selected cycle based on
the matching criteria. This process may take several minutes.
</div>
<ul style="margin-top: 20px">
<li>Property UBID Mathcing Criteria: <strong>{$ property_ubid_matching $}</strong></li>
<li>TaxLot UBID Matching Criteria: <strong>{$ taxlot_ubid_matching $}</strong></li>
</ul>

<div class="form-group cycle-select-container" ng-show="!uploader.in_progress">
<label for="cycle-select">Select Cycle</label>
<select
id="cycle-select"
class="form-control"
ng-model="selected_cycle"
ng-options="cycle.name for cycle in cycles"
ng-change="cycle_change()"
placeholder="{$:: 'Cycle Name' | translate $}"
></select>
</div>
<div ng-show="uploader.in_progress" style="margin: 20px 0">Selected Cycle: <strong>{$ selected_cycle.name $}</strong></div>

<div class="progress_bar_container" ng-if="uploader.in_progress">
<div class="progress_bar_copy_top">Progress:</div>
<uib-progressbar class="progress-striped active" value="uploader.progress" type="success"></uib-progressbar>
<div class="progress_bar_copy_bottom">{$ uploader.progress | number:0 $}% {$:: 'Complete' | translate $} {$ uploader.status_message ? ': ' + uploader.status_message : '' $}</div>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-primary" ng-click="close()" autofocus ng-disabled="uploader.in_progress" translate>Close</button>
<button type="button" class="btn btn-warning" ng-click="trigger_match_merge()" ng-disabled="uploader.in_progress" translate>Match and Merge</button>
</div>
Loading

0 comments on commit 6c7daad

Please sign in to comment.