Merge pull request #3021 from SEED-platform/3018_delete_column_timeou…

…t_tasks

3018 Delete Column Timeout Refactor

seed/static/seed/js/controllers/delete_column_modal_controller.js

@@ -24,10 +24,9 @@ angular.module('BE.seed.controller.delete_column_modal', [])
       $scope.delete = function () {
         $scope.state = 'pending';
         columns_service.delete_column_for_org(organization_id, column.id).then(function (result) {
-          $scope.state = 'running';
-          $scope.startTime = moment();
+          $scope.state = 'evaluating';
           $scope.interval = $interval(function () {
-            $scope.updateTime();
+            $scope.state == 'running' ? $scope.updateTime() : $scope.setRunningState();
           }, 1000);
           $scope.updateTime();
           uploader_service.check_progress_loop(result.data.progress_key, 0, 1, function (response) {
@@ -63,6 +62,12 @@ angular.module('BE.seed.controller.delete_column_modal', [])
         }
       };
 
+      $scope.setRunningState = function () {
+        $scope.eta = $scope.etaFn();
+        $scope.eta ? 
+          ($scope.state = 'running', $scope.startTime = moment()) : null
+      };
+
       $scope.updateTime = function () {
         $scope.elapsed = $scope.elapsedFn();
         $scope.eta = $scope.etaFn();

seed/static/seed/partials/delete_column_modal.html

@@ -13,6 +13,10 @@ <h4 class="modal-title" ng-switch-when="done" translate>Column Deleted</h4>
             <uib-progressbar class="progress-striped active" value="100" type="info"></uib-progressbar>
             <p>Please wait, scanning for affected records...</p>
         </div>
+        <div ng-switch-when="evaluating">
+            <uib-progressbar class="progress-striped active" value="100" type="info"></uib-progressbar>
+            <p>Please wait, evaulating affected records. This may take several minutes...</p>
+        </div>
         <div ng-switch-when="running">
             <uib-progressbar class="progress-striped active" value="progressBar.progress" type="success"><span class="status-text">{$ progressBar.status_message $}</span></uib-progressbar>
             <table class="time-table">
@@ -29,7 +33,7 @@ <h4 class="modal-title" ng-switch-when="done" translate>Column Deleted</h4>
         <div ng-switch-when="done">{$ result $}</div>
     </div>
 </div>
-<div class="modal-footer" ng-if="state !== 'pending' && state !== 'running'">
+<div class="modal-footer" ng-if="state !== 'pending' && state !== 'running' && state !== 'evaluating'">
     <button type="button" class="btn btn-default" ng-click="cancel()" ng-if="state !== 'done'" ng-disabled="state === 'running'">Cancel</button>
     <button type="button" class="btn btn-info" ng-click="delete()" ng-if="state !== 'done'" ng-disabled="state === 'running'" autofocus>Delete</button>
     <button type="button" class="btn btn-info" ng-click="refresh()" ng-if="state === 'done'" autofocus>Refresh</button>

seed/tasks.py

@@ -36,6 +36,7 @@
     TaxLotView
 )
 
+
 logger = get_task_logger(__name__)
 
 
@@ -185,8 +186,7 @@ def _finish_delete(results, org_pk, prog_key):
     return progress_data.finish_with_success()
 
 
-@shared_task
-def _finish_delete_column(results, column_id, prog_key):
+def _finish_delete_column(column_id, prog_key):
     # Delete all mappings from raw column names to the mapped column, then delete the mapped column
     column = Column.objects.get(id=column_id)
     ColumnMapping.objects.filter(column_mapped=column).delete()
@@ -322,49 +322,46 @@ def _finish_delete_cycle(cycle_id, prog_key):
 @lock_and_track
 def delete_organization_column(column_pk, org_pk, prog_key=None, chunk_size=100, *args, **kwargs):
     """Deletes an extra_data column from all merged property/taxlot states."""
-
-    column = Column.objects.get(id=column_pk, organization_id=org_pk)
-
     progress_data = ProgressData.from_key(prog_key) if prog_key else ProgressData(
         func_name='delete_organization_column', unique_id=column_pk)
 
+    _evaluate_delete_organization_column.subtask((column_pk, org_pk, progress_data.key, chunk_size)).apply_async()
+
+    return progress_data.result()
+
+
+@shared_task
+def _evaluate_delete_organization_column(column_pk, org_pk, prog_key, chunk_size, *args, **kwargs):
+    """ Find -States with column to be deleted """
+    column = Column.objects.get(id=column_pk, organization_id=org_pk)
+
     ids = []
 
     if column.table_name == 'PropertyState':
-        ids = list(
-            PropertyState.objects.filter(organization_id=org_pk, data_state=DATA_STATE_MATCHING,
-                                         extra_data__has_key=column.column_name).values_list('id', flat=True)
-        )
+        ids = PropertyState.objects.filter(organization_id=org_pk, data_state=DATA_STATE_MATCHING,
+                                           extra_data__has_key=column.column_name).values_list('id', flat=True)
     elif column.table_name == 'TaxLotState':
-        ids = list(
-            TaxLotState.objects.filter(organization_id=org_pk, data_state=DATA_STATE_MATCHING,
-                                       extra_data__has_key=column.column_name).values_list('id', flat=True)
-        )
+        ids = TaxLotState.objects.filter(organization_id=org_pk, data_state=DATA_STATE_MATCHING,
+                                         extra_data__has_key=column.column_name).values_list('id', flat=True)
 
+    progress_data = ProgressData.from_key(prog_key)
     total = len(ids)
-
-    # total is the number of records divided by the chunk size
-    progress_data.total = total / float(chunk_size)
+    progress_data.total = total / float(chunk_size) + 1
     progress_data.data['completed_records'] = 0
     progress_data.data['total_records'] = total
     progress_data.save()
 
-    tasks = []
-    # we could also use .s instead of .subtask and not wrap the *args
     for chunk_ids in batch(ids, chunk_size):
-        tasks.append(
-            _delete_organization_column_chunk.subtask(
-                (chunk_ids, column.column_name, column.table_name, progress_data.key)
-            )
+        _delete_organization_column_chunk(
+            chunk_ids, column.column_name, column.table_name, progress_data.key
         )
-    chord(tasks, interval=15)(_finish_delete_column.subtask([column.id, progress_data.key]))
 
-    return progress_data.result()
+    _finish_delete_column(column_pk, progress_data.key)
 
 
-@shared_task
 def _delete_organization_column_chunk(chunk_ids, column_name, table_name, prog_key, *args, **kwargs):
     """updates a list of ``chunk_ids`` and increments the cache"""
+
     if table_name == 'PropertyState':
         states = PropertyState.objects.filter(id__in=chunk_ids)
     else: