Merge pull request #3176 from SEED-platform/3163-fix/differentiate-nu…

…ll-and-missing

3163 fix/differentiate null and missing

seed/lib/merging/merging.py

@@ -135,7 +135,7 @@ def _merge_geocoding_results(merged_state, state1, state2, priorities, can_attrs
         setattr(merged_state, geo_attr, getattr(geo_state, geo_attr, None))
 
 
-def _merge_extra_data(ed1, ed2, priorities, recognize_empty_columns, ignore_merge_protection=False):
+def _merge_extra_data(ed1, ed2, priorities, recognize_empty_columns, ignore_merge_protection=False, state2_present_columns=None):
     """
     Merge extra_data field between two extra data dictionaries, return result.
 
@@ -153,7 +153,7 @@ def _merge_extra_data(ed1, ed2, priorities, recognize_empty_columns, ignore_merg
         if (val1 and val2) or recognize_empty:
             # decide based on the priority which one to use
             col_prior = priorities.get(key, 'Favor New')
-            if ignore_merge_protection or col_prior == 'Favor New':
+            if (ignore_merge_protection or col_prior == 'Favor New') or (state2_present_columns and key in state2_present_columns):
                 extra_data[key] = val2
             else:  # favor the existing field
                 extra_data[key] = val1
@@ -186,15 +186,19 @@ def merge_state(merged_state, state1, state2, priorities, ignore_merge_protectio
     ).values_list('column_name', flat=True)
 
     default = state2
+    state2_present_columns = None
+    if state2.import_file is not None and state2.import_file.cached_mapped_columns is not None:
+        null = None  # noqa
+        state2_present_columns = [column["to_field"] for column in eval(state2.import_file.cached_mapped_columns)]
     for attr in can_attrs:
         recognize_empty = attr in recognize_empty_columns
-
-        attr_values = [
-            value
-            for value
-            in list(can_attrs[attr].values())
-            if value is not None or recognize_empty
-        ]
+        attr_values = []
+        for value in list(can_attrs[attr].values()):
+            if value is None and recognize_empty:
+                if state2_present_columns is None or attr in state2_present_columns:
+                    attr_values.append(value)
+            elif value is not None or recognize_empty:
+                attr_values.append(value)
 
         attr_value = None
         # Two, differing values are set.
@@ -231,7 +235,8 @@ def merge_state(merged_state, state1, state2, priorities, ignore_merge_protectio
         state2.extra_data,
         priorities['extra_data'],
         recognize_empty_ed_columns,
-        ignore_merge_protection
+        ignore_merge_protection,
+        state2_present_columns
     )
 
     # merge measures, scenarios, simulations

seed/lib/merging/tests/test_merging.py

@@ -338,6 +338,7 @@ def test_merge_extra_data(self):
             'field_3': 'only_in_ed1',
             'field_4': 'only_in_ed2'
         }
+        logger.error(f'--- {result}')
         self.assertDictEqual(result, expected)
 
     def test_recognize_empty_column_setting_allows_empty_values_to_overwrite_nonempty_values(self):

seed/static/seed/partials/column_settings.html

@@ -88,7 +88,6 @@ <h4 translate>MODIFYING_COLUMN_SETTINGS</h4>
                     <td>
                         <ul>
                             <li translate>RECOGNIZE_EMPTY_EXPLANATION</li>
-                            <li style="font-weight: bold;" translate>RECOGNIZE_EMPTY_IMPORT_WARNING</li>
                         </ul>
                     </td>
                 </tr>

seed/views/v3/analyses.py

@@ -112,7 +112,6 @@ def list(self, request):
                 .distinct()
                 .order_by('-id')
             )
-            logger.error(f'--- {analyses_queryset.query}')
         else:
             analyses_queryset = (
                 Analysis.objects.filter(organization=organization_id)