PIPE-888

pipeline/h/heuristics/fieldnames.py

@@ -1,17 +1,16 @@
 import pipeline.infrastructure.api as api
+import pipeline.infrastructure.utils as utils
 
 
 class IntentFieldnames(api.Heuristic):
     def calculate(self, ms, intent, spw=None):
         scans = ms.get_scans(scan_intent=intent, spw=spw)
-        fields = {fld for scan in scans for fld in scan.fields}
-
+        fields = utils.deduplicate(fld for scan in scans for fld in scan.fields)
         identifiers = []
-
-        # Check whether each field can be uniquely identified by its name 
+        # Check whether each field can be uniquely identified by its name
         # alone, or whether it has also been observed with other intents and
         # must therefore be referred to by numeric ID. This sometimes occurs
-        # when a field is (mistakenly) duplicated in the MS, once with 
+        # when a field is (mistakenly) duplicated in the MS, once with
         # POINTING intent and again with say BANDPASS intent.
         for field in fields:
             with_intent = ms.get_fields(name=[field.name], intent=intent)

pipeline/h/tasks/common/viewflaggers.py

@@ -3,18 +3,17 @@
 import math
 import operator
 import os
+
 import numpy as np
 
 import pipeline.infrastructure as infrastructure
 import pipeline.infrastructure.basetask as basetask
 import pipeline.infrastructure.exceptions as exceptions
 import pipeline.infrastructure.logging as logging
 import pipeline.infrastructure.vdp as vdp
-from pipeline.h.tasks.common import arrayflaggerbase
-from pipeline.h.tasks.common import flaggableviewresults
-from pipeline.h.tasks.common import ozone
+from pipeline.h.tasks.common import arrayflaggerbase, flaggableviewresults, ozone
 from pipeline.infrastructure import casa_tools
-from pipeline.infrastructure.utils.utils import find_ranges
+from pipeline.infrastructure.utils import deduplicate, find_ranges
 
 LOG = infrastructure.get_logger(__name__)
 
@@ -225,7 +224,7 @@ def prepare(self):
 
         # Create final set of flags by removing duplicates from our accumulated
         # flags
-        flags = list(set(flags))
+        flags = deduplicate(flags)
 
         # If flags were found...
         if len(flags) > 0:
@@ -1528,7 +1527,7 @@ def prepare(self):
                 break
 
         # Create final set of flags by removing duplicates from our accumulated flags
-        flags = list(set(flags))
+        flags = deduplicate(flags)
 
         # If flags were found...
         if len(flags) > 0:

pipeline/hif/tasks/fluxscale/fluxscale.py

@@ -33,8 +33,7 @@ def reference(self):
         field_fn = fieldnames.IntentFieldnames()
         reference_fields = field_fn.calculate(self.ms, self.refintent)
         # run the answer through a set, just in case there are duplicates
-        fields = {s for s in utils.safe_split(reference_fields)}
-
+        fields = utils.deduplicate(s for s in utils.safe_split(reference_fields))
         return ','.join(fields)
 
     refintent = vdp.VisDependentProperty(default='AMPLITUDE')
@@ -67,10 +66,10 @@ def transfer(self):
         references = set(self.ms.get_fields(task_arg=self.reference))
         diff = transfers.difference(references)
 
-        transfer_names = {f.name for f in diff}
+        transfer_names = sorted(f.name for f in diff)
         fields_with_name = self.ms.get_fields(name=transfer_names)
         if len(fields_with_name) is not len(diff) or len(diff) is not len(transfer_names):
-            return ','.join([str(f.id) for f in diff])
+            return ','.join(sorted(str(f.id) for f in diff))
         else:
             return ','.join(transfer_names)
 

pipeline/hif/tasks/setmodel/setmodel.py

@@ -27,7 +27,7 @@ def reference(self):
         field_fn = fieldnames.IntentFieldnames()
         reference_fields = field_fn.calculate(self.ms, self.refintent)
         # run the answer through a set, just in case there are duplicates
-        fields = {s for s in utils.safe_split(reference_fields)}
+        fields = utils.deduplicate(s for s in utils.safe_split(reference_fields))
         return ','.join(fields)
 
     @vdp.VisDependentProperty
@@ -47,10 +47,10 @@ def transfer(self):
         references = set(self.ms.get_fields(task_arg=self.reference))
         diff = transfers.difference(references)
 
-        transfer_names = {f.name for f in diff}
+        transfer_names = sorted(f.name for f in diff)
         fields_with_name = self.ms.get_fields(name=transfer_names)
         if len(fields_with_name) is not len(diff) or len(diff) is not len(transfer_names):
-            return ','.join([str(f.id) for f in diff])
+            return ','.join(sorted(str(f.id) for f in diff))
         else:
             return ','.join(transfer_names)
 

pipeline/hifa/heuristics/wvrgcal.py

@@ -2,7 +2,7 @@
 
 import pipeline.infrastructure as infrastructure
 from pipeline.infrastructure import casa_tools
-from pipeline.infrastructure.utils import list_to_str
+from pipeline.infrastructure.utils import deduplicate, list_to_str
 
 LOG = infrastructure.get_logger(__name__)
 
@@ -159,16 +159,10 @@ def _calculate_tie(self, ms):
 
         # and format it
         if len(tied) > 1:
-            # eliminate duplicate names, remove quotes from names; these cause 
+            # eliminate duplicate names, remove quotes from names; these cause
             # wvrgcal to segfault
-            tie = set()
-            for name in set(tied):
-                name = name.replace('"', '')
-                name = name.replace("'", "")
-                tie.add(name)                
-
-            tie = ','.join([name for name in tie])
-            tie = ['%s' % tie]
+            tie = deduplicate([name.replace('"', '').replace("'", '') for name in tied])
+            tie = [','.join(tie)]
             return tie
         else:
             return []

pipeline/hifa/tasks/applycal/ampphase_vs_freq_qa.py

@@ -2,7 +2,6 @@
 import functools
 import operator
 import os
-import warnings
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Tuple
@@ -13,6 +12,7 @@
 import pipeline.infrastructure.logging as logging
 from pipeline.domain import MeasurementSet
 from pipeline.domain.measures import FrequencyUnits
+
 from . import mswrapper, qa_utils
 from .qa_utils import UnitFactorType
 
@@ -917,9 +917,7 @@ def fit_angular_model(angular_model: Callable, nu: np.ndarray, angdata: np.ndarr
     """
     f_aux = lambda omega_phi: get_chi2_ang_model(angular_model, nu, omega_phi[0], omega_phi[1], angdata, angsigma)
     angle = np.ma.angle(angdata[~angdata.mask])
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore', np.exceptions.ComplexWarning)
-        phi_init = np.ma.median(angle)
+    phi_init = np.ma.median(angle)
     fitres = scipy.optimize.minimize(f_aux, np.array([0.0, phi_init]), method='L-BFGS-B')
     return fitres