diff --git a/pipeline/h/heuristics/fieldnames.py b/pipeline/h/heuristics/fieldnames.py index 25ccaff862..2963f3af6a 100644 --- a/pipeline/h/heuristics/fieldnames.py +++ b/pipeline/h/heuristics/fieldnames.py @@ -1,17 +1,16 @@ import pipeline.infrastructure.api as api +import pipeline.infrastructure.utils as utils class IntentFieldnames(api.Heuristic): def calculate(self, ms, intent, spw=None): scans = ms.get_scans(scan_intent=intent, spw=spw) - fields = {fld for scan in scans for fld in scan.fields} - + fields = utils.deduplicate(fld for scan in scans for fld in scan.fields) identifiers = [] - - # Check whether each field can be uniquely identified by its name + # Check whether each field can be uniquely identified by its name # alone, or whether it has also been observed with other intents and # must therefore be referred to by numeric ID. This sometimes occurs - # when a field is (mistakenly) duplicated in the MS, once with + # when a field is (mistakenly) duplicated in the MS, once with # POINTING intent and again with say BANDPASS intent. for field in fields: with_intent = ms.get_fields(name=[field.name], intent=intent) diff --git a/pipeline/h/tasks/common/viewflaggers.py b/pipeline/h/tasks/common/viewflaggers.py index 06c68f9ab3..7fe39b2f4f 100644 --- a/pipeline/h/tasks/common/viewflaggers.py +++ b/pipeline/h/tasks/common/viewflaggers.py @@ -3,6 +3,7 @@ import math import operator import os + import numpy as np import pipeline.infrastructure as infrastructure @@ -10,11 +11,9 @@ import pipeline.infrastructure.exceptions as exceptions import pipeline.infrastructure.logging as logging import pipeline.infrastructure.vdp as vdp -from pipeline.h.tasks.common import arrayflaggerbase -from pipeline.h.tasks.common import flaggableviewresults -from pipeline.h.tasks.common import ozone +from pipeline.h.tasks.common import arrayflaggerbase, flaggableviewresults, ozone from pipeline.infrastructure import casa_tools -from pipeline.infrastructure.utils.utils import find_ranges +from pipeline.infrastructure.utils import deduplicate, find_ranges LOG = infrastructure.get_logger(__name__) @@ -225,7 +224,7 @@ def prepare(self): # Create final set of flags by removing duplicates from our accumulated # flags - flags = list(set(flags)) + flags = deduplicate(flags) # If flags were found... if len(flags) > 0: @@ -1528,7 +1527,7 @@ def prepare(self): break # Create final set of flags by removing duplicates from our accumulated flags - flags = list(set(flags)) + flags = deduplicate(flags) # If flags were found... if len(flags) > 0: diff --git a/pipeline/hif/tasks/fluxscale/fluxscale.py b/pipeline/hif/tasks/fluxscale/fluxscale.py index 3ddcbed0c3..0269298e53 100644 --- a/pipeline/hif/tasks/fluxscale/fluxscale.py +++ b/pipeline/hif/tasks/fluxscale/fluxscale.py @@ -33,8 +33,7 @@ def reference(self): field_fn = fieldnames.IntentFieldnames() reference_fields = field_fn.calculate(self.ms, self.refintent) # run the answer through a set, just in case there are duplicates - fields = {s for s in utils.safe_split(reference_fields)} - + fields = utils.deduplicate(s for s in utils.safe_split(reference_fields)) return ','.join(fields) refintent = vdp.VisDependentProperty(default='AMPLITUDE') @@ -67,10 +66,10 @@ def transfer(self): references = set(self.ms.get_fields(task_arg=self.reference)) diff = transfers.difference(references) - transfer_names = {f.name for f in diff} + transfer_names = sorted(f.name for f in diff) fields_with_name = self.ms.get_fields(name=transfer_names) if len(fields_with_name) is not len(diff) or len(diff) is not len(transfer_names): - return ','.join([str(f.id) for f in diff]) + return ','.join(sorted(str(f.id) for f in diff)) else: return ','.join(transfer_names) diff --git a/pipeline/hif/tasks/setmodel/setmodel.py b/pipeline/hif/tasks/setmodel/setmodel.py index e77f846b67..c09d71fa19 100644 --- a/pipeline/hif/tasks/setmodel/setmodel.py +++ b/pipeline/hif/tasks/setmodel/setmodel.py @@ -27,7 +27,7 @@ def reference(self): field_fn = fieldnames.IntentFieldnames() reference_fields = field_fn.calculate(self.ms, self.refintent) # run the answer through a set, just in case there are duplicates - fields = {s for s in utils.safe_split(reference_fields)} + fields = utils.deduplicate(s for s in utils.safe_split(reference_fields)) return ','.join(fields) @vdp.VisDependentProperty @@ -47,10 +47,10 @@ def transfer(self): references = set(self.ms.get_fields(task_arg=self.reference)) diff = transfers.difference(references) - transfer_names = {f.name for f in diff} + transfer_names = sorted(f.name for f in diff) fields_with_name = self.ms.get_fields(name=transfer_names) if len(fields_with_name) is not len(diff) or len(diff) is not len(transfer_names): - return ','.join([str(f.id) for f in diff]) + return ','.join(sorted(str(f.id) for f in diff)) else: return ','.join(transfer_names) diff --git a/pipeline/hifa/heuristics/wvrgcal.py b/pipeline/hifa/heuristics/wvrgcal.py index 5d3b3fb69c..7ed66dd885 100644 --- a/pipeline/hifa/heuristics/wvrgcal.py +++ b/pipeline/hifa/heuristics/wvrgcal.py @@ -2,7 +2,7 @@ import pipeline.infrastructure as infrastructure from pipeline.infrastructure import casa_tools -from pipeline.infrastructure.utils import list_to_str +from pipeline.infrastructure.utils import deduplicate, list_to_str LOG = infrastructure.get_logger(__name__) @@ -159,16 +159,10 @@ def _calculate_tie(self, ms): # and format it if len(tied) > 1: - # eliminate duplicate names, remove quotes from names; these cause + # eliminate duplicate names, remove quotes from names; these cause # wvrgcal to segfault - tie = set() - for name in set(tied): - name = name.replace('"', '') - name = name.replace("'", "") - tie.add(name) - - tie = ','.join([name for name in tie]) - tie = ['%s' % tie] + tie = deduplicate([name.replace('"', '').replace("'", '') for name in tied]) + tie = [','.join(tie)] return tie else: return [] diff --git a/pipeline/hifa/tasks/applycal/ampphase_vs_freq_qa.py b/pipeline/hifa/tasks/applycal/ampphase_vs_freq_qa.py index efaf3a5339..1ccc62604f 100644 --- a/pipeline/hifa/tasks/applycal/ampphase_vs_freq_qa.py +++ b/pipeline/hifa/tasks/applycal/ampphase_vs_freq_qa.py @@ -2,7 +2,6 @@ import functools import operator import os -import warnings from dataclasses import dataclass from pathlib import Path from typing import Callable, Tuple @@ -13,6 +12,7 @@ import pipeline.infrastructure.logging as logging from pipeline.domain import MeasurementSet from pipeline.domain.measures import FrequencyUnits + from . import mswrapper, qa_utils from .qa_utils import UnitFactorType @@ -917,9 +917,7 @@ def fit_angular_model(angular_model: Callable, nu: np.ndarray, angdata: np.ndarr """ f_aux = lambda omega_phi: get_chi2_ang_model(angular_model, nu, omega_phi[0], omega_phi[1], angdata, angsigma) angle = np.ma.angle(angdata[~angdata.mask]) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', np.exceptions.ComplexWarning) - phi_init = np.ma.median(angle) + phi_init = np.ma.median(angle) fitres = scipy.optimize.minimize(f_aux, np.array([0.0, phi_init]), method='L-BFGS-B') return fitres