Merge 3f3eb32 into d2de2ff

.travis.yml

@@ -2,17 +2,18 @@ language: python
 python:
   # We don't actually use the Travis Python, but this keeps it organized.
   - "2.7"
+  - "3.6"
 
 # Cache pip-installed dependencies
 cache:
     pip: true
 
 env:
-  - PYUVDATA_VERSION="@d6e1c2f80d7b5b59d7794c8f28d867a7030a7c75"
   - PYUVDATA_VERSION=""
+  #- PYUVDATA_VERSION="@d6e1c2f80d7b5b59d7794c8f28d867a7030a7c75"
 
-allow_failures:
-  - env: PYUVDATA_VERSION=""
+#allow_failures:
+#  - env: PYUVDATA_VERSION=""
 
 install:
   # ensure that we have the full tag information available for version.py
@@ -36,7 +37,7 @@ install:
   # create environment and install dependencies
   - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy nose pip matplotlib coverage
   - source activate test-environment
-  - conda install -c conda-forge healpy aipy
+  - conda install -c conda-forge healpy aipy scikit-learn
   - pip install coveralls
   - pip install h5py
   - pip install git+https://github.com/HERA-Team/pyuvdata.git$PYUVDATA_VERSION

hera_pspec/container.py

@@ -48,7 +48,7 @@ def _open(self):
             self._update_header()
 
             # Denote as Container
-            if 'pspec_type' not in self.data.attrs.keys():
+            if 'pspec_type' not in list(self.data.attrs.keys()):
                 self.data.attrs['pspec_type'] = self.__class__.__name__
 
     def _store_pspec(self, pspec_group, uvp):
@@ -88,7 +88,7 @@ def _load_pspec(self, pspec_group):
             Returns a UVPSpec object constructed from the input HDF5 group.
         """
         # Check that group is tagged as containing UVPSpec (pspec_type attribute)
-        if 'pspec_type' in pspec_group.attrs.keys():
+        if 'pspec_type' in list(pspec_group.attrs.keys()):
             if pspec_group.attrs['pspec_type'] != uvpspec.UVPSpec.__name__:
                 raise TypeError("HDF5 group is not tagged as a UVPSpec object.")
         else:
@@ -104,13 +104,13 @@ def _update_header(self):
         Update the header in the HDF5 file with useful metadata, including the
         git version of hera_pspec.
         """
-        if 'header' not in self.data.keys():
+        if 'header' not in list(self.data.keys()):
             hdr = self.data.create_group('header')
         else:
             hdr = self.data['header']
 
         # Check if versions of hera_pspec are the same
-        if 'hera_pspec.git_hash' in hdr.attrs.keys():
+        if 'hera_pspec.git_hash' in list(hdr.attrs.keys()):
             if hdr.attrs['hera_pspec.git_hash'] != version.git_hash:
                 print("WARNING: HDF5 file was created by a different version "
                       "of hera_pspec.")
@@ -138,13 +138,13 @@ def set_pspec(self, group, psname, pspec, overwrite=False):
         """
         if self.mode == 'r':
             raise IOError("HDF5 file was opened read-only; cannot write to file.")
-
-        if getattr(group, '__iter__', False):
+        
+        if isinstance(group, (tuple, list, dict)):
             raise ValueError("Only one group can be specified at a time.")
 
         # Handle input arguments that are iterable (i.e. sequences, but not str)
-        if getattr(psname, '__iter__', False):
-            if getattr(pspec, '__iter__', False) and len(pspec) == len(psname):
+        if isinstance(psname, list):
+            if isinstance(pspec, list) and len(pspec) == len(psname):
                 # Recursively call set_pspec() on each item of the list
                 for _psname, _pspec in zip(psname, pspec):
                     if not isinstance(_pspec, uvpspec.UVPSpec):
@@ -156,26 +156,26 @@ def set_pspec(self, group, psname, pspec, overwrite=False):
                 # Raise exception if psname is a list, but pspec is not
                 raise ValueError("If psname is a list, pspec must be a list of "
                                  "the same length.")
-        if getattr(pspec, '__iter__', False) \
-          and not getattr(psname, '__iter__', False):
+        if isinstance(pspec, list) and not isinstance(psname, list):
             raise ValueError("If pspec is a list, psname must also be a list.")
         # No lists should pass beyond this point
 
         # Check that input is of the correct type
         if not isinstance(pspec, uvpspec.UVPSpec):
+            print("pspec:", type(pspec), pspec)
             raise TypeError("pspec must be a UVPSpec object.")
 
         key1 = "%s" % group
         key2 = "%s" % psname
 
         # Check that the group exists
-        if key1 not in self.data.keys():
+        if key1 not in list(self.data.keys()):
             grp = self.data.create_group(key1)
         else:
             grp = self.data[key1]
 
         # Check that the psname exists
-        if key2 not in grp.keys():
+        if key2 not in list(grp.keys()):
             # Create group if it doesn't exist
             psgrp = grp.create_group(key2)
         else:
@@ -215,7 +215,7 @@ def get_pspec(self, group, psname=None):
         """
         # Check that group is in keys and extract it if so
         key1 = "%s" % group
-        if key1 in self.data.keys():
+        if key1 in list(self.data.keys()):
             grp = self.data[key1]
         else:
             raise KeyError("No group named '%s'" % key1)
@@ -225,7 +225,7 @@ def get_pspec(self, group, psname=None):
             key2 = "%s" % psname
 
             # Load power spectrum if it exists
-            if key2 in grp.keys():
+            if key2 in list(grp.keys()):
                 return self._load_pspec(grp[key2])
             else:
                 raise KeyError("No pspec named '%s' in group '%s'" % (key2, key1))
@@ -234,7 +234,7 @@ def get_pspec(self, group, psname=None):
         # Otherwise, extract all available power spectra
         uvp = []
         def pspec_filter(n, obj):
-            if u'pspec_type' in obj.attrs.keys():
+            if u'pspec_type' in list(obj.attrs.keys()):
                 uvp.append(self._load_pspec(obj))
 
         # Traverse the entire set of groups/datasets looking for pspecs
@@ -258,15 +258,15 @@ def spectra(self, group):
         """
         # Check that group is in keys and extract it if so
         key1 = "%s" % group
-        if key1 in self.data.keys():
+        if key1 in list(self.data.keys()):
             grp = self.data[key1]
         else:
             raise KeyError("No group named '%s'" % key1)
 
         # Filter to look for pspec objects
         ps_list = []
         def pspec_filter(n, obj):
-            if u'pspec_type' in obj.attrs.keys():
+            if u'pspec_type' in list(obj.attrs.keys()):
                 ps_list.append(n)
 
         # Traverse the entire set of groups/datasets looking for pspecs
@@ -282,7 +282,7 @@ def groups(self):
         group_list : list of str
             List of group names.
         """
-        groups = self.data.keys()
+        groups = list(self.data.keys())
         if u'header' in groups: groups.remove(u'header')
         return groups
 
@@ -373,7 +373,7 @@ def combine_psc_spectra(psc, groups=None, dset_split_str='_x_', ext_split_str='_
     # Iterate over groups
     for grp in groups:
         # Get spectra in this group
-        spectra = psc.data[grp].keys()
+        spectra = list(psc.data[grp].keys())
 
         # Get unique spectra by splitting and then re-joining
         unique_spectra = []

hera_pspec/grouping.py

@@ -47,7 +47,7 @@ def group_baselines(bls, Ngroups, keep_remainder=False, randomize=False,
         List of grouped baselines.
     """
     Nbls = len(bls) # Total baselines
-    n = Nbls / Ngroups # Baselines per group
+    n = Nbls // Ngroups # Baselines per group
     rem = Nbls - n*Ngroups
 
     # Sanity check on number of groups
@@ -192,15 +192,15 @@ def average_spectra(uvp_in, blpair_groups=None, time_avg=False,
 
         # Convert blpair_groups to list of blpair group integers
         if isinstance(blpair_groups[0][0], tuple):
-            new_blpair_grps = [map(lambda blp: uvp.antnums_to_blpair(blp), blpg)
+            new_blpair_grps = [[uvp.antnums_to_blpair(blp) for blp in blpg]
                                for blpg in blpair_groups]
             blpair_groups = new_blpair_grps
     else:
         # If not, each baseline pair is its own group
-        blpair_groups = map(lambda blp: [blp], np.unique(uvp.blpair_array))
+        blpair_groups = [[blp] for blp in np.unique(uvp.blpair_array)]
         assert blpair_weights is None, "Cannot specify blpair_weights if "\
                                        "blpair_groups is None."
-        blpair_weights = map(lambda blp: [1.,], np.unique(uvp.blpair_array))
+        blpair_weights = [[1.,] for blp in np.unique(uvp.blpair_array)]
 
     # Print warning if a blpair appears more than once in all of blpair_groups
     all_blpairs = [item for sublist in blpair_groups for item in sublist]
@@ -236,8 +236,8 @@ def average_spectra(uvp_in, blpair_groups=None, time_avg=False,
 
     # For baseline pairs not in blpair_groups, add them as their own group
     extra_blpairs = set(uvp.blpair_array) - set(all_blpairs)
-    blpair_groups += map(lambda blp: [blp], extra_blpairs)
-    blpair_weights += map(lambda blp: [1.,], extra_blpairs)
+    blpair_groups += [[blp] for blp in extra_blpairs]
+    blpair_weights += [[1.,] for blp in extra_blpairs]
 
     # Create new data arrays
     data_array, wgts_array = odict(), odict()
@@ -255,9 +255,9 @@ def average_spectra(uvp_in, blpair_groups=None, time_avg=False,
         spw_stats = odict([[stat, []] for stat in stat_l])
         if store_cov:
             spw_cov = []
-
+        
         # Iterate over polarizations
-        for i, p in enumerate(uvp.pol_array):
+        for i, p in enumerate(uvp.polpair_array):
             pol_data, pol_wgts, pol_ints, pol_nsmp = [], [], [], []
             pol_stats = odict([[stat, []] for stat in stat_l])
             if store_cov:
@@ -413,7 +413,8 @@ def average_spectra(uvp_in, blpair_groups=None, time_avg=False,
 
     # Update arrays
     bl_arr = np.array(sorted(set(bl_arr)))
-    bl_vecs = np.array(map(lambda bl: uvp.bl_vecs[uvp.bl_array.tolist().index(bl)], bl_arr))
+    bl_vecs = np.array([uvp.bl_vecs[uvp.bl_array.tolist().index(bl)] 
+                        for bl in bl_arr])
 
     # Assign arrays and metadata to UVPSpec object
     uvp.Ntimes = len(np.unique(time_avg_arr))
@@ -506,15 +507,15 @@ def fold_spectra(uvp):
                                                                          +leftright\
                                                                          +rightleft\
                                                                          +rightright)
-                uvp.cov_array[spw][:, :Ndlys/2, :, :] = 0.0
-                uvp.cov_array[spw][:, :, :Ndlys/2, : :] = 0.0
+                uvp.cov_array[spw][:, :Ndlys//2, :, :] = 0.0
+                uvp.cov_array[spw][:, :, :Ndlys//2, : :] = 0.0
 
             # fold stats array if it exists: sum in inverse quadrature
             if hasattr(uvp, 'stats_array'):
                 for stat in uvp.stats_array.keys():
                     left = uvp.stats_array[stat][spw][:, 1:Ndlys//2, :][:, ::-1, :]
                     right = uvp.stats_array[stat][spw][:, Ndlys//2+1:, :]
-                    uvp.stats_array[stat][spw][:, Ndlys//2+1:, :] = (np.sum([1/left**2.0, 1/right**2.0], axis=0))**(-0.5)
+                    uvp.stats_array[stat][spw][:, Ndlys//2+1:, :] = (np.sum([1./left**2.0, 1./right**2.0], axis=0))**(-0.5)
                     uvp.data_array[spw][:, :Ndlys//2, :] = np.nan
 
         else:
@@ -535,15 +536,15 @@ def fold_spectra(uvp):
                                                                          +leftright\
                                                                          +rightleft\
                                                                          +rightright)
-                uvp.cov_array[spw][:, :Ndlys/2, :, :] = 0.0
-                uvp.cov_array[spw][:, :, :Ndlys/2, : :] = 0.0
+                uvp.cov_array[spw][:, :Ndlys//2, :, :] = 0.0
+                uvp.cov_array[spw][:, :, :Ndlys//2, : :] = 0.0
 
             # fold stats array if it exists: sum in inverse quadrature
             if hasattr(uvp, 'stats_array'):
                 for stat in uvp.stats_array.keys():
                     left = uvp.stats_array[stat][spw][:, :Ndlys//2, :][:, ::-1, :]
                     right = uvp.stats_array[stat][spw][:, Ndlys//2+1:, :]
-                    uvp.stats_array[stat][spw][:, Ndlys//2+1:, :] = (np.sum([1/left**2.0, 1/right**2.0], axis=0))**(-0.5)
+                    uvp.stats_array[stat][spw][:, Ndlys//2+1:, :] = (np.sum([1./left**2.0, 1./right**2.0], axis=0))**(-0.5)
                     uvp.data_array[spw][:, :Ndlys//2, :] = np.nan
 
     uvp.folded = True
@@ -631,14 +632,15 @@ def bootstrap_average_blpairs(uvp_list, blpair_groups, time_avg=False,
 
     # Convert blpair tuples into blpair integers if necessary
     if isinstance(blpair_groups[0][0], tuple):
-        new_blp_grps = [map(lambda blp: uvputils._antnums_to_blpair(blp), blpg)
+        new_blp_grps = [[uvputils._antnums_to_blpair(blp) for blp in blpg]
                         for blpg in blpair_groups]
         blpair_groups = new_blp_grps
 
     # Homogenise input UVPSpec objects in terms of available polarizations 
     # and spectral windows
     if len(uvp_list) > 1:
-        uvp_list = uvpspec_utils.select_common(uvp_list, spws=True, pols=True, inplace=False)
+        uvp_list = uvpspec_utils.select_common(uvp_list, spws=True, 
+                                               pols=True, inplace=False)
 
     # Loop over UVPSpec objects, looking for available blpairs in each
     avail_blpairs = [np.unique(uvp.blpair_array) for uvp in uvp_list]
@@ -785,12 +787,14 @@ def bootstrap_resampled_error(uvp, blpair_groups=None, time_avg=False, Nsamples=
 
     # get all keys in uvp_avg and get data from each uvp_boot
     keys = uvp_avg.get_all_keys()
-    uvp_boot_data = odict([(k, np.array(map(lambda u: u.get_data(k), uvp_boots))) for k in keys])
+    uvp_boot_data = odict([(k, np.array([u.get_data(k) for u in uvp_boots])) 
+                           for k in keys])
 
     # calculate various error estimates
     if normal_std:
         for k in keys:
-            nstd = np.std(uvp_boot_data[k].real, axis=0) + 1j*np.std(uvp_boot_data[k].imag, axis=0)
+            nstd = np.std(uvp_boot_data[k].real, axis=0) \
+                 + 1j*np.std(uvp_boot_data[k].imag, axis=0)
             uvp_avg.set_stats("bs_std", k, nstd)
 
     if robust_std:
@@ -895,7 +899,9 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
     assert len(groups) > 0, "No groups exist in PSpecContainer"
 
     # get spectra if not fed
-    all_spectra = utils.flatten([map(lambda s: os.path.join(grp, s), psc.spectra(grp)) for grp in groups])
+    all_spectra = utils.flatten([ [os.path.join(grp, s) 
+                                   for s in psc.spectra(grp)] 
+                                 for grp in groups ])
     if spectra is None:
         spectra = all_spectra
     else:
@@ -910,11 +916,15 @@ def bootstrap_run(filename, spectra=None, blpair_groups=None, time_avg=False, Ns
         # run boostrap_resampled_error
         uvp = psc.get_pspec(grp, spc)
         (uvp_avg, uvp_boots,
-         uvp_wgts) = bootstrap_resampled_error(uvp, blpair_groups=blpair_groups, time_avg=time_avg,
-                                              Nsamples=Nsamples, seed=seed, normal_std=normal_std,
-                                              robust_std=robust_std, cintervals=cintervals,
-                                              bl_error_tol=bl_error_tol, add_to_history=add_to_history,
-                                              verbose=verbose)
+         uvp_wgts) = bootstrap_resampled_error(uvp, blpair_groups=blpair_groups, 
+                                               time_avg=time_avg,
+                                               Nsamples=Nsamples, seed=seed, 
+                                               normal_std=normal_std,
+                                               robust_std=robust_std, 
+                                               cintervals=cintervals,
+                                               bl_error_tol=bl_error_tol, 
+                                               add_to_history=add_to_history,
+                                               verbose=verbose)
 
         # set averaged uvp
         psc.set_pspec(grp, spc+"_avg", uvp_avg, overwrite=overwrite)
@@ -930,7 +940,7 @@ def get_bootstrap_run_argparser():
            description="argument parser for grouping.bootstrap_run()")
 
     def list_of_lists_of_tuples(s):
-        s = map(lambda x: map(int, x.split()), s.split(','))
+        s = [[int(_x) for _x in x.split()] for x in s.split(',')]
         return s
 
     # Add list of arguments