Merge branch 'master' of github.com:PaulHancock/Aegean

Conflicts:
	CHANGELOG.md

.travis.yml

@@ -11,9 +11,9 @@ install:
   - tar -xvf pprocess-0.5.3.tar
   - cd pprocess-0.5.3
   - if [[ ${TRAVIS_PYTHON_VERSION} == 3.6 ]]; then 2to3 -w .; fi
-  - pip install -e .
+  - pip install .
   - cd ..
-  - pip install -e .
+  - pip install .
   - pip install coveralls
   - pip install codacy-coverage
 # command to run tests

AegeanTools/AeRes.py

@@ -0,0 +1,144 @@
+#! /usr/bin/env python
+"""
+Aegean Residual (AeRes) has the following capability:
+- convert a catalogue into an image model
+- subtract image model from image
+- write model and residual files
+"""
+
+__author__ = "Paul Hancock"
+
+import logging
+import numpy as np
+from astropy.io import fits
+from AegeanTools import catalogs, fitting, wcs_helpers
+
+FWHM2CC = 1 / (2 * np.sqrt(2 * np.log(2)))
+
+
+def load_sources(filename):
+    """
+    Open a file, read contents, return a list of all the sources in that file.
+    @param filename:
+    @return: list of OutputSource objects
+    """
+    catalog = catalogs.table_to_source_list(catalogs.load_table(filename))
+    logging.info("read {0} sources from {1}".format(len(catalog), filename))
+    return catalog
+
+
+def make_model(sources, shape, wcshelper, mask=False, frac=None, sigma=4):
+    """
+
+    @param sources: a list of AegeanTools.models.SimpleSource objects
+    @param shape: the shape of the input (and output) image
+    @param wcshelper: an AegeanTools.wcs_helpers.WCSHelper object corresponding to the input image
+    @param mask: If true then mask pixels instead of subtracting the sources
+    @param frac: pixels that are brighter than frac*peak_flux for each source will be masked if mask=True
+    @param sigma: pixels that are brighter than rms*sigma be masked if mask=True
+    @return:
+    """
+
+    # Model array
+    m = np.zeros(shape, dtype=np.float32)
+    factor = 5
+
+    i_count = 0
+    for src in sources:
+        xo, yo, sx, sy, theta = wcshelper.sky2pix_ellipse([src.ra, src.dec], src.a/3600, src.b/3600, src.pa)
+        phi = np.radians(theta)
+
+        # skip sources that have a center that is outside of the image
+        if not 0 < xo < shape[0]:
+            logging.debug("source {0} is not within image".format(src.island))
+            continue
+        if not 0 < yo < shape[1]:
+            logging.debug("source {0} is not within image".format(src.island))
+            continue
+
+        # pixels over which this model is calculated
+        xoff = factor*(abs(sx*np.cos(phi)) + abs(sy*np.sin(phi)))
+        xmin = xo - xoff
+        xmax = xo + xoff
+
+        yoff = factor*(abs(sx*np.sin(phi)) + abs(sy*np.cos(phi)))
+        ymin = yo - yoff
+        ymax = yo + yoff
+
+        # clip to the image size
+        ymin = max(np.floor(ymin), 0)
+        ymax = min(np.ceil(ymax), shape[1])
+
+        xmin = max(np.floor(xmin), 0)
+        xmax = min(np.ceil(xmax), shape[0])
+
+        if not np.all(np.isfinite([ymin, ymax, xmin, xmax])):
+            continue
+
+        if logging.getLogger().isEnabledFor(logging.DEBUG):  # pragma: no cover
+            logging.debug("Source ({0},{1})".format(src.island, src.source))
+            logging.debug(" xo, yo: {0} {1}".format(xo, yo))
+            logging.debug(" sx, sy: {0} {1}".format(sx, sy))
+            logging.debug(" theta, phi: {0} {1}".format(theta, phi))
+            logging.debug(" xoff, yoff: {0} {1}".format(xoff, yoff))
+            logging.debug(" xmin, xmax, ymin, ymax: {0}:{1} {2}:{3}".format(xmin, xmax, ymin, ymax))
+            logging.debug(" flux, sx, sy: {0} {1} {2}".format(src.peak_flux, sx, sy))
+
+        # positions for which we want to make the model
+        x, y = np.mgrid[xmin:xmax, ymin:ymax]
+        x = list(map(int, x.ravel()))
+        y = list(map(int, y.ravel()))
+
+        # TODO: understand why xo/yo -1 is needed
+        model = fitting.elliptical_gaussian(x, y, src.peak_flux, xo-1, yo-1, sx*FWHM2CC, sy*FWHM2CC, theta)
+
+        # Mask the output image if requested
+        if mask:
+            if frac is not None:
+                indices = np.where(model >= (frac*src.peak_flux))
+            else:
+                indices = np.where(model >= (sigma*src.local_rms))
+            model[indices] = np.nan
+
+        m[x, y] += model
+        i_count += 1
+    logging.info("modeled {0} sources".format(i_count))
+    return m
+
+
+def make_residual(fitsfile, catalog, rfile, mfile=None, add=False, mask=False, frac=None, sigma=4):
+    """
+
+    @param fitsfile: Input fits image filename
+    @param catalog: Input catalog filename of a type supported by Aegean
+    @param rfile: Residual image filename
+    @param mfile: model image filename
+    @param add: add the model instead of subtracting it
+    @param mask: If true then mask pixels instead of subtracting the sources
+    @param frac: pixels that are brighter than frac*peak_flux for each source will be masked if mask=True
+    @return:
+    """
+    source_list = load_sources(catalog)
+    # force two axes so that we dump redundant stokes/freq axes if they are present.
+    hdulist = fits.open(fitsfile, naxis=2)
+    # ignore dimensions of length 1
+    data = np.squeeze(hdulist[0].data)
+    header = hdulist[0].header
+
+    wcshelper = wcs_helpers.WCSHelper.from_header(header)
+
+    model = make_model(source_list, data.shape, wcshelper, mask, frac, sigma)
+
+    if add or mask:
+        residual = data + model
+    else:
+        residual = data - model
+
+    hdulist[0].data = residual
+    hdulist.writeto(rfile, overwrite=True)
+    logging.info("wrote residual to {0}".format(rfile))
+    if mfile is not None:
+        hdulist[0].data = model
+        hdulist.writeto(mfile, overwrite=True)
+        logging.info("wrote model to {0}".format(mfile))
+    return

AegeanTools/BANE.py

@@ -5,6 +5,10 @@
 The function filter_image should be imported from elsewhere and run as is.
 """
 
+__author__ = 'Paul Hancock'
+__version__ = 'v1.6.0'
+__date__ = '2018-06-28'
+
 # standard imports
 from astropy.io import fits
 import copy
@@ -21,9 +25,8 @@
 # Aegean tools
 from .fits_interp import compress
 
-__author__ = 'Paul Hancock'
-__version__ = 'v1.6.0'
-__date__ = '2018-06-28'
+# global variables for multiprocessing
+ibkg = irms = None
 
 def sigmaclip(arr, lo, hi, reps=3):
     """
@@ -390,7 +393,6 @@ def filter_mc_sharemem(filename, step_size, box_size, cores, shape, dobkg=True,
 
     img_y, img_x = shape
     # initialise some shared memory
-    alen = shape[0]*shape[1]
     if dobkg:
         global ibkg
         bkg = np.ctypeslib.as_ctypes(np.empty(shape, dtype=np.float32))

AegeanTools/MIMAS.py

@@ -1,13 +1,17 @@
 #! /usr/bin/env python
-from __future__ import print_function
-
 """
 MIMAS - The Multi-resolution Image Mask for Aegean Software
 
 TODO: Write an in/out reader for MOC formats described by
 http://arxiv.org/abs/1505.02937
 """
 
+from __future__ import print_function
+
+__author__ = "Paul Hancock"
+__version__ = 'v1.3.0'
+__date__ = '2018-05-05'
+
 import logging
 import numpy as np
 
@@ -21,10 +25,6 @@
 from .regions import Region
 from .catalogs import load_table, write_table
 
-__author__ = "Paul Hancock"
-__version__ = 'v1.3.0'
-__date__ = '2018-05-05'
-
 # seems like this fails sometimes
 if six.PY2:
     import cPickle
@@ -177,7 +177,7 @@ def mask_file(regionfile, infile, outfile, negate=False):
     region = cPickle.load(open(regionfile, 'rb'))
     try:
         wcs = pywcs.WCS(im[0].header, naxis=2)
-    except:
+    except:  # TODO: figure out what error is being thrown
         wcs = pywcs.WCS(str(im[0].header), naxis=2)
 
     if len(im[0].data.shape) > 2:

AegeanTools/__init__.py

@@ -11,7 +11,7 @@
 """
 __author__ = 'Paul Hancock'
 __version__ = '2.0.2'
-__date__ = '2018-06-21'
+__date__ = '2018-07-02'
 __citation__ = """
 % If your work makes use of AegeanTools please cite the following papers as appropriate:
 

AegeanTools/catalogs.py

@@ -1,10 +1,10 @@
 #! /usr/bin/env python
-from __future__ import print_function
-
 """
 Module for reading at writing catalogs
 """
 
+from __future__ import print_function
+
 __author__ = "Paul Hancock"
 __version__ = "1.0"
 __date__ = "2016-07-26"
@@ -26,12 +26,12 @@
 from astropy.io.votable import from_table, parse_single_table
 from astropy.io.votable import writeto as writetoVO
 
-try:
-    import h5py
-
-    hdf5_supported = True
-except ImportError:
-    hdf5_supported = False
+# try:
+#     import h5py
+#
+#     hdf5_supported = True
+# except ImportError:
+#     hdf5_supported = False
 
 import sqlite3
 
@@ -82,7 +82,6 @@ def show_formats():
         "ann": "Kvis annotation",
         "reg": "DS9 regions file",
         "fits": "FITS Binary Table",
-        "hdf5": "HDF-5 format",
         "csv": "Comma separated values",
         "tab": "tabe separated values",
         "tex": "LaTeX table format",
@@ -110,10 +109,10 @@ def get_table_formats():
     fmts = ['reg', 'fits']
     fmts.extend(['vo', 'vot', 'xml'])
     fmts.extend(['csv', 'tab', 'tex', 'html'])
-    if hdf5_supported:
-        fmts.append('hdf5')
-    else:
-        log.info("HDF5 is not supported by your environment")
+    # if hdf5_supported:
+    #     fmts.append('hdf5')
+    # else:
+    #     log.info("HDF5 is not supported by your environment")
     # assume this is always possible -> though it may not be on some systems
     fmts.extend(['db', 'sqlite'])
     return fmts
@@ -383,8 +382,10 @@ def write_catalog(filename, catalog, fmt=None, meta=None, prefix=None):
         pre = prefix + '_'
 
     def writer(filename, catalog, fmt=None):
-        # construct a dict of the data
-        # this method preserves the data types in the VOTable
+        """
+        construct a dict of the data
+        this method preserves the data types in the VOTable
+        """
         tab_dict = {}
         name_list = []
         for name in catalog[0].names:
@@ -625,12 +626,6 @@ def writeAnn(filename, catalog, fmt):
     --------
     AegeanTools.catalogs.writeIslandContours
     """
-    """
-    Input:
-        filename - file to write to
-        catalog - a list of OutputSource or SimpleSource
-        fmt - [.ann|.reg] format to use
-    """
     if fmt not in ['reg', 'ann']:
         log.warning("Format not supported for island boxes{0}".format(fmt))
         return  # fmt not supported

AegeanTools/cluster.py

@@ -1,13 +1,13 @@
 #! /usr/bin/env python
-from __future__ import print_function
-
 """
 Cluster and crossmatch tools and analysis functions.
 
 Includes:
 - DBSCAN clustering
 """
 
+from __future__ import print_function
+
 __author__= "Paul Hancock"
 
 import numpy as np
@@ -251,7 +251,7 @@ def regroup(catalog, eps, far=None, dist=norm_dist):
             log.error("catalog is not understood.")
             log.error("catalog: Should be a list of objects with the following properties[units]:\n" +
                       "ra[deg],dec[deg], a[arcsec],b[arcsec],pa[deg], peak_flux[any]")
-            raise
+            raise e
 
     log.info("Regrouping islands within catalog")
     log.debug("Calculating distances")