From 68771376ec4477cd75e966a0e3c6b7c9d961718f Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Fri, 30 Aug 2019 10:40:25 -0700
Subject: [PATCH 01/10] Initial commit for AGN number-density tests

---
 descqa/AGN_NumberDensities.py                 | 333 ++++++++++++++++++
 descqa/configs/AGN_NumberDensity_SDSSg.yaml   |   9 +
 descqa/configs/AGN_NumberDensity_SDSSi.yaml   |   9 +
 .../AGNdata/SDSS/richards_2006_table2.dat     |  25 ++
 .../AGNdata/SDSS/richards_2006_table6.dat     | 105 ++++++
 5 files changed, 481 insertions(+)
 create mode 100644 descqa/AGN_NumberDensities.py
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSg.yaml
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSi.yaml
 create mode 100644 descqa/data/AGNdata/SDSS/richards_2006_table2.dat
 create mode 100644 descqa/data/AGNdata/SDSS/richards_2006_table6.dat

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
new file mode 100644
index 00000000..76387627
--- /dev/null
+++ b/descqa/AGN_NumberDensities.py
@@ -0,0 +1,333 @@
+from __future__ import unicode_literals, absolute_import, division
+import os
+import numpy as np
+import re
+from scipy.interpolate import interp1d
+from .base import BaseValidationTest, TestResult
+from .plotting import plt
+from GCR import GCRQuery
+try:
+    from itertools import zip_longest
+except ImportError:
+    from itertools import izip_longest as zip_longest
+import matplotlib.gridspec as gridspec
+
+possible_observations = {
+    'SDSS': {
+        'filename_template': 'AGNdata/SDSS/richards_2006_table2.dat',
+        'usecols': {'g': {'dN/dm':(0, 1, 2, 5),
+                          'N<m':(0, 3, 4, 5),
+                         },
+                    'i': {'dN/dm':(0, 6, 7, 10),
+                          'N<m':(0, 8, 9, 10),
+                         },
+                   },
+        'colnames': ('mag', 'n', 'err', 'N_Q'),
+        'skiprows': 5,
+        'missing_values':'...',
+        'bin-width': 0.25,
+        'label': 'SDSS DR3',
+        'color': 'r',
+        'title': 'Richards et. al. (2006)  $\\rm M_{}<{}$, $\\rm {}<z<{}$',
+        'ytitle':{'dN/dm':'$\\rm N (deg^{{-2}} {}mag^{{-1}})$',
+                  'N<m':'$\\rm N(<{}) (deg^{{-2}})$',
+                 },
+    },
+}
+
+__all__ = ['AGN_NumberDensity']
+
+
+class AGN_NumberDensity(BaseValidationTest):
+    """
+    AGN number desnsity test
+    """
+    def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1],
+                 observation='SDSS', **kwargs):
+        """
+        parameters
+        ----------
+        band : string
+            photometric band
+
+        rest_frame_band : string
+            photometric band
+
+        Mag_lim : float
+            absolute magnitude upper limits
+
+        z_lim: list
+            lower and upper limits of redshifts to select
+
+        observation : string
+            string indicating which obsrvational data to use for validating
+        """
+        # pylint: disable=super-init-not-called
+
+        # catalog quantities needed
+        possible_mag_fields = ('mag_{}_agnonly_sdss',
+                               'mag_{}_agnonly_lsst',
+                              ) 
+        self.possible_mag_fields = [f.format(band) for f in possible_mag_fields]
+        possible_Mag_fields = ('Mag_true_{}_agnonly_sdss_z0',
+                               'Mag_true_{}_agnonly_lsst_z0',
+                              ) 
+        self.possible_Mag_fields = [f.format(rest_frame_band) for f in possible_Mag_fields]
+
+        # attach some attributes to the test
+        self.band = band
+        self.rest_frame_band = rest_frame_band
+        self.Mag_lim = Mag_lim
+        self.z_lim = list(z_lim)
+        self.font_size = kwargs.get('font_size', 18)
+        self.title_size = kwargs.get('title_size', 20)
+        self.legend_size = kwargs.get('legend_size', 16)
+        self.no_title = kwargs.get('no_title', False) 
+        self.nrows = kwargs.get('nrows', 1)
+        self.ncolumns  = kwargs.get('ncolumns', 2)
+        self.mag_lo = kwargs.get('mag_lo', 14)
+        self.mag_hi = kwargs.get('mag_hi', 24)
+        self.validation_range = kwargs.get('validation_range', (16., 19.))
+        self.truncate_cat_name = kwargs.get('truncate_cat_name', False)
+        self.figx_p = kwargs.get('figx_p', 11)
+        self.figy_p = kwargs.get('figy_p', 7)
+        self.msize = kwargs.get('msize', 6)
+        
+        # set color of lines in plots
+        colors = plt.cm.jet(np.linspace(0,1,2)) # pylint: disable=no-member
+        if band == 'g': self.line_color = colors[0]
+        elif band == 'i': self.line_color = colors[1]
+        else: self.line_color='black'
+
+        # check for validation observation
+        if not observation:
+            print('Warning: no data file supplied, no observation requested; only catalog data will be shown.')
+        elif observation not in possible_observations:
+            raise ValueError('Observation: {} not available for this test.'.format(observation))
+        else:
+            self.validation_data = self.get_validation_data(band, observation)
+
+        # prepare summary plot
+        self.summary_fig, self.summary_axs = self.setup_subplots()
+        self.first_pass = True
+
+    def get_validation_data(self, band, observation):
+        """
+        load (observational) data to use for validation test
+        """
+        data_args = possible_observations[observation]
+        data_path = os.path.join(self.data_dir, data_args['filename_template'].format(band))
+
+
+        if not os.path.exists(data_path):
+            raise ValueError("{}-band data file {} not found".format(band, data_path))
+
+        if not os.path.getsize(data_path):
+            raise ValueError("{}-band data file {} is empty".format(band, data_path))
+
+        save_keys = [k for k in data_args.keys() if 'cols' not in k and 'file' not in k and 'skip' not in k]
+        validation_data = dict(zip(save_keys, [data_args[k] for k in save_keys]))
+        validation_data['data'] = {}
+        print(validation_data)
+        
+        for k, v in data_args['usecols'][band].items():
+            data = np.genfromtxt(data_path, unpack=True, usecols=v, skip_header=data_args['skiprows'],
+                                 missing_values=data_args['missing_values'])
+            validation_data['data'][k] = dict(zip(data_args['colnames'], data))
+            
+        return validation_data
+
+
+    def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
+        """
+        """
+
+        mag_field_key = catalog_instance.first_available(*self.possible_mag_fields)
+        Mag_field_key = catalog_instance.first_available(*self.possible_Mag_fields)
+        if not mag_field_key:
+            return TestResult(skipped=True, 
+                              summary='Catalog is missing requested quantity: {}'.format(self.possible_mag_fields))
+
+        if not Mag_field_key:
+            return TestResult(skipped=True,
+                              summary='Catalog is missing requested quantity: {}'.format(self.possible_Mag_fields))
+        
+        # check to see if catalog is a light cone
+        # this is required since we must be able to calculate the angular area
+        if not catalog_instance.lightcone:
+            return TestResult(skipped=True, summary="Catalog is not a light cone.")
+
+        # check to see the angular area if an attribute of the catalog
+        try:
+            sky_area = catalog_instance.sky_area
+        except AttributeError:
+            return TestResult(skipped=True, summary="Catalog needs an attribute 'sky_area'.")
+        
+        filtername = mag_field_key.split('_')[(-1 if mag_field_key.startswith('m') else -2)].upper()  #extract filtername
+        filelabel = '_'.join((filtername, self.band))
+        
+        z_filters = ['redshift > {}'.format(self.z_lim[0]), 'redshift < {}'.format(self.z_lim[1])]
+        Mag_filters = ['{} < {}'.format(Mag_field_key, self.Mag_lim)]
+
+        # retreive data from mock catalog
+        catalog_data = catalog_instance.get_quantities([mag_field_key], filters=z_filters+ Mag_filters)
+        d = GCRQuery(*((np.isfinite, col) for col in catalog_data)).filter(catalog_data)
+        mags = d[mag_field_key]
+                       
+        #####################################################
+        # caclulate the number densities of AGN
+        #####################################################
+
+        # get the total number of AGN in catalog
+        N_tot = len(mags)
+        
+        # define the apparent magnitude bins for plotting purposes
+        dmag = self.validation_data.get('bin_width', 0.25)
+
+        mag_bins = np.arange(self.mag_lo, self.mag_hi + dmag, dmag)
+        mag_cen = (mag_bins[:-1] + mag_bins[1:])/2
+
+        # calculate differential binned data (validation data does not divide by bin width)
+        Ndm, _ = np.histogram(mags, bins=mag_bins)
+        dn = Ndm/sky_area
+        
+        # calculate N(<mag) in the bins
+        N = np.cumsum(Ndm)
+        n_cum = N/sky_area
+
+        #################################################
+        # plot the differential and cumulative apparent magnitude function
+        #################################################
+
+        fig, axs = self.setup_subplots()
+        
+        if self.truncate_cat_name:
+            catalog_name = re.split('_', catalog_name)[0]
+        results={'catalog':{}, 'data':{}}
+        colname = 'n'
+        
+        for ax, summary_ax, mag_pts, cat_data, ytit, (k, val_data) in zip(axs, self.summary_axs, 
+                                                                          [mag_cen, mag_bins[1:]],
+                                                                          [dn, n_cum],
+                                                                          [dmag, self.band],
+                                                                          self.validation_data['data'].items()):
+            # plot
+            ax[0].plot(mag_pts, cat_data, label=catalog_name, color=self.line_color)
+            ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'], label=self.validation_data['label'], 
+                           color=self.validation_data['color'], ms=self.msize, fmt='o')
+            summary_ax[0].plot(mag_pts, cat_data, label=catalog_name, color=self.line_color)
+            if self.first_pass:
+                summary_ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'],
+                                       label=self.validation_data['label'], 
+                                       color=self.validation_data['color'], ms=self.msize, fmt='o')
+                self.decorate_plot(summary_ax[0], self.validation_data['ytitle'][k].format(ytit), scale='log')
+
+            #decorate
+            self.decorate_plot(ax[0], self.validation_data['ytitle'][k].format(ytit), scale='log')
+            
+            # get fractional diffrence between the mock catalog and validation data
+            mag_val_pts, delta = self.get_frac_diff(mag_pts, cat_data, val_data['mag'], val_data['n'],
+                                       self.validation_range)
+            ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
+            summary_ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
+            self.decorate_plot(ax[1], ylabel=r'$\Delta n/n$', scale='linear', xlabel=r'$\rm m_{}$'.format(self.band))
+            if self.first_pass:
+                self.decorate_plot(summary_ax[1], ylabel=r'$\Delta n/n$', scale='linear',
+                                   xlabel=r'$\rm m_{}$'.format(self.band))
+            
+            #save plotted points
+            N_tot_data = np.sum(val_data['N_Q']) # total number of AGN
+            results['catalog'][k] = {'mag': mag_pts, colname:cat_data}
+            results['data'][k] = {'mag': val_data['mag'], colname:val_data['n'], colname + '+-':val_data['err']}
+
+        if not self.no_title:
+            fig.suptitle(self.validation_data['title'].format(self.rest_frame_band, self.Mag_lim,
+                                                              self.z_lim[0], self.z_lim[1]),
+                         fontsize=self.title_size, y=1.00)
+            self.summary_fig.suptitle(self.validation_data['title'].format(self.rest_frame_band, self.Mag_lim,
+                                                                           self.z_lim[0], self.z_lim[1]),
+                                      fontsize=self.title_size, y=1.00)
+
+        #save results for catalog and validation data in txt files
+        for filename, dtype, ntot in zip_longest((filelabel, re.sub(' ', '_', self.validation_data['label'])),
+                                                    ('catalog', 'data'),
+                                                    (N_tot, N_tot_data)):
+            if filename and dtype:
+                with open(os.path.join(output_dir, 'Nagn_' + filename + '.txt'), 'ab') as f_handle:
+                    for key, value in results[dtype].items():
+                        self.save_quantities(colname, value, f_handle,
+                                             comment=' '.join((key, ('#AGN = {}'.format(ntot) or ''))))
+
+        if self.first_pass: #turn off validation data plot in summary for remaining catalogs
+            self.first_pass = False
+            
+        #make final adjustments to plots and save figure
+        self.post_process_plot(fig)
+        fig.savefig(os.path.join(output_dir, 'Nagn_vs_mag_{}.png'.format(self.band)))
+        plt.close(fig)
+
+        return TestResult(0, inspect_only=True)
+
+    def setup_subplots(self):
+        fig = plt.figure(figsize=(self.figx_p, self.figy_p))
+        gs = fig.add_gridspec(2*self.nrows, self.ncolumns, height_ratios=[3, 1])
+        axs = []
+        axs.append([fig.add_subplot(gs[0])])
+        axs[0].append(fig.add_subplot(gs[2], sharex=axs[0][0]))
+        axs.append([fig.add_subplot(gs[1])])
+        axs[1].append(fig.add_subplot(gs[3], sharex=axs[1][0]))
+        
+        return fig, axs
+
+        
+    @staticmethod
+    def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
+        mask = (val_data > 0)
+        x = val_mags[mask]
+        y = np.log10(val_data[mask])
+        f = interp1d(x, y, fill_value='extrapolate')
+        val_mask = (mag_pts > validation_range[0]) & (mag_pts < validation_range[1])
+        interp_data = 10**f(mag_pts[val_mask])
+        
+        return mag_pts[val_mask], (cat_data[val_mask] - interp_data)/interp_data
+
+
+    def decorate_plot(self, ax, ylabel, scale='log', xlabel=None):
+        ax.set_ylabel(ylabel, size=self.font_size)
+        ax.legend(loc='best', fancybox=True, framealpha=0.5, fontsize=self.legend_size, numpoints=1)
+        ax.set_yscale(scale)
+        if xlabel:
+            ax.set_xlabel(xlabel)
+        else:
+            for axlabel in ax.get_xticklabels():
+                axlabel.set_visible(False)
+            
+        
+    @staticmethod
+    def post_process_plot(fig):
+        fig.tight_layout()
+        fig.subplots_adjust(hspace=0)
+
+
+    @staticmethod
+    def save_quantities(keyname, results, filename, comment=''):
+        if keyname in results:
+            if keyname+'-' in results and keyname+'+' in results:
+                fields = ('mag', keyname, keyname+'-', keyname+'+')
+                header = ', '.join(('Data columns are: <mag>', keyname, keyname+'-', keyname+'+'))
+            elif keyname+'+-' in results:
+                fields = ('mag', keyname, keyname+'+-')
+                header = ', '.join(('Data columns are: <mag>', keyname, keyname+'+-'))
+            else:
+                fields = ('mag', keyname)
+                header = ', '.join(('Data columns are: <mag>', keyname))
+            np.savetxt(filename, np.vstack((results[k] for k in fields)).T, fmt='%12.4e', header=': '.join([comment, header]))
+
+
+    def conclude_test(self, output_dir):
+        """
+        """
+        self.post_process_plot(self.summary_fig)
+        self.summary_fig.savefig(os.path.join(output_dir, 'summary.png'))
+        print('saved')
+        plt.close(self.summary_fig)
diff --git a/descqa/configs/AGN_NumberDensity_SDSSg.yaml b/descqa/configs/AGN_NumberDensity_SDSSg.yaml
new file mode 100644
index 00000000..6f372ba4
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSg.yaml
@@ -0,0 +1,9 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: g
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.4, 2.1]
+observation: 'SDSS'
+validation_range: [15., 19.]
+included_by_default: true
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSi.yaml b/descqa/configs/AGN_NumberDensity_SDSSi.yaml
new file mode 100644
index 00000000..c2289e24
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSi.yaml
@@ -0,0 +1,9 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: i
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [16., 19.5]
+included_by_default: true
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/data/AGNdata/SDSS/richards_2006_table2.dat b/descqa/data/AGNdata/SDSS/richards_2006_table2.dat
new file mode 100644
index 00000000..02b4e04e
--- /dev/null
+++ b/descqa/data/AGNdata/SDSS/richards_2006_table2.dat
@@ -0,0 +1,25 @@
+# RICHARDS ET AL. 2006_AJ_131_2766
+# Differential counts for g band with 0.4 < z < 2.2, M_g < 22.5, N per 0.25 mag deg^-2
+# Differential counts for i band with 0.3 < z < 2.2, M_i < 22.5, N per 0.25 mag deg^-2
+# Cumulative number counts N(<x) in units of deg^-2 for  0.3 < z < 2.2
+M         N_g  dN_g      N(<g) dN(<g)  N_Q    N_i  dN_i      N(<i) dN(<i)  N_Q
+15.475    0.00 0.00      0.00 0.00       2    0.00 0.00      0.00 0.00       1 
+15.725    0.00 0.00      0.00 0.00       3    0.00 0.00      0.00 0.00       3 
+15.975    0.00 0.00      0.01 0.00       4    0.00 0.00      0.00 0.00       2 
+16.225    0.01 0.00      0.01 0.00      11    0.01 0.00      0.01 0.00      10 
+16.475    0.01 0.00      0.02 0.00      10    0.01 0.00      0.02 0.00      12 
+16.725    0.02 0.00      0.04 0.00      32    0.01 0.00      0.03 0.00      23 
+16.975    0.06 0.01      0.10 0.01      85    0.04 0.01      0.08 0.01      67 
+17.225    0.08 0.01      0.17 0.01     117    0.07 0.01      0.15 0.01     107 
+17.475    0.12 0.01      0.29 0.01     190    0.11 0.01      0.26 0.01     174 
+17.725    0.23 0.01      0.53 0.01     357    0.21 0.01      0.47 0.01     327 
+17.975    0.31 0.01      0.83 0.01     472    0.33 0.01      0.80 0.01     511 
+18.225    0.56 0.02      1.40 0.02     868    0.55 0.02      1.35 0.02     849 
+18.475    0.78 0.02      2.18 0.02    1202    0.82 0.02      2.17 0.02    1257 
+18.725    1.19 0.03      3.37 0.03    1839    1.25 0.03      3.42 0.03    1923 
+18.975    1.70 0.03      5.07 0.03    2620    1.86 0.04      5.28 0.03    2870 
+19.225    1.61 0.03      6.68 0.03    2484    2.62 0.04      7.90 0.04    4028 
+19.475    0.56 0.02      7.24 0.02     855   ...    ...      ...  ...      900 
+19.725    0.13 0.01      7.37 0.01     206   ...    ...      ...  ...      216 
+19.975    0.03 0.01      7.41 0.00      53   ...    ...      ...  ...       55 
+20.225    0.01 0.00      7.42 0.00      19   ...    ...      ...  ...       20 
diff --git a/descqa/data/AGNdata/SDSS/richards_2006_table6.dat b/descqa/data/AGNdata/SDSS/richards_2006_table6.dat
new file mode 100644
index 00000000..2ccea17f
--- /dev/null
+++ b/descqa/data/AGNdata/SDSS/richards_2006_table6.dat
@@ -0,0 +1,105 @@
+# RICHARDS ET AL. 2006_AJ_131_2766
+#phi in units of Mpc^-3 mag^-1
+#Fill indicates how well column is covered by data
+z      M_i(z=2)   log_phi dlog_phi Fill zbar N_Q   N_Q^corr
+0.49    -26.25     -7.74    5.48   1   0.59   11   11.6 
+0.49    -25.95     -7.46    7.55   1   0.58   21   22.2 
+0.49    -25.65     -7.26    9.47   1   0.58   33   34.8 
+0.49    -25.35     -7.07    11.90   1   0.55   52   54.9 
+0.49    -25.05     -6.81    16.01   1   0.56   94   99.3 
+0.49    -24.75     -6.59    20.52   1   0.58   154   163.0 
+0.49    -24.45     -6.42    24.94   1   0.57   228   241.0 
+0.49    -24.15     -6.23    35.98   0   0.54   337   358.2 
+0.49    -23.85     -6.08    43.96   0   0.51   358   377.2 
+0.49    -23.55     -5.97    75.48   0   0.45   311   331.3 
+0.49    -23.25     -5.81    102.81   0   0.41   290   307.2 
+0.49    -22.95     -6.35    174.24   0   0.41   39   46.9 
+0.87    -27.45     -8.39    1.81   1   0.95   5   5.3 
+0.87    -27.15     -7.81    3.53   1   0.95   19   20.0 
+0.87    -26.85     -7.63    4.36   1   0.95   29   30.5 
+0.87    -26.55     -7.29    6.48   1   0.94   64   67.4 
+0.87    -26.25     -6.96    9.38   1   0.91   134   141.2 
+0.87    -25.95     -6.84    10.84   1   0.92   179   188.5 
+0.87    -25.65     -6.62    13.87   1   0.92   293   308.6 
+0.87    -25.35     -6.42    17.57   1   0.93   470   495.1 
+0.87    -25.05     -6.30    24.49   0   0.89   492   520.0 
+0.87    -24.75     -6.23    51.36   0   0.78   284   307.6 
+0.87    -24.45     -6.13    100.79   0   0.73   122   133.7 
+0.87    -24.15     -5.86    1446.59   0   0.74   9   13.8 
+1.25    -28.05     -8.15    2.06   1   1.31   12   12.6 
+1.25    -27.75     -7.86    2.85   1   1.32   23   24.2 
+1.25    -27.45     -7.76    3.20   1   1.27   29   30.5 
+1.25    -27.15     -7.36    5.07   1   1.30   73   76.8 
+1.25    -26.85     -7.08    7.00   1   1.30   139   146.3 
+1.25    -26.55     -6.83    9.37   1   1.29   249   262.1 
+1.25    -26.25     -6.58    12.44   1   1.29   439   462.1 
+1.25    -25.95     -6.33    16.62   1   1.28   783   824.2 
+1.25    -25.65     -6.24    22.56   0   1.23   733   773.5 
+1.25    -25.35     -6.12    48.01   0   1.13   360   382.8 
+1.25    -25.05     -5.74    1332.39   0   1.09   12   14.6 
+1.63    -28.65     -8.25    1.70   1   1.66   11   11.6 
+1.63    -28.35     -8.08    2.06   1   1.67   16   16.8 
+1.63    -28.05     -7.78    2.91   1   1.68   32   33.7 
+1.63    -27.75     -7.39    4.57   1   1.66   79   83.2 
+1.63    -27.45     -7.12    6.23   1   1.67   147   154.7 
+1.63    -27.15     -6.86    8.43   1   1.66   269   283.2 
+1.63    -26.85     -6.63    10.94   1   1.65   453   476.8 
+1.63    -26.55     -6.40    14.35   1   1.66   779   820.0 
+1.63    -26.25     -6.26    19.09   0   1.62   892   940.9 
+1.63    -25.95     -6.14    38.86   0   1.52   477   506.0 
+2.01    -28.95     -8.41    1.37   1   1.99   8   8.4 
+2.01    -28.65     -7.94    2.37   1   2.02   24   25.3 
+2.01    -28.35     -7.75    2.94   1   2.02   37   39.0 
+2.01    -28.05     -7.37    4.53   1   2.01   88   92.7 
+2.01    -27.75     -7.15    5.86   1   2.03   147   155.0 
+2.01    -27.45     -6.87    8.03   1   2.02   276   291.0 
+2.01    -27.15     -6.67    10.22   1   2.01   447   471.1 
+2.01    -26.85     -6.45    13.97   0   1.99   648   682.7 
+2.01    -26.55     -6.24    40.12   0   1.90   430   464.3 
+2.01    -26.25     -6.03    495.10   0   1.87   30   39.4 
+2.40    -28.95     -8.12    2.07   1   2.44   14   17.9 
+2.40    -28.65     -8.03    2.30   1   2.41   17   21.8 
+2.40    -28.35     -7.59    3.71   1   2.37   48   59.7 
+2.40    -28.05     -7.29    5.28   1   2.37   98   121.2 
+2.40    -27.75     -7.05    7.01   1   2.37   164   208.1 
+2.40    -27.45     -6.82    9.50   0   2.38   265   356.0 
+2.40    -27.15     -6.61    22.11   0   2.30   246   309.0 
+2.40    -26.85     -6.24    249.51   0   2.25   37   48.8 
+2.80    -29.55     -8.57    1.76   1   2.70   3   6.2 
+2.80    -29.25     -8.34    1.84   1   2.82   7   10.6 
+2.80    -28.95     -8.08    6.51   1   2.82   10   19.2 
+2.80    -28.65     -7.95    3.68   1   2.80   12   25.9 
+2.80    -28.35     -7.65    11.94   1   2.83   24   51.5 
+2.80    -28.05     -7.33    15.75   1   2.81   50   107.9 
+2.80    -27.75     -7.11    20.63   1   2.82   75   180.9 
+2.80    -27.45     -7.07    24.17   1   2.79   73   199.8 
+3.25    -29.55     -8.54    1.10   1   3.35   7   8.2 
+3.25    -29.25     -8.30    1.45   1   3.25   12   14.2 
+3.25    -28.95     -8.33    1.40   1   3.20   11   13.1 
+3.25    -28.65     -7.94    2.18   1   3.25   28   32.5 
+3.25    -28.35     -7.64    3.12   1   3.23   54   64.6 
+3.25    -28.05     -7.47    3.89   1   3.25   77   95.9 
+3.25    -27.75     -7.32    4.72   1   3.24   103   134.3 
+3.25    -27.45     -7.09    6.47   1   3.21   161   228.8 
+3.25    -27.15     -6.99    7.48   1   3.22   191   287.9 
+3.25    -26.85     -6.80    10.40   0   3.20   234   377.5 
+3.25    -26.55     -6.63    39.58   0   3.11   52   103.7 
+3.75    -28.95     -8.24    1.56   1   3.69   14   15.7 
+3.75    -28.65     -8.02    2.03   1   3.67   22   25.7 
+3.75    -28.35     -7.65    3.06   1   3.77   54   60.5 
+3.75    -28.05     -7.64    3.11   1   3.76   55   62.1 
+3.75    -27.75     -7.44    4.07   1   3.74   83   99.3 
+3.75    -27.45     -7.27    4.96   1   3.76   119   144.2 
+3.75    -27.15     -7.16    6.59   0   3.72   117   158.4 
+3.75    -26.85     -6.89    25.48   0   3.59   27   54.6 
+4.25    -29.25     -8.48    1.16   1   4.25   8   8.4 
+4.25    -28.65     -8.18    1.64   1   4.22   16   16.9 
+4.25    -28.35     -8.11    1.79   1   4.19   19   20.0 
+4.25    -28.05     -7.91    2.26   1   4.21   30   31.8 
+4.25    -27.75     -7.81    2.53   1   4.24   38   40.2 
+4.25    -27.45     -7.59    3.84   0   4.16   44   47.1 
+4.75    -28.65     -8.52    1.14   1   4.66   7   7.4 
+4.75    -28.35     -8.46    1.22   1   4.71   8   8.4 
+4.75    -28.05     -8.00    2.07   1   4.66   23   24.2 
+4.75    -27.75     -8.06    1.94   0   4.66   20   21.1 
+4.75    -27.45     -7.78    20.57   0   4.62   6   8.3 

From ec9f6f2670681db03a366224d52e057cc35c8c41 Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Fri, 30 Aug 2019 12:03:26 -0700
Subject: [PATCH 02/10] Fix travis issues

---
 descqa/AGN_NumberDensities.py | 77 ++++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index 76387627..0feec33c 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -1,9 +1,8 @@
 from __future__ import unicode_literals, absolute_import, division
 import os
-import numpy as np
 import re
+import numpy as np
 from scipy.interpolate import interp1d
-from .base import BaseValidationTest, TestResult
 from .plotting import plt
 from GCR import GCRQuery
 try:
@@ -11,6 +10,7 @@
 except ImportError:
     from itertools import izip_longest as zip_longest
 import matplotlib.gridspec as gridspec
+from .base import BaseValidationTest, TestResult
 
 possible_observations = {
     'SDSS': {
@@ -67,11 +67,11 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1
         # catalog quantities needed
         possible_mag_fields = ('mag_{}_agnonly_sdss',
                                'mag_{}_agnonly_lsst',
-                              ) 
+                              )
         self.possible_mag_fields = [f.format(band) for f in possible_mag_fields]
         possible_Mag_fields = ('Mag_true_{}_agnonly_sdss_z0',
                                'Mag_true_{}_agnonly_lsst_z0',
-                              ) 
+                              )
         self.possible_Mag_fields = [f.format(rest_frame_band) for f in possible_Mag_fields]
 
         # attach some attributes to the test
@@ -82,9 +82,9 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1
         self.font_size = kwargs.get('font_size', 18)
         self.title_size = kwargs.get('title_size', 20)
         self.legend_size = kwargs.get('legend_size', 16)
-        self.no_title = kwargs.get('no_title', False) 
+        self.no_title = kwargs.get('no_title', False)
         self.nrows = kwargs.get('nrows', 1)
-        self.ncolumns  = kwargs.get('ncolumns', 2)
+        self.ncolumns = kwargs.get('ncolumns', 2)
         self.mag_lo = kwargs.get('mag_lo', 14)
         self.mag_hi = kwargs.get('mag_hi', 24)
         self.validation_range = kwargs.get('validation_range', (16., 19.))
@@ -92,12 +92,15 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1
         self.figx_p = kwargs.get('figx_p', 11)
         self.figy_p = kwargs.get('figy_p', 7)
         self.msize = kwargs.get('msize', 6)
-        
+
         # set color of lines in plots
-        colors = plt.cm.jet(np.linspace(0,1,2)) # pylint: disable=no-member
-        if band == 'g': self.line_color = colors[0]
-        elif band == 'i': self.line_color = colors[1]
-        else: self.line_color='black'
+        colors = plt.cm.jet(np.linspace(0, 1, 2)) # pylint: disable=no-member
+        if band == 'g':
+            self.line_color = colors[0]
+        elif band == 'i':
+            self.line_color = colors[1]
+        else:
+            self.line_color = 'black'
 
         # check for validation observation
         if not observation:
@@ -128,7 +131,6 @@ def get_validation_data(self, band, observation):
         save_keys = [k for k in data_args.keys() if 'cols' not in k and 'file' not in k and 'skip' not in k]
         validation_data = dict(zip(save_keys, [data_args[k] for k in save_keys]))
         validation_data['data'] = {}
-        print(validation_data)
         
         for k, v in data_args['usecols'][band].items():
             data = np.genfromtxt(data_path, unpack=True, usecols=v, skip_header=data_args['skiprows'],
@@ -141,17 +143,16 @@ def get_validation_data(self, band, observation):
     def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         """
         """
-
         mag_field_key = catalog_instance.first_available(*self.possible_mag_fields)
         Mag_field_key = catalog_instance.first_available(*self.possible_Mag_fields)
         if not mag_field_key:
-            return TestResult(skipped=True, 
+            return TestResult(skipped=True,
                               summary='Catalog is missing requested quantity: {}'.format(self.possible_mag_fields))
 
         if not Mag_field_key:
             return TestResult(skipped=True,
                               summary='Catalog is missing requested quantity: {}'.format(self.possible_Mag_fields))
-        
+
         # check to see if catalog is a light cone
         # this is required since we must be able to calculate the angular area
         if not catalog_instance.lightcone:
@@ -162,10 +163,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
             sky_area = catalog_instance.sky_area
         except AttributeError:
             return TestResult(skipped=True, summary="Catalog needs an attribute 'sky_area'.")
-        
+
         filtername = mag_field_key.split('_')[(-1 if mag_field_key.startswith('m') else -2)].upper()  #extract filtername
         filelabel = '_'.join((filtername, self.band))
-        
+
         z_filters = ['redshift > {}'.format(self.z_lim[0]), 'redshift < {}'.format(self.z_lim[1])]
         Mag_filters = ['{} < {}'.format(Mag_field_key, self.Mag_lim)]
 
@@ -173,14 +174,14 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         catalog_data = catalog_instance.get_quantities([mag_field_key], filters=z_filters+ Mag_filters)
         d = GCRQuery(*((np.isfinite, col) for col in catalog_data)).filter(catalog_data)
         mags = d[mag_field_key]
-                       
+
         #####################################################
         # caclulate the number densities of AGN
         #####################################################
 
         # get the total number of AGN in catalog
         N_tot = len(mags)
-        
+
         # define the apparent magnitude bins for plotting purposes
         dmag = self.validation_data.get('bin_width', 0.25)
 
@@ -190,7 +191,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         # calculate differential binned data (validation data does not divide by bin width)
         Ndm, _ = np.histogram(mags, bins=mag_bins)
         dn = Ndm/sky_area
-        
+
         # calculate N(<mag) in the bins
         N = np.cumsum(Ndm)
         n_cum = N/sky_area
@@ -200,17 +201,17 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         #################################################
 
         fig, axs = self.setup_subplots()
-        
+
         if self.truncate_cat_name:
             catalog_name = re.split('_', catalog_name)[0]
-        results={'catalog':{}, 'data':{}}
+        results = {'catalog':{}, 'data':{}}
         colname = 'n'
-        
+
         for ax, summary_ax, mag_pts, cat_data, ytit, (k, val_data) in zip(axs, self.summary_axs, 
                                                                           [mag_cen, mag_bins[1:]],
                                                                           [dn, n_cum],
                                                                           [dmag, self.band],
-                                                                          self.validation_data['data'].items()):
+                                                                           self.validation_data['data'].items()):
             # plot
             ax[0].plot(mag_pts, cat_data, label=catalog_name, color=self.line_color)
             ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'], label=self.validation_data['label'], 
@@ -227,14 +228,14 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
             
             # get fractional diffrence between the mock catalog and validation data
             mag_val_pts, delta = self.get_frac_diff(mag_pts, cat_data, val_data['mag'], val_data['n'],
-                                       self.validation_range)
+                                                    self.validation_range)
             ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
             summary_ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
             self.decorate_plot(ax[1], ylabel=r'$\Delta n/n$', scale='linear', xlabel=r'$\rm m_{}$'.format(self.band))
             if self.first_pass:
                 self.decorate_plot(summary_ax[1], ylabel=r'$\Delta n/n$', scale='linear',
                                    xlabel=r'$\rm m_{}$'.format(self.band))
-            
+
             #save plotted points
             N_tot_data = np.sum(val_data['N_Q']) # total number of AGN
             results['catalog'][k] = {'mag': mag_pts, colname:cat_data}
@@ -250,8 +251,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         #save results for catalog and validation data in txt files
         for filename, dtype, ntot in zip_longest((filelabel, re.sub(' ', '_', self.validation_data['label'])),
-                                                    ('catalog', 'data'),
-                                                    (N_tot, N_tot_data)):
+                                                 ('catalog', 'data'),
+                                                 (N_tot, N_tot_data)):
             if filename and dtype:
                 with open(os.path.join(output_dir, 'Nagn_' + filename + '.txt'), 'ab') as f_handle:
                     for key, value in results[dtype].items():
@@ -260,7 +261,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         if self.first_pass: #turn off validation data plot in summary for remaining catalogs
             self.first_pass = False
-            
+
         #make final adjustments to plots and save figure
         self.post_process_plot(fig)
         fig.savefig(os.path.join(output_dir, 'Nagn_vs_mag_{}.png'.format(self.band)))
@@ -268,7 +269,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         return TestResult(0, inspect_only=True)
 
+
     def setup_subplots(self):
+        """
+        """
         fig = plt.figure(figsize=(self.figx_p, self.figy_p))
         gs = fig.add_gridspec(2*self.nrows, self.ncolumns, height_ratios=[3, 1])
         axs = []
@@ -276,12 +280,14 @@ def setup_subplots(self):
         axs[0].append(fig.add_subplot(gs[2], sharex=axs[0][0]))
         axs.append([fig.add_subplot(gs[1])])
         axs[1].append(fig.add_subplot(gs[3], sharex=axs[1][0]))
-        
+
         return fig, axs
 
         
     @staticmethod
     def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
+        """
+        """
         mask = (val_data > 0)
         x = val_mags[mask]
         y = np.log10(val_data[mask])
@@ -293,6 +299,8 @@ def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
 
 
     def decorate_plot(self, ax, ylabel, scale='log', xlabel=None):
+        """
+        """
         ax.set_ylabel(ylabel, size=self.font_size)
         ax.legend(loc='best', fancybox=True, framealpha=0.5, fontsize=self.legend_size, numpoints=1)
         ax.set_yscale(scale)
@@ -301,16 +309,20 @@ def decorate_plot(self, ax, ylabel, scale='log', xlabel=None):
         else:
             for axlabel in ax.get_xticklabels():
                 axlabel.set_visible(False)
-            
-        
+
+
     @staticmethod
     def post_process_plot(fig):
+        """
+        """
         fig.tight_layout()
         fig.subplots_adjust(hspace=0)
 
 
     @staticmethod
     def save_quantities(keyname, results, filename, comment=''):
+        """
+        """
         if keyname in results:
             if keyname+'-' in results and keyname+'+' in results:
                 fields = ('mag', keyname, keyname+'-', keyname+'+')
@@ -326,6 +338,7 @@ def save_quantities(keyname, results, filename, comment=''):
 
     def conclude_test(self, output_dir):
         """
+        output_dir: output directory
         """
         self.post_process_plot(self.summary_fig)
         self.summary_fig.savefig(os.path.join(output_dir, 'summary.png'))

From 2374aa7c2dfbac90927faa99ac406ebfa45a5930 Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Fri, 30 Aug 2019 12:49:38 -0700
Subject: [PATCH 03/10] More travis issues fixed

---
 descqa/AGN_NumberDensities.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index 0feec33c..6b72d846 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -3,13 +3,12 @@
 import re
 import numpy as np
 from scipy.interpolate import interp1d
-from .plotting import plt
 from GCR import GCRQuery
 try:
     from itertools import zip_longest
 except ImportError:
     from itertools import izip_longest as zip_longest
-import matplotlib.gridspec as gridspec
+from .plotting import plt
 from .base import BaseValidationTest, TestResult
 
 possible_observations = {
@@ -63,7 +62,8 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1
             string indicating which obsrvational data to use for validating
         """
         # pylint: disable=super-init-not-called
-
+        # pylint: disable=too-many-instance-attributes
+        
         # catalog quantities needed
         possible_mag_fields = ('mag_{}_agnonly_sdss',
                                'mag_{}_agnonly_lsst',
@@ -131,18 +131,19 @@ def get_validation_data(self, band, observation):
         save_keys = [k for k in data_args.keys() if 'cols' not in k and 'file' not in k and 'skip' not in k]
         validation_data = dict(zip(save_keys, [data_args[k] for k in save_keys]))
         validation_data['data'] = {}
-        
+
         for k, v in data_args['usecols'][band].items():
             data = np.genfromtxt(data_path, unpack=True, usecols=v, skip_header=data_args['skiprows'],
                                  missing_values=data_args['missing_values'])
             validation_data['data'][k] = dict(zip(data_args['colnames'], data))
-            
+
         return validation_data
 
 
     def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         """
         """
+
         mag_field_key = catalog_instance.first_available(*self.possible_mag_fields)
         Mag_field_key = catalog_instance.first_available(*self.possible_Mag_fields)
         if not mag_field_key:
@@ -211,15 +212,15 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
                                                                           [mag_cen, mag_bins[1:]],
                                                                           [dn, n_cum],
                                                                           [dmag, self.band],
-                                                                           self.validation_data['data'].items()):
+                                                                          self.validation_data['data'].items()):
             # plot
             ax[0].plot(mag_pts, cat_data, label=catalog_name, color=self.line_color)
-            ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'], label=self.validation_data['label'], 
+            ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'], label=self.validation_data['label'],
                            color=self.validation_data['color'], ms=self.msize, fmt='o')
             summary_ax[0].plot(mag_pts, cat_data, label=catalog_name, color=self.line_color)
             if self.first_pass:
                 summary_ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'],
-                                       label=self.validation_data['label'], 
+                                       label=self.validation_data['label'],
                                        color=self.validation_data['color'], ms=self.msize, fmt='o')
                 self.decorate_plot(summary_ax[0], self.validation_data['ytitle'][k].format(ytit), scale='log')
 
@@ -283,7 +284,7 @@ def setup_subplots(self):
 
         return fig, axs
 
-        
+
     @staticmethod
     def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
         """
@@ -294,7 +295,7 @@ def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
         f = interp1d(x, y, fill_value='extrapolate')
         val_mask = (mag_pts > validation_range[0]) & (mag_pts < validation_range[1])
         interp_data = 10**f(mag_pts[val_mask])
-        
+
         return mag_pts[val_mask], (cat_data[val_mask] - interp_data)/interp_data
 
 

From 0c225bc5f672fa9d4dd4a49997d83f1ff1b05e96 Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Mon, 9 Sep 2019 14:05:01 -0700
Subject: [PATCH 04/10] Updates to implement agn-fraction choices

---
 descqa/AGN_NumberDensities.py                 | 82 +++++++++++++------
 descqa/configs/AGN_NumberDensity_SDSSg.yaml   |  6 +-
 .../AGN_NumberDensity_SDSSg_frac.01.yaml      | 11 +++
 descqa/configs/AGN_NumberDensity_SDSSi.yaml   |  2 +
 .../AGN_NumberDensity_SDSSi_frac.01.yaml      | 11 +++
 .../AGNdata/SDSS/richards_2006_table2.dat     | 41 +++++-----
 .../SDSS/richards_2006_table2.dat.original    | 25 ++++++
 7 files changed, 132 insertions(+), 46 deletions(-)
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml
 create mode 100644 descqa/data/AGNdata/SDSS/richards_2006_table2.dat.original

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index 6b72d846..082361a3 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -27,7 +27,7 @@
         'bin-width': 0.25,
         'label': 'SDSS DR3',
         'color': 'r',
-        'title': 'Richards et. al. (2006)  $\\rm M_{}<{}$, $\\rm {}<z<{}$',
+        'title': 'Richards et. al. (2006)  $\\rm M^{{AGN+galaxy}}_{}<{}$, $\\rm {}<z<{}$',
         'ytitle':{'dN/dm':'$\\rm N (deg^{{-2}} {}mag^{{-1}})$',
                   'N<m':'$\\rm N(<{}) (deg^{{-2}})$',
                  },
@@ -63,22 +63,33 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1
         """
         # pylint: disable=super-init-not-called
         # pylint: disable=too-many-instance-attributes
-        
+
+        self.no_agn_extinction = kwargs.get('no_agn_extinction', True)        
         # catalog quantities needed
-        possible_mag_fields = ('mag_{}_agnonly_sdss',
-                               'mag_{}_agnonly_lsst',
-                              )
-        self.possible_mag_fields = [f.format(band) for f in possible_mag_fields]
-        possible_Mag_fields = ('Mag_true_{}_agnonly_sdss_z0',
-                               'Mag_true_{}_agnonly_lsst_z0',
+        noagnext = 'no_agn_extinction_' if self.no_agn_extinction else ''
+        possible_mag_fields = ('mag_{}_{}sdss',
+                               'mag_{}_{}lsst',
                               )
-        self.possible_Mag_fields = [f.format(rest_frame_band) for f in possible_Mag_fields]
+        self.possible_mag_fields = [f.format(band, noagnext) for f in possible_mag_fields]
 
+        # cut on host galaxy + AGN flux 
+        possible_Mag_fields = ('Mag_true_{}_{}sdss_z0',
+                               'Mag_true_{}_{}lsst_z0',
+                              )
+        self.possible_Mag_fields = [f.format(rest_frame_band, noagnext) for f in possible_Mag_fields]
+        
+        # for fraction cut
+        possible_agn_mag_fields = ('mag_{}_agnonly_{}sdss',
+                                     'mag_{}_agnonly_{}lsst',
+                                    )
+        self.possible_agn_mag_fields = [f.format(band, noagnext) for f in possible_agn_mag_fields] 
+        
         # attach some attributes to the test
         self.band = band
         self.rest_frame_band = rest_frame_band
         self.Mag_lim = Mag_lim
         self.z_lim = list(z_lim)
+        self.agn_flux_fraction = kwargs.get('agn_flux_fraction', 0.5)
         self.font_size = kwargs.get('font_size', 18)
         self.title_size = kwargs.get('title_size', 20)
         self.legend_size = kwargs.get('legend_size', 16)
@@ -146,13 +157,16 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         mag_field_key = catalog_instance.first_available(*self.possible_mag_fields)
         Mag_field_key = catalog_instance.first_available(*self.possible_Mag_fields)
+        mag_agn_key = catalog_instance.first_available(*self.possible_agn_mag_fields)
         if not mag_field_key:
             return TestResult(skipped=True,
                               summary='Catalog is missing requested quantity: {}'.format(self.possible_mag_fields))
-
         if not Mag_field_key:
             return TestResult(skipped=True,
                               summary='Catalog is missing requested quantity: {}'.format(self.possible_Mag_fields))
+        if not mag_agn_key:
+            return TestResult(skipped=True,
+                              summary='Catalog is missing requested quantity: {}'.format(self.possible_agn_mag_fields))
 
         # check to see if catalog is a light cone
         # this is required since we must be able to calculate the angular area
@@ -170,19 +184,31 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         z_filters = ['redshift > {}'.format(self.z_lim[0]), 'redshift < {}'.format(self.z_lim[1])]
         Mag_filters = ['{} < {}'.format(Mag_field_key, self.Mag_lim)]
-
+        
         # retreive data from mock catalog
-        catalog_data = catalog_instance.get_quantities([mag_field_key], filters=z_filters+ Mag_filters)
+        catalog_data = catalog_instance.get_quantities([mag_field_key, mag_agn_key], filters=z_filters+ Mag_filters)
         d = GCRQuery(*((np.isfinite, col) for col in catalog_data)).filter(catalog_data)
-        mags = d[mag_field_key]
+        
+        #select point-like sources according to specified agn_flux_fraction
+        N_all = len(d[mag_field_key])
+        fluxid = 'AGN+galaxy'
+        if self.agn_flux_fraction > 0. and self.agn_flux_fraction < 1.:
+            point_like_mask = (d[mag_agn_key] - d[mag_field_key] < -2.5*np.log10(self.agn_flux_fraction))
+            mags = d[mag_field_key][point_like_mask]
+        elif self.agn_flux_fraction == 0.:
+            mags = d[mag_field_key]
+        else:
+            mags = d[mag_agn_key]
+            fluxid = 'AGN'
 
         #####################################################
         # caclulate the number densities of AGN
         #####################################################
 
-        # get the total number of AGN in catalog
+        # get the total number of AGN passing cut and save for txt file
         N_tot = len(mags)
-
+        total_txt = '{}/{} with AGN magnitude fraction > {}'.format(N_tot, N_all, self.agn_flux_fraction)
+        
         # define the apparent magnitude bins for plotting purposes
         dmag = self.validation_data.get('bin_width', 0.25)
 
@@ -191,6 +217,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         # calculate differential binned data (validation data does not divide by bin width)
         Ndm, _ = np.histogram(mags, bins=mag_bins)
+        print(Ndm)
         dn = Ndm/sky_area
 
         # calculate N(<mag) in the bins
@@ -208,6 +235,9 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         results = {'catalog':{}, 'data':{}}
         colname = 'n'
 
+        legend_title = '; '.join(('$\\rm F_{{AGN}}/F_{{AGN+galaxy}} > {}$'.format(self.agn_flux_fraction),
+                                  'No AGN extinction' if self.no_agn_extinction else 'AGN extinction'))
+
         for ax, summary_ax, mag_pts, cat_data, ytit, (k, val_data) in zip(axs, self.summary_axs, 
                                                                           [mag_cen, mag_bins[1:]],
                                                                           [dn, n_cum],
@@ -225,17 +255,20 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
                 self.decorate_plot(summary_ax[0], self.validation_data['ytitle'][k].format(ytit), scale='log')
 
             #decorate
-            self.decorate_plot(ax[0], self.validation_data['ytitle'][k].format(ytit), scale='log')
-            
+            self.decorate_plot(ax[0], self.validation_data['ytitle'][k].format(ytit), scale='log',
+                               legend_title=legend_title)
+
             # get fractional diffrence between the mock catalog and validation data
             mag_val_pts, delta = self.get_frac_diff(mag_pts, cat_data, val_data['mag'], val_data['n'],
                                                     self.validation_range)
             ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
             summary_ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
-            self.decorate_plot(ax[1], ylabel=r'$\Delta n/n$', scale='linear', xlabel=r'$\rm m_{}$'.format(self.band))
+            self.decorate_plot(ax[1], ylabel=r'$\Delta n/n$', scale='linear',
+                               xlabel=r'$\rm m_{}^{{{}}}$'.format(self.band, fluxid),
+                               legend_title=legend_title)
             if self.first_pass:
                 self.decorate_plot(summary_ax[1], ylabel=r'$\Delta n/n$', scale='linear',
-                                   xlabel=r'$\rm m_{}$'.format(self.band))
+                                   xlabel='$\\rm m_{}^{{{}}}$'.format(self.band, fluxid))
 
             #save plotted points
             N_tot_data = np.sum(val_data['N_Q']) # total number of AGN
@@ -253,7 +286,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         #save results for catalog and validation data in txt files
         for filename, dtype, ntot in zip_longest((filelabel, re.sub(' ', '_', self.validation_data['label'])),
                                                  ('catalog', 'data'),
-                                                 (N_tot, N_tot_data)):
+                                                 (total_txt, N_tot_data)):
             if filename and dtype:
                 with open(os.path.join(output_dir, 'Nagn_' + filename + '.txt'), 'ab') as f_handle:
                     for key, value in results[dtype].items():
@@ -299,14 +332,15 @@ def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
         return mag_pts[val_mask], (cat_data[val_mask] - interp_data)/interp_data
 
 
-    def decorate_plot(self, ax, ylabel, scale='log', xlabel=None):
+    def decorate_plot(self, ax, ylabel, scale='log', xlabel=None, legend_title=''):
         """
         """
         ax.set_ylabel(ylabel, size=self.font_size)
-        ax.legend(loc='best', fancybox=True, framealpha=0.5, fontsize=self.legend_size, numpoints=1)
+        ax.legend(loc='upper left', fancybox=True, framealpha=0.5, title=legend_title,
+                  fontsize=self.legend_size, numpoints=1)
         ax.set_yscale(scale)
         if xlabel:
-            ax.set_xlabel(xlabel)
+            ax.set_xlabel(xlabel, size=self.font_size)
         else:
             for axlabel in ax.get_xticklabels():
                 axlabel.set_visible(False)
@@ -318,7 +352,7 @@ def post_process_plot(fig):
         """
         fig.tight_layout()
         fig.subplots_adjust(hspace=0)
-
+        fig.subplots_adjust(top=0.94)
 
     @staticmethod
     def save_quantities(keyname, results, filename, comment=''):
diff --git a/descqa/configs/AGN_NumberDensity_SDSSg.yaml b/descqa/configs/AGN_NumberDensity_SDSSg.yaml
index 6f372ba4..fe156fcf 100644
--- a/descqa/configs/AGN_NumberDensity_SDSSg.yaml
+++ b/descqa/configs/AGN_NumberDensity_SDSSg.yaml
@@ -2,8 +2,10 @@ subclass_name: AGN_NumberDensities.AGN_NumberDensity
 band: g
 rest_frame_band: i
 Mag_lim: -22.5
-z_lim: [0.4, 2.1]
+z_lim: [0.3, 2.2]
 observation: 'SDSS'
-validation_range: [15., 19.]
+validation_range: [15.5, 19.]
+agn_flux_fraction: 0.5
+no_agn_extinction: true
 included_by_default: true
 description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml b/descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml
new file mode 100644
index 00000000..7595566d
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml
@@ -0,0 +1,11 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: g
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [15.5, 19.]
+agn_flux_fraction: 0.01
+no_agn_extinction: true
+included_by_default: false
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSi.yaml b/descqa/configs/AGN_NumberDensity_SDSSi.yaml
index c2289e24..24d81f1f 100644
--- a/descqa/configs/AGN_NumberDensity_SDSSi.yaml
+++ b/descqa/configs/AGN_NumberDensity_SDSSi.yaml
@@ -5,5 +5,7 @@ Mag_lim: -22.5
 z_lim: [0.3, 2.2]
 observation: 'SDSS'
 validation_range: [16., 19.5]
+agn_flux_fraction: 0.5
+no_agn_extinction: true
 included_by_default: true
 description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml b/descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml
new file mode 100644
index 00000000..7cf34991
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml
@@ -0,0 +1,11 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: i
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [15.5, 19.]
+agn_flux_fraction: 0.01
+no_agn_extinction: true
+included_by_default: false
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/data/AGNdata/SDSS/richards_2006_table2.dat b/descqa/data/AGNdata/SDSS/richards_2006_table2.dat
index 02b4e04e..2cd8bd5e 100644
--- a/descqa/data/AGNdata/SDSS/richards_2006_table2.dat
+++ b/descqa/data/AGNdata/SDSS/richards_2006_table2.dat
@@ -2,24 +2,25 @@
 # Differential counts for g band with 0.4 < z < 2.2, M_g < 22.5, N per 0.25 mag deg^-2
 # Differential counts for i band with 0.3 < z < 2.2, M_i < 22.5, N per 0.25 mag deg^-2
 # Cumulative number counts N(<x) in units of deg^-2 for  0.3 < z < 2.2
+# Table differs from published version; i-band columns moved up by 1 row as instructed by Gordon Richards 
 M         N_g  dN_g      N(<g) dN(<g)  N_Q    N_i  dN_i      N(<i) dN(<i)  N_Q
-15.475    0.00 0.00      0.00 0.00       2    0.00 0.00      0.00 0.00       1 
-15.725    0.00 0.00      0.00 0.00       3    0.00 0.00      0.00 0.00       3 
-15.975    0.00 0.00      0.01 0.00       4    0.00 0.00      0.00 0.00       2 
-16.225    0.01 0.00      0.01 0.00      11    0.01 0.00      0.01 0.00      10 
-16.475    0.01 0.00      0.02 0.00      10    0.01 0.00      0.02 0.00      12 
-16.725    0.02 0.00      0.04 0.00      32    0.01 0.00      0.03 0.00      23 
-16.975    0.06 0.01      0.10 0.01      85    0.04 0.01      0.08 0.01      67 
-17.225    0.08 0.01      0.17 0.01     117    0.07 0.01      0.15 0.01     107 
-17.475    0.12 0.01      0.29 0.01     190    0.11 0.01      0.26 0.01     174 
-17.725    0.23 0.01      0.53 0.01     357    0.21 0.01      0.47 0.01     327 
-17.975    0.31 0.01      0.83 0.01     472    0.33 0.01      0.80 0.01     511 
-18.225    0.56 0.02      1.40 0.02     868    0.55 0.02      1.35 0.02     849 
-18.475    0.78 0.02      2.18 0.02    1202    0.82 0.02      2.17 0.02    1257 
-18.725    1.19 0.03      3.37 0.03    1839    1.25 0.03      3.42 0.03    1923 
-18.975    1.70 0.03      5.07 0.03    2620    1.86 0.04      5.28 0.03    2870 
-19.225    1.61 0.03      6.68 0.03    2484    2.62 0.04      7.90 0.04    4028 
-19.475    0.56 0.02      7.24 0.02     855   ...    ...      ...  ...      900 
-19.725    0.13 0.01      7.37 0.01     206   ...    ...      ...  ...      216 
-19.975    0.03 0.01      7.41 0.00      53   ...    ...      ...  ...       55 
-20.225    0.01 0.00      7.42 0.00      19   ...    ...      ...  ...       20 
+15.475    0.00 0.00      0.00 0.00       2    0.00 0.00      0.00 0.00       3
+15.725    0.00 0.00      0.00 0.00       3    0.00 0.00      0.00 0.00       2 
+15.975    0.00 0.00      0.01 0.00       4    0.01 0.00      0.02 0.00      10
+16.225    0.01 0.00      0.01 0.00      11    0.01 0.00      0.02 0.00      12 
+16.475    0.01 0.00      0.02 0.00      10    0.01 0.00      0.03 0.00      23 
+16.725    0.02 0.00      0.04 0.00      32    0.04 0.01      0.08 0.01      67 
+16.975    0.06 0.01      0.10 0.01      85    0.07 0.01      0.15 0.01     107 
+17.225    0.08 0.01      0.17 0.01     117    0.11 0.01      0.26 0.01     174 
+17.475    0.12 0.01      0.29 0.01     190    0.21 0.01      0.47 0.01     327  
+17.725    0.23 0.01      0.53 0.01     357    0.33 0.01      0.80 0.01     511 
+17.975    0.31 0.01      0.83 0.01     472    0.55 0.02      1.35 0.02     849 
+18.225    0.56 0.02      1.40 0.02     868    0.82 0.02      2.17 0.02    1257 
+18.475    0.78 0.02      2.18 0.02    1202    1.25 0.03      3.42 0.03    1923 
+18.725    1.19 0.03      3.37 0.03    1839    1.86 0.04      5.28 0.03    2870 
+18.975    1.70 0.03      5.07 0.03    2620    2.62 0.04      7.90 0.04    4028 
+19.225    1.61 0.03      6.68 0.03    2484    ...    ...      ...  ...      900  
+19.475    0.56 0.02      7.24 0.02     855    ...    ...      ...  ...      216  
+19.725    0.13 0.01      7.37 0.01     206    ...    ...      ...  ...       55 
+19.975    0.03 0.01      7.41 0.00      53    ...    ...      ...  ...       20 
+20.225    0.01 0.00      7.42 0.00      19    ...    ...      ...  ...      ...
diff --git a/descqa/data/AGNdata/SDSS/richards_2006_table2.dat.original b/descqa/data/AGNdata/SDSS/richards_2006_table2.dat.original
new file mode 100644
index 00000000..02b4e04e
--- /dev/null
+++ b/descqa/data/AGNdata/SDSS/richards_2006_table2.dat.original
@@ -0,0 +1,25 @@
+# RICHARDS ET AL. 2006_AJ_131_2766
+# Differential counts for g band with 0.4 < z < 2.2, M_g < 22.5, N per 0.25 mag deg^-2
+# Differential counts for i band with 0.3 < z < 2.2, M_i < 22.5, N per 0.25 mag deg^-2
+# Cumulative number counts N(<x) in units of deg^-2 for  0.3 < z < 2.2
+M         N_g  dN_g      N(<g) dN(<g)  N_Q    N_i  dN_i      N(<i) dN(<i)  N_Q
+15.475    0.00 0.00      0.00 0.00       2    0.00 0.00      0.00 0.00       1 
+15.725    0.00 0.00      0.00 0.00       3    0.00 0.00      0.00 0.00       3 
+15.975    0.00 0.00      0.01 0.00       4    0.00 0.00      0.00 0.00       2 
+16.225    0.01 0.00      0.01 0.00      11    0.01 0.00      0.01 0.00      10 
+16.475    0.01 0.00      0.02 0.00      10    0.01 0.00      0.02 0.00      12 
+16.725    0.02 0.00      0.04 0.00      32    0.01 0.00      0.03 0.00      23 
+16.975    0.06 0.01      0.10 0.01      85    0.04 0.01      0.08 0.01      67 
+17.225    0.08 0.01      0.17 0.01     117    0.07 0.01      0.15 0.01     107 
+17.475    0.12 0.01      0.29 0.01     190    0.11 0.01      0.26 0.01     174 
+17.725    0.23 0.01      0.53 0.01     357    0.21 0.01      0.47 0.01     327 
+17.975    0.31 0.01      0.83 0.01     472    0.33 0.01      0.80 0.01     511 
+18.225    0.56 0.02      1.40 0.02     868    0.55 0.02      1.35 0.02     849 
+18.475    0.78 0.02      2.18 0.02    1202    0.82 0.02      2.17 0.02    1257 
+18.725    1.19 0.03      3.37 0.03    1839    1.25 0.03      3.42 0.03    1923 
+18.975    1.70 0.03      5.07 0.03    2620    1.86 0.04      5.28 0.03    2870 
+19.225    1.61 0.03      6.68 0.03    2484    2.62 0.04      7.90 0.04    4028 
+19.475    0.56 0.02      7.24 0.02     855   ...    ...      ...  ...      900 
+19.725    0.13 0.01      7.37 0.01     206   ...    ...      ...  ...      216 
+19.975    0.03 0.01      7.41 0.00      53   ...    ...      ...  ...       55 
+20.225    0.01 0.00      7.42 0.00      19   ...    ...      ...  ...       20 

From b368d20b7026a43a5bc8c2ae0c0887da6e16a03e Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Mon, 9 Sep 2019 14:07:00 -0700
Subject: [PATCH 05/10] Remove extra print statement

---
 descqa/AGN_NumberDensities.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index 082361a3..51295ed6 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -217,7 +217,6 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         # calculate differential binned data (validation data does not divide by bin width)
         Ndm, _ = np.histogram(mags, bins=mag_bins)
-        print(Ndm)
         dn = Ndm/sky_area
 
         # calculate N(<mag) in the bins

From 103aad5e2af72edf2c2642adf447c23a4acf3227 Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Wed, 18 Sep 2019 14:19:23 -0700
Subject: [PATCH 06/10] Added duty-cycle option

---
 descqa/AGN_NumberDensities.py                 | 37 +++++++++++++------
 descqa/configs/AGN_NumberDensity_SDSSg.yaml   |  3 +-
 .../AGN_NumberDensity_SDSSg_frac.5.yaml       | 12 ++++++
 descqa/configs/AGN_NumberDensity_SDSSi.yaml   |  3 +-
 .../AGN_NumberDensity_SDSSi_frac.5.yaml       | 12 ++++++
 5 files changed, 54 insertions(+), 13 deletions(-)
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSg_frac.5.yaml
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSi_frac.5.yaml

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index 51295ed6..c89c02ec 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -41,7 +41,7 @@ class AGN_NumberDensity(BaseValidationTest):
     """
     AGN number desnsity test
     """
-    def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1],
+    def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=(0.4, 2.1),
                  observation='SDSS', **kwargs):
         """
         parameters
@@ -83,13 +83,15 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=[0.4, 2.1
                                      'mag_{}_agnonly_{}lsst',
                                     )
         self.possible_agn_mag_fields = [f.format(band, noagnext) for f in possible_agn_mag_fields] 
+        self.duty_cycle_quantity = 'duty_cycle_on'
         
         # attach some attributes to the test
         self.band = band
         self.rest_frame_band = rest_frame_band
         self.Mag_lim = Mag_lim
         self.z_lim = list(z_lim)
-        self.agn_flux_fraction = kwargs.get('agn_flux_fraction', 0.5)
+        self.duty_cycle_on = kwargs.get('duty_cycle_on', True)
+        self.agn_flux_fraction = kwargs.get('agn_flux_fraction', 0.)
         self.font_size = kwargs.get('font_size', 18)
         self.title_size = kwargs.get('title_size', 20)
         self.legend_size = kwargs.get('legend_size', 16)
@@ -158,6 +160,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         mag_field_key = catalog_instance.first_available(*self.possible_mag_fields)
         Mag_field_key = catalog_instance.first_available(*self.possible_Mag_fields)
         mag_agn_key = catalog_instance.first_available(*self.possible_agn_mag_fields)
+        duty_cycle_key = catalog_instance.first_available(self.duty_cycle_quantity)
         if not mag_field_key:
             return TestResult(skipped=True,
                               summary='Catalog is missing requested quantity: {}'.format(self.possible_mag_fields))
@@ -167,7 +170,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         if not mag_agn_key:
             return TestResult(skipped=True,
                               summary='Catalog is missing requested quantity: {}'.format(self.possible_agn_mag_fields))
-
+        if self.duty_cycle_on and not duty_cycle_key:
+            return TestResult(skipped=True,
+                              summary='Catalog is missing requested quantity: {}'.format(self.duty_cycle_quantity))
+        
         # check to see if catalog is a light cone
         # this is required since we must be able to calculate the angular area
         if not catalog_instance.lightcone:
@@ -184,9 +190,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
         z_filters = ['redshift > {}'.format(self.z_lim[0]), 'redshift < {}'.format(self.z_lim[1])]
         Mag_filters = ['{} < {}'.format(Mag_field_key, self.Mag_lim)]
+        filters = z_filters + Mag_filters + [duty_cycle_key] if self.duty_cycle_on else z_filters + Mag_filters
         
         # retreive data from mock catalog
-        catalog_data = catalog_instance.get_quantities([mag_field_key, mag_agn_key], filters=z_filters+ Mag_filters)
+        catalog_data = catalog_instance.get_quantities([mag_field_key, mag_agn_key], filters=filters)
         d = GCRQuery(*((np.isfinite, col) for col in catalog_data)).filter(catalog_data)
         
         #select point-like sources according to specified agn_flux_fraction
@@ -200,15 +207,21 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         else:
             mags = d[mag_agn_key]
             fluxid = 'AGN'
-
+            
         #####################################################
         # caclulate the number densities of AGN
         #####################################################
 
         # get the total number of AGN passing cut and save for txt file
         N_tot = len(mags)
-        total_txt = '{}/{} with AGN magnitude fraction > {}'.format(N_tot, N_all, self.agn_flux_fraction)
-        
+        if self.agn_flux_fraction > 0.:
+            total_txt = '{}/{} with AGN magnitude fraction > {}'.format(N_tot, N_all, self.agn_flux_fraction)
+            fraction_txt = '$\\rm F_{{AGN}}/F_{{Total}} > {}$'.format(self.agn_flux_fraction)
+        else:
+            total_txt = '{} AGN'.format(N_tot)
+            fraction_txt = ''
+        fraction_txt = '$\\rm F_{{AGN}}/F_{{Total}} > {}$'.format(self.agn_flux_fraction)
+            
         # define the apparent magnitude bins for plotting purposes
         dmag = self.validation_data.get('bin_width', 0.25)
 
@@ -234,8 +247,9 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         results = {'catalog':{}, 'data':{}}
         colname = 'n'
 
-        legend_title = '; '.join(('$\\rm F_{{AGN}}/F_{{AGN+galaxy}} > {}$'.format(self.agn_flux_fraction),
-                                  'No AGN extinction' if self.no_agn_extinction else 'AGN extinction'))
+        legend_title = '; '.join(('Duty-cycle on' if self.duty_cycle_on else 'No duty-cycle',
+                                  fraction_txt,
+                                  'No AGN ext.' if self.no_agn_extinction else 'AGN ext.'))
 
         for ax, summary_ax, mag_pts, cat_data, ytit, (k, val_data) in zip(axs, self.summary_axs, 
                                                                           [mag_cen, mag_bins[1:]],
@@ -335,8 +349,9 @@ def decorate_plot(self, ax, ylabel, scale='log', xlabel=None, legend_title=''):
         """
         """
         ax.set_ylabel(ylabel, size=self.font_size)
-        ax.legend(loc='upper left', fancybox=True, framealpha=0.5, title=legend_title,
-                  fontsize=self.legend_size, numpoints=1)
+        leg = ax.legend(loc='upper left', fancybox=True, framealpha=0.5,
+                        fontsize=self.legend_size, numpoints=1)
+        leg.set_title(legend_title, prop={'size':'medium'})
         ax.set_yscale(scale)
         if xlabel:
             ax.set_xlabel(xlabel, size=self.font_size)
diff --git a/descqa/configs/AGN_NumberDensity_SDSSg.yaml b/descqa/configs/AGN_NumberDensity_SDSSg.yaml
index fe156fcf..ff40a80b 100644
--- a/descqa/configs/AGN_NumberDensity_SDSSg.yaml
+++ b/descqa/configs/AGN_NumberDensity_SDSSg.yaml
@@ -5,7 +5,8 @@ Mag_lim: -22.5
 z_lim: [0.3, 2.2]
 observation: 'SDSS'
 validation_range: [15.5, 19.]
-agn_flux_fraction: 0.5
+agn_flux_fraction: 0.
+duty_cycle_on: true
 no_agn_extinction: true
 included_by_default: true
 description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSg_frac.5.yaml b/descqa/configs/AGN_NumberDensity_SDSSg_frac.5.yaml
new file mode 100644
index 00000000..6fafee23
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSg_frac.5.yaml
@@ -0,0 +1,12 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: g
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [15.5, 19.]
+agn_flux_fraction: 0.5
+no_agn_extinction: true
+duty_cycle_on: false
+included_by_default: false
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSi.yaml b/descqa/configs/AGN_NumberDensity_SDSSi.yaml
index 24d81f1f..ecca081f 100644
--- a/descqa/configs/AGN_NumberDensity_SDSSi.yaml
+++ b/descqa/configs/AGN_NumberDensity_SDSSi.yaml
@@ -5,7 +5,8 @@ Mag_lim: -22.5
 z_lim: [0.3, 2.2]
 observation: 'SDSS'
 validation_range: [16., 19.5]
-agn_flux_fraction: 0.5
+agn_flux_fraction: 0.
+duty_cycle_on: true
 no_agn_extinction: true
 included_by_default: true
 description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSi_frac.5.yaml b/descqa/configs/AGN_NumberDensity_SDSSi_frac.5.yaml
new file mode 100644
index 00000000..1378496e
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSi_frac.5.yaml
@@ -0,0 +1,12 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: i
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [15.5, 19.]
+agn_flux_fraction: 0.5
+no_agn_extinction: true
+duty_cycle_on: false
+included_by_default: false
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'

From bc115812afd55b7557fafcbf62eb2eeaf5351ced Mon Sep 17 00:00:00 2001
From: Eve Kovacs <kovacs@anl.gov>
Date: Wed, 18 Sep 2019 16:24:02 -0500
Subject: [PATCH 07/10] Delete AGN_NumberDensity_SDSSg_frac.01.yaml

---
 descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml

diff --git a/descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml b/descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml
deleted file mode 100644
index 7595566d..00000000
--- a/descqa/configs/AGN_NumberDensity_SDSSg_frac.01.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-subclass_name: AGN_NumberDensities.AGN_NumberDensity
-band: g
-rest_frame_band: i
-Mag_lim: -22.5
-z_lim: [0.3, 2.2]
-observation: 'SDSS'
-validation_range: [15.5, 19.]
-agn_flux_fraction: 0.01
-no_agn_extinction: true
-included_by_default: false
-description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'

From d3869409ac2683de3458807923bfb90a5b428388 Mon Sep 17 00:00:00 2001
From: Eve Kovacs <kovacs@anl.gov>
Date: Wed, 18 Sep 2019 16:24:21 -0500
Subject: [PATCH 08/10] Delete AGN_NumberDensity_SDSSi_frac.01.yaml

---
 descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml

diff --git a/descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml b/descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml
deleted file mode 100644
index 7cf34991..00000000
--- a/descqa/configs/AGN_NumberDensity_SDSSi_frac.01.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-subclass_name: AGN_NumberDensities.AGN_NumberDensity
-band: i
-rest_frame_band: i
-Mag_lim: -22.5
-z_lim: [0.3, 2.2]
-observation: 'SDSS'
-validation_range: [15.5, 19.]
-agn_flux_fraction: 0.01
-no_agn_extinction: true
-included_by_default: false
-description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'

From 68465489cb3d4a7aad4bfa7c3c5238f5b15e89c8 Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Fri, 20 Sep 2019 17:08:35 -0700
Subject: [PATCH 09/10] Minor change to adjust for not using extinction as
 default in reader

---
 descqa/AGN_NumberDensities.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index c89c02ec..93608e70 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -66,7 +66,7 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=(0.4, 2.1
 
         self.no_agn_extinction = kwargs.get('no_agn_extinction', True)        
         # catalog quantities needed
-        noagnext = 'no_agn_extinction_' if self.no_agn_extinction else ''
+        noagnext = '' if self.no_agn_extinction else 'extincted_agn_'
         possible_mag_fields = ('mag_{}_{}sdss',
                                'mag_{}_{}lsst',
                               )

From 35b4cef3a13929f45521136b085650466a74dbe2 Mon Sep 17 00:00:00 2001
From: evevkovacs <kovacs@anl.gov>
Date: Fri, 31 Jan 2020 11:31:05 -0800
Subject: [PATCH 10/10] Cosmetic changes for DC2 paper plots

---
 descqa/AGN_NumberDensities.py                 | 75 ++++++++++++-------
 .../configs/AGN_NumberDensity_SDSSg_fig.yaml  | 15 ++++
 .../configs/AGN_NumberDensity_SDSSi_fig.yaml  | 15 ++++
 3 files changed, 77 insertions(+), 28 deletions(-)
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSg_fig.yaml
 create mode 100644 descqa/configs/AGN_NumberDensity_SDSSi_fig.yaml

diff --git a/descqa/AGN_NumberDensities.py b/descqa/AGN_NumberDensities.py
index 93608e70..b2f5b031 100644
--- a/descqa/AGN_NumberDensities.py
+++ b/descqa/AGN_NumberDensities.py
@@ -14,20 +14,20 @@
 possible_observations = {
     'SDSS': {
         'filename_template': 'AGNdata/SDSS/richards_2006_table2.dat',
-        'usecols': {'g': {'dN/dm':(0, 1, 2, 5),
-                          'N<m':(0, 3, 4, 5),
+        'usecols': {'g': {'N<m':(0, 3, 4, 5),
+                          'dN/dm':(0, 1, 2, 5),
                          },
-                    'i': {'dN/dm':(0, 6, 7, 10),
-                          'N<m':(0, 8, 9, 10),
+                    'i': {'N<m':(0, 8, 9, 10),
+                          'dN/dm':(0, 6, 7, 10),
                          },
                    },
         'colnames': ('mag', 'n', 'err', 'N_Q'),
         'skiprows': 5,
         'missing_values':'...',
         'bin-width': 0.25,
-        'label': 'SDSS DR3',
+        'label': 'Richards et. al. (2006)\nSDSS',
         'color': 'r',
-        'title': 'Richards et. al. (2006)  $\\rm M^{{AGN+galaxy}}_{}<{}$, $\\rm {}<z<{}$',
+        'title': '$\\rm M_{}<{}$, $\\rm {}<z<{}$',
         'ytitle':{'dN/dm':'$\\rm N (deg^{{-2}} {}mag^{{-1}})$',
                   'N<m':'$\\rm N(<{}) (deg^{{-2}})$',
                  },
@@ -92,20 +92,23 @@ def __init__(self, band='g', rest_frame_band='i', Mag_lim=-22.5, z_lim=(0.4, 2.1
         self.z_lim = list(z_lim)
         self.duty_cycle_on = kwargs.get('duty_cycle_on', True)
         self.agn_flux_fraction = kwargs.get('agn_flux_fraction', 0.)
-        self.font_size = kwargs.get('font_size', 18)
+        self.font_size = kwargs.get('font_size', 20)
         self.title_size = kwargs.get('title_size', 20)
-        self.legend_size = kwargs.get('legend_size', 16)
+        self.legend_size = kwargs.get('legend_size', 18)
         self.no_title = kwargs.get('no_title', False)
+        self.no_legend_title = kwargs.get('no_legend_title', False)
+        self.truncate_legend_title = kwargs.get('truncate_legend', True)
         self.nrows = kwargs.get('nrows', 1)
-        self.ncolumns = kwargs.get('ncolumns', 2)
+        self.cumulative_only = kwargs.get('cumulative_only', False)
+        self.ncolumns = 1 if self.cumulative_only else kwargs.get('ncolumns', 2)
         self.mag_lo = kwargs.get('mag_lo', 14)
-        self.mag_hi = kwargs.get('mag_hi', 24)
+        self.mag_hi = kwargs.get('mag_hi', 22)
         self.validation_range = kwargs.get('validation_range', (16., 19.))
         self.truncate_cat_name = kwargs.get('truncate_cat_name', False)
-        self.figx_p = kwargs.get('figx_p', 11)
+        self.figx_p = kwargs.get('figx_p', 7)
         self.figy_p = kwargs.get('figy_p', 7)
         self.msize = kwargs.get('msize', 6)
-
+        
         # set color of lines in plots
         colors = plt.cm.jet(np.linspace(0, 1, 2)) # pylint: disable=no-member
         if band == 'g':
@@ -247,15 +250,29 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         results = {'catalog':{}, 'data':{}}
         colname = 'n'
 
-        legend_title = '; '.join(('Duty-cycle on' if self.duty_cycle_on else 'No duty-cycle',
-                                  fraction_txt,
-                                  'No AGN ext.' if self.no_agn_extinction else 'AGN ext.'))
-
+        legend_title = ''
+        if not self.no_legend_title:
+            if self.truncate_legend_title:
+                legend_title = 'Duty-cycle on'
+            else:
+                legend_title = '; '.join(('Duty-cycle on' if self.duty_cycle_on else 'No duty-cycle',
+                                          fraction_txt,
+                                          'No AGN ext.' if self.no_agn_extinction else 'AGN ext.'))
+        
+        if not self.cumulative_only:
+            mag_pts_list = [mag_bins[1:], mag_cen]
+            cat_data_list = [n_cum, dn]
+            ytitles = [self.band, dmag]
+            val_tuples = self.validation_data['data'].items()
+        else:
+            mag_pts_list = [mag_bins[1:]]
+            cat_data_list = [n_cum]
+            ytitles = [self.band]
+            val_tuples = [ (k, v) for k, v in self.validation_data['data'].items() if 'N<m' in k]
         for ax, summary_ax, mag_pts, cat_data, ytit, (k, val_data) in zip(axs, self.summary_axs, 
-                                                                          [mag_cen, mag_bins[1:]],
-                                                                          [dn, n_cum],
-                                                                          [dmag, self.band],
-                                                                          self.validation_data['data'].items()):
+                                                                          mag_pts_list,
+                                                                          cat_data_list, ytitles,
+                                                                          val_tuples):
             # plot
             ax[0].plot(mag_pts, cat_data, label=catalog_name, color=self.line_color)
             ax[0].errorbar(val_data['mag'], val_data['n'], yerr=val_data['err'], label=self.validation_data['label'],
@@ -278,7 +295,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
             summary_ax[1].plot(mag_val_pts, delta, color=self.line_color, label='Frac. Diff.')
             self.decorate_plot(ax[1], ylabel=r'$\Delta n/n$', scale='linear',
                                xlabel=r'$\rm m_{}^{{{}}}$'.format(self.band, fluxid),
-                               legend_title=legend_title)
+                               legend_title=legend_title, legend_loc='lower right')
             if self.first_pass:
                 self.decorate_plot(summary_ax[1], ylabel=r'$\Delta n/n$', scale='linear',
                                    xlabel='$\\rm m_{}^{{{}}}$'.format(self.band, fluxid))
@@ -320,13 +337,15 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
     def setup_subplots(self):
         """
         """
-        fig = plt.figure(figsize=(self.figx_p, self.figy_p))
+        fig = plt.figure(figsize=(self.figx_p*self.ncolumns, self.figy_p))
         gs = fig.add_gridspec(2*self.nrows, self.ncolumns, height_ratios=[3, 1])
         axs = []
         axs.append([fig.add_subplot(gs[0])])
-        axs[0].append(fig.add_subplot(gs[2], sharex=axs[0][0]))
-        axs.append([fig.add_subplot(gs[1])])
-        axs[1].append(fig.add_subplot(gs[3], sharex=axs[1][0]))
+        gs_index = 1 if self.cumulative_only else 2
+        axs[0].append(fig.add_subplot(gs[gs_index], sharex=axs[0][0]))
+        if not self.cumulative_only:
+            axs.append([fig.add_subplot(gs[1])])
+            axs[1].append(fig.add_subplot(gs[3], sharex=axs[1][0]))
 
         return fig, axs
 
@@ -345,13 +364,13 @@ def get_frac_diff(mag_pts, cat_data, val_mags, val_data, validation_range):
         return mag_pts[val_mask], (cat_data[val_mask] - interp_data)/interp_data
 
 
-    def decorate_plot(self, ax, ylabel, scale='log', xlabel=None, legend_title=''):
+    def decorate_plot(self, ax, ylabel, scale='log', xlabel=None, legend_title='', legend_loc='upper left'):
         """
         """
         ax.set_ylabel(ylabel, size=self.font_size)
-        leg = ax.legend(loc='upper left', fancybox=True, framealpha=0.5,
+        leg = ax.legend(loc=legend_loc, fancybox=True, framealpha=0.5,
                         fontsize=self.legend_size, numpoints=1)
-        leg.set_title(legend_title, prop={'size':'medium'})
+        leg.set_title(legend_title, prop={'size':self.legend_size})
         ax.set_yscale(scale)
         if xlabel:
             ax.set_xlabel(xlabel, size=self.font_size)
diff --git a/descqa/configs/AGN_NumberDensity_SDSSg_fig.yaml b/descqa/configs/AGN_NumberDensity_SDSSg_fig.yaml
new file mode 100644
index 00000000..42998f63
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSg_fig.yaml
@@ -0,0 +1,15 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: g
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [15.5, 19.]
+agn_flux_fraction: 0.
+duty_cycle_on: true
+no_agn_extinction: true
+cumulative_only: true
+truncate_cat_name: true
+included_by_default: false
+no_legend_title: true
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'
diff --git a/descqa/configs/AGN_NumberDensity_SDSSi_fig.yaml b/descqa/configs/AGN_NumberDensity_SDSSi_fig.yaml
new file mode 100644
index 00000000..e07b2052
--- /dev/null
+++ b/descqa/configs/AGN_NumberDensity_SDSSi_fig.yaml
@@ -0,0 +1,15 @@
+subclass_name: AGN_NumberDensities.AGN_NumberDensity
+band: i
+rest_frame_band: i
+Mag_lim: -22.5
+z_lim: [0.3, 2.2]
+observation: 'SDSS'
+validation_range: [16., 19.5]
+agn_flux_fraction: 0.
+duty_cycle_on: true
+no_agn_extinction: true
+cumulative_only: true
+truncate_cat_name: true
+no_legend_title: true
+included_by_default: false
+description: 'Plot N and N(<mag) distributions for selected magnitude bound and redshift range in specified band and compare with SDSS data from Richards 2006'