Changes for running and testing the roman_rubin catalog

descqa/DeltaSigmaTest.py

@@ -58,7 +58,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
         if self.data == 'sdss_lowz':
 
             # Singh et al (2015) (http://adsabs.harvard.edu/abs/2015MNRAS.450.2195S) measurements on the SDSS LOWZ sample.
-		
+
             res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2',
                     'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss'])
 
@@ -101,10 +101,15 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
             fig2 = plt.figure(2, figsize=(12,5))
 
         if self.data == 'sdss_main':
-            res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2',
-                    'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss', 'stellar_mass_bulge', 'stellar_mass_disk','Mag_true_g_sdss_z0','Mag_true_r_sdss_z0'])
+            if not catalog_instance.has_quantities(['stellar_mass']):
+                catalog_instance.add_derived_quantity('stellar_mass', np.add, 'stellar_mass_bulge', 'stellar_mass_disk')
+            res = catalog_instance.get_quantities([
+                 'redshift_true', 'ra', 'dec', 'shear_1', 'shear_2', 'stellar_mass',
+                 'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss',
+                 'Mag_true_g_sdss_z0', 'Mag_true_r_sdss_z0',
+             ])
+            sm = res['stellar_mass']
             gr = res['Mag_true_g_sdss_z0'] - res['Mag_true_r_sdss_z0'] # larger number means redder
-            sm = res['stellar_mass_bulge'] + res['stellar_mass_disk']
 
             SM_min = np.array([10,10.7,11.2,11.6])
             SM_max = np.array([10.4,11.0,11.4,15.0])
@@ -175,7 +180,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
             gt = gt/counts
 
             outfile = os.path.join(output_dir, 'DS_'+str(self.data)+'_'+str(i)+'.dat')
-            np.savetxt(outfile, np.vstack((rp, gt)).T)
+            np.savetxt(outfile, np.vstack((rp.value, gt.value)).T)
 
 
             if self.data == 'sdss_lowz':

descqa/NumberDensityVersusRedshift.py

@@ -176,7 +176,6 @@ def __init__(self, band='i', N_zbins=10, zlo=0., zhi=1.1,
 
         self._other_kwargs = kwargs
 
-
     def init_plots(self, mlo, mhi):
         #get magnitude cuts based on validation data or default limits (only catalog data plotted)
         mag_lo = self.validation_data.get('mag_lo', [float(m) for m in range(int(mhi), int(mlo+1))])
@@ -401,7 +400,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
 
     def get_jackknife_errors(self, N_jack, jackknife_data, N):
         nn = np.stack((jackknife_data[self.ra], jackknife_data[self.dec]), axis=1)
-        _, jack_labels, _ = k_means(n_clusters=N_jack, random_state=0, X=nn)
+        _, jack_labels, _ = k_means(n_clusters=N_jack, random_state=0, X=nn, n_init='auto')
 
         #make histograms for jackknife regions
         Njack_array = np.zeros((N_jack, len(self.zbins)-1), dtype=np.int)
@@ -513,7 +512,7 @@ def save_quantities(keyname, results, filename, comment=''):
             else:
                 fields = ('meanz', keyname)
                 header = ', '.join(('Data columns are: <z>', keyname, ' '))
-            np.savetxt(filename, np.vstack((results[k] for k in fields)).T, fmt='%12.4e', header=header+comment)
+            np.savetxt(filename, np.vstack(list(results[k] for k in fields)).T, fmt='%12.4e', header=header+comment)
 
 
     def conclude_test(self, output_dir):

descqa/configs/readiness_roman_rubin.yaml

@@ -0,0 +1,194 @@
+subclass_name: readiness_test.CheckQuantities
+description: 'Plot histograms of listed quantities and perform range, finiteness, mean and standard deviation checks.'
+included_by_default: true
+
+quantities_to_check:
+  - quantities: ['dec_true', 'dec']
+    label: 'deg'
+    min: [-50, -49]
+    max: [-39, -38]
+    median: [-46, -42]
+    mean: [-46, -42]
+    std: [5, 10]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: 0
+
+  - quantities: ['ra_true', 'ra']
+    label: 'deg'
+    min: [4, 5]
+    max: [15, 16]
+    median: [9, 11]
+    mean: [9, 11]
+    std: [8, 12]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: 0
+
+  - quantities: ['redshift_true', 'redshift']
+    label: redshift
+    min: [-0.05, 0.05]
+    max: [2.95, 3.05]
+    median: [1.6, 2.0]
+    mean: [1.8, 2.2]
+    std: [0.6, 0.7]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: [0, 0.001]
+
+  - quantities: ['ellipticity', 'ellipticity_[!12]*']
+    label: ellipticity_total
+    min: [0, 0.01]
+    max: [0.8, 1]
+    median: [0, 1]
+    mean: [0, 1]
+    std: [0.1, 0.4]
+    f_nan: [0, 0.3]
+    f_inf: 0
+    f_outlier: [0, 0.06]
+
+  - quantities: 'size*'
+    label: size
+    log: true
+    min: [-6, -1.5]
+    max: [0.5, 3]
+    median: [-2, 0]
+    mean: [-2, 0]
+    std: [0.1, 0.5]
+    f_nan: 0
+    f_zero: 0
+    f_inf: [0, 0.3]
+    f_outlier: [0, 0.01]
+    plot_min: -3
+    plot_max: 1
+
+  - quantities: size*_bulge*
+    f_nan: 0
+    f_inf: 0
+    min: [0, null]
+    plot_min: 0
+    plot_max: 3
+
+  - quantities: 'velocity_*'
+    label: velocity
+    min: [-10500., -9500.]
+    max: [9500., 10500.]
+    median: [-100., 100.]
+    mean: [-100., 100.]
+    std: [200., 600.]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: [0, 0.1]
+
+  - quantities: 'shear_*'
+    min: [-0.25, 0]
+    max: [0, 0.25]
+    median: [-0.01, 0.01]
+    mean: [-0.01, 0.01]
+    std: [0, 0.01]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: [0, 0.04]
+
+  - quantities: 'position_angle*'
+    min: [0, 0.001]
+    max: [179.99, 180]
+    median: [89.9, 90.1]
+    mean: [89.9, 90.1]
+    std: [0, 90.0]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: 0
+
+  - quantities: 'convergence'
+    min: [-0.4, 0]
+    max: [0, 0.8]
+    median: [-0.01, 0.01]
+    mean: [-0.01, 0.01]
+    std: [0, 0.03]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: [0, 0.05]
+
+  - quantities: 'magnification'
+    min: [0.5, 1]
+    max: [1, 20]
+    median: [0.5, 1.5]
+    mean: [0.5, 1.5]
+    std: [0, 0.1]
+    f_nan: 0
+    f_inf: 0
+    f_outlier: [0, 0.05]
+
+  - quantities: 'mag_*'
+    label: mag
+    min: [null, 15]
+    max: [25, null]
+    mean: [20, 32]
+    median: [20, 32]
+    std: [0, 5]
+    f_nan: 0
+    f_inf: 0
+    f_zero: 0
+    f_outlier: [0, 0.05]
+
+  - quantities: 'Mag_*lsst*'
+    label: Mag
+    min: [null, -23]
+    max: [-12, null]
+    mean: [-17, -8]
+    median: [-17, -8]
+    std: [0, 5]
+    f_nan: 0
+    f_inf: 0
+    f_zero: 0
+    f_outlier: [0, 0.05]
+
+  - quantities: 'stellar_mass*'
+    log: true
+    min: [null, 10]
+    max: [10, 13]
+    median: [5.5, 7]
+    mean: [5.5, 7]
+    std: [0.5, 1.5]
+    f_nan: 0
+    f_outlier: [0, 0.025]
+
+  - quantities: 'ROMAN_obs_*'
+    label: ROMAN obs magnitudes
+    min: [null, 15]
+    max: [25, null]
+    mean: [20, 32]
+    median: [20, 32]
+    std: [0, 5]
+    f_nan: 0
+    f_inf: 0
+    f_zero: 0
+    f_outlier: [0, 0.03]
+
+  - quantities: 'ROMAN_rest_*'
+    label: Roman rest magnitudes
+    min: [null, -23]
+    max: [-12, null]
+    mean: [-17, -8]
+    median: [-17, -8]
+    std: [0, 5]
+    f_nan: 0
+    f_zero: 0
+    f_outlier: [0, 0.05]
+
+  - quantities: bulge_to_total_ratio*
+    min: [0, 1]
+    max: [0, 1]
+    median: [0, 1]
+    mean: [0, 1]
+    f_nan: 0
+    f_inf: 0
+
+relations_to_check:
+  - '1.0 / magnification ~== (1.0 - convergence)**2.0 - shear_1**2.0 - shear_2**2.0'
+
+uniqueness_to_check:
+  - quantity: galaxy_id
+  - quantity: halo_id
+    mask: is_central

run_master.sh

@@ -10,6 +10,11 @@ set -e
 source /global/common/software/lsst/common/miniconda/setup_current_python.sh ""
 PYTHON='python'
 
+# increase maximum number of threads (default is 8)
+export NUMEXPR_MAX_THREADS=256
+# set number of threads to prevent thread error
+export OMP_NUM_THREADS=64
+
 # set output directory
 OUTPUTDIR="/global/cfs/cdirs/lsst/groups/CS/descqa/run/v2"