Skip to content

Commit

Permalink
for travis
Browse files Browse the repository at this point in the history
  • Loading branch information
@echaussidon
echaussidon committed Dec 22, 2020
1 parent f4853eb commit fae8935
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 68 deletions.
66 changes: 33 additions & 33 deletions py/desitarget/train/data_preparation/funcs.py
View file
Expand Up @@ -68,7 +68,7 @@ def shift_photo_north(gflux=None, rflux=None, zflux=None):
#------------------------------------------------------------------------------

def _Flux2MagFunc(OBJ_Data_flux, OBJ_Data_mw_transmission):
OBJ_flux = OBJ_Data_flux / OBJ_Data_mw_transmission
OBJ_flux = OBJ_Data_flux/OBJ_Data_mw_transmission
limitInf = 1.e-04
OBJ_flux = OBJ_flux.clip(limitInf)
OBJ_mag = np.where(OBJ_flux > limitInf, 22.5 - 2.5 * np.log10(OBJ_flux), 0.)
Expand All @@ -93,22 +93,22 @@ def Flux2MagFunc(dataArray):
W1flux = W1flux.clip(limitInf)
W2flux = W2flux.clip(limitInf)

g=np.where(gflux>limitInf,22.5-2.5*np.log10(gflux), 0.)
r=np.where(rflux>limitInf,22.5-2.5*np.log10(rflux), 0.)
z=np.where(zflux>limitInf,22.5-2.5*np.log10(zflux), 0.)
W1=np.where(W1flux>limitInf, 22.5-2.5*np.log10(W1flux), 0.)
W2=np.where(W2flux>limitInf, 22.5-2.5*np.log10(W2flux), 0.)

g[np.isnan(g)]=0.
g[np.isinf(g)]=0.
r[np.isnan(r)]=0.
r[np.isinf(r)]=0.
z[np.isnan(z)]=0.
z[np.isinf(z)]=0.
W1[np.isnan(W1)]=0.
W1[np.isinf(W1)]=0.
W2[np.isnan(W2)]=0.
W2[np.isinf(W2)]=0.
g = np.where(gflux > limitInf,22.5-2.5*np.log10(gflux), 0.)
r = np.where(rflux > limitInf,22.5-2.5*np.log10(rflux), 0.)
z = np.where(zflux > limitInf,22.5-2.5*np.log10(zflux), 0.)
W1 = np.where(W1flux > limitInf, 22.5-2.5*np.log10(W1flux), 0.)
W2 = np.where(W2flux > limitInf, 22.5-2.5*np.log10(W2flux), 0.)

g[np.isnan(g)] = 0.
g[np.isinf(g)] = 0.
r[np.isnan(r)] = 0.
r[np.isinf(r)] = 0.
z[np.isnan(z)] = 0.
z[np.isinf(z)] = 0.
W1[np.isnan(W1)] = 0.
W1[np.isinf(W1)] = 0.
W2[np.isnan(W2)] = 0.
W2[np.isinf(W2)] = 0.

return g,r,z,W1,W2

Expand All @@ -117,17 +117,17 @@ def Flux2MagFunc(dataArray):
def ColorsFunc(nbEntries,nfeatures,g,r,z,W1,W2):
colors = np.zeros((nbEntries,nfeatures))

colors[:,0]=g-r
colors[:,1]=r-z
colors[:,2]=g-z
colors[:,3]=g-W1
colors[:,4]=r-W1
colors[:,5]=z-W1
colors[:,6]=g-W2
colors[:,7]=r-W2
colors[:,8]=z-W2
colors[:,9]=W1-W2
colors[:,10]=r
colors[:,0] = g-r
colors[:,1] = r-z
colors[:,2] = g-z
colors[:,3] = g-W1
colors[:,4] = r-W1
colors[:,5] = z-W1
colors[:,6] = g-W2
colors[:,7] = r-W2
colors[:,8] = z-W2
colors[:,9] = W1-W2
colors[:,10] = r

return colors

Expand All @@ -142,7 +142,7 @@ def GetColorsFunc(data, color_names):
#------------------------------------------------------------------------------

def AreaFunc(dec1, dec2, alpha1, alpha2):
res = (alpha2 - alpha1) * (np.sin(dec2) - np.sin(dec1)) * (180. / np.pi)**2
res = (alpha2 - alpha1) * (np.sin(dec2) - np.sin(dec1)) * (180./np.pi)**2
return res

#------------------------------------------------------------------------------
Expand All @@ -151,10 +151,10 @@ def RA_DEC_AreaFunc(OBJ_RA, OBJ_DEC, binVect_RA, binVect_DEC, N_OBJ_th=2):
RA_DEC_meshgrid = np.meshgrid(binVect_RA, binVect_DEC)
OBJ_dNdRAdDEC = np.histogram2d(x=OBJ_DEC, y=OBJ_RA, bins=[binVect_DEC, binVect_RA])[0]
med_OBJ_dNdRAdDEC = np.median(OBJ_dNdRAdDEC)
skyArea_meshgrid = AreaFunc(RA_DEC_meshgrid[1][0:-1, 0:-1] * np.pi / 180.,
RA_DEC_meshgrid[1][1:, 0:-1] * np.pi / 180.,
RA_DEC_meshgrid[0][0:-1, 0 :-1] * np.pi / 180.,
RA_DEC_meshgrid[0][0:-1, 1:] * np.pi / 180.)
skyArea_meshgrid = AreaFunc(RA_DEC_meshgrid[1][0:-1, 0:-1] * np.pi/180.,
RA_DEC_meshgrid[1][1:, 0:-1] * np.pi/180.,
RA_DEC_meshgrid[0][0:-1, 0 :-1] * np.pi/180.,
RA_DEC_meshgrid[0][0:-1, 1:] * np.pi/180.)
area_OK = (OBJ_dNdRAdDEC) >= N_OBJ_th
res_area = np.sum(skyArea_meshgrid[area_OK])
return res_area, med_OBJ_dNdRAdDEC
Expand Down
8 changes: 4 additions & 4 deletions py/desitarget/train/data_preparation/make_test_sample.py
View file
Expand Up @@ -86,7 +86,7 @@ def make_test_sample(fpn_input, fpn_output, RELEASE='DR9', is_north=False):
# Display some infos
OBJ_ra = OBJ_data['RA']
OBJ_dec = OBJ_data['DEC']
n_QSO = np.sum(OBJ_data['zred']>0.)
n_QSO = np.sum(OBJ_data['zred'] > 0.)

min_ra = OBJ_ra.min()
max_ra = OBJ_ra.max()
Expand Down Expand Up @@ -116,13 +116,13 @@ def make_test_sample(fpn_input, fpn_output, RELEASE='DR9', is_north=False):
OBJ_Colors = ColorsFunc(n_OBJ, n_colors, OBJ_gmag, OBJ_rmag, OBJ_zmag, OBJ_W1mag, OBJ_W2mag)

for i, col_name in enumerate(color_names):
col = pyfits.Column(name = col_name, format = 'D', array = OBJ_Colors[:,i])
col = pyfits.Column(name=col_name, format='D', array=OBJ_Colors[:,i])
list_cols.append(col)

OBJ_hdu = pyfits.BinTableHDU(data = OBJ_data)
if is_north:
for i, col_name in enumerate(['FLUX_G', 'FLUX_R', 'FLUX_Z']):
col = pyfits.Column(name = col_name+'_s', format = 'D', array = OBJ_data[col_name].copy())
col = pyfits.Column(name=col_name + '_s', format='D', array=OBJ_data[col_name].copy())
list_cols.append(col)
OBJ_data[col_name] = backup_FLUX_GRZ[i][OBJ_selection_OK]

Expand All @@ -132,5 +132,5 @@ def make_test_sample(fpn_input, fpn_output, RELEASE='DR9', is_north=False):

#***TEST SAMPLE STORING***

OBJ_hdu.writeto(fpn_output, overwrite = True)
OBJ_hdu.writeto(fpn_output, overwrite=True)
print("Save :", fpn_output)
48 changes: 17 additions & 31 deletions py/desitarget/train/data_preparation/make_training_samples.py
View file
Expand Up @@ -16,7 +16,6 @@
from desitarget.train.data_preparation.PredCountsFromQLF_ClassModule import PredCountsFromQLF_Class

#------------------------------------------------------------------------------

#***QLF DATA FILE PATH NAME***

fpn_QLF_data = '../../py/desitarget/train/data_preparation/ROSS4_tabR.txt'
Expand All @@ -29,7 +28,6 @@ def make_training_samples(fpn_QSO_input, fpn_STARS_input, fpn_QSO_output, fpn_ST
remove_test_region = True

#------------------------------------------------------------------------------

#***CONFIGURATION***

NORM_MODE = "NORMALIZE_PER_RMAG_BINS" # "NORMALIZE_PER_RMAG_BINS" by default
Expand All @@ -54,32 +52,29 @@ def make_training_samples(fpn_QSO_input, fpn_STARS_input, fpn_QSO_output, fpn_ST
QSO_MAX_MAG_ERR_LEVEL = 0.02 # 0.02 by default

#------------------------------------------------------------------------------

#***FUNCTION***
def MAG_ERR_Func(FLUX, FLUX_IVAR):
res = 2.5 / np.log(10.)
res /= (np.abs(FLUX) * np.sqrt(FLUX_IVAR))
res = 2.5/np.log(10.)
res /= (np.abs(FLUX)*np.sqrt(FLUX_IVAR))
res[np.isinf(res)] = 0.
res[np.isnan(res)] = 0.
res[res < 0.] = 0.
return res

#------------------------------------------------------------------------------

zred_binVect = np.arange(min_zred, max_zred + zred_binStep / 2., zred_binStep)
zred_binVect = np.arange(min_zred, max_zred + zred_binStep/2., zred_binStep)

rmag_binVect = np.trunc(np.arange(min_rmag, max_rmag + rmag_binStep / 2., rmag_binStep) * 10.) / 10.
rmag_binVect = np.trunc(np.arange(min_rmag, max_rmag + rmag_binStep/2., rmag_binStep)*10.)/10.
rmag_binVect = np.minimum(rmag_binVect, max_rmag)
n_rmag_bin = rmag_binVect.size - 1

#------------------------------------------------------------------------------

#***CHECK "NORM_MODE" VALIDITY***
if (NORM_MODE != "NORMALIZE_PER_RMAG_BINS"):
assert(False), "NORM_MODE {:s} non valid".format(NORM_MODE)

#------------------------------------------------------------------------------

#***COMPUTE QLF4Compl_dNdrmag***
predCountsObj = PredCountsFromQLF_Class()
predCountsObj.LoadQLF_Data(fpn_QLF_data)
Expand All @@ -95,7 +90,6 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
QLF4Compl_dNdrmag = np.sum(QLF4Compl_dNdzdrmag, axis = 1)

#------------------------------------------------------------------------------

#***STARS SELECTION***

STARS_data = pyfits.open(fpn_STARS_input, memmap = True)[1].data
Expand Down Expand Up @@ -131,9 +125,7 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
STARS_PSF_OK = STARS_data['TYPE'] == "PSF"

# --> we allready check that
noTestRegion_OK = ~ ((STARS_data.RA <= 45.) & (STARS_data.RA >= 30.) &
(np.abs(STARS_data.DEC) <= 5.))

noTestRegion_OK = ~ ((STARS_data.RA <= 45.) & (STARS_data.RA >= 30.) & (np.abs(STARS_data.DEC) <= 5.))

# STARS_OK = STARS_rmag_OK & STARS_g_z_W1_W2_mag_OK & STARS_noBSinBLOB_OK
STARS_OK = STARS_rmag_OK & STARS_g_z_W1_W2_mag_OK & STARS_maskbits_OK
Expand All @@ -146,11 +138,10 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
STARS_rmag = STARS_rmag[STARS_OK]

n_STARS = len(STARS_data)
print("n_STARS after selection / before normalization :", n_STARS)
print("n_STARS after selection/before normalization :", n_STARS)
print()

#------------------------------------------------------------------------------

#***QSO SELECTION***

QSO_data = pyfits.open(fpn_QSO_input, memmap = True)[1].data
Expand Down Expand Up @@ -237,43 +228,40 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
QSO_dNdrmag = np.histogram(QSO_rmag, bins = rmag_binVect)[0]

#------------------------------------------------------------------------------

# Given only for informative purpose to assess the completeness of the sample

max_QSO_dNdrmag_ind = np.argmax(QSO_dNdrmag)
scale_factor = QLF4Compl_dNdrmag[max_QSO_dNdrmag_ind] / QSO_dNdrmag[max_QSO_dNdrmag_ind]
scale_factor = QLF4Compl_dNdrmag[max_QSO_dNdrmag_ind]/QSO_dNdrmag[max_QSO_dNdrmag_ind]

QSO_W_drmag = QLF4Compl_dNdrmag / (scale_factor * QSO_dNdrmag)
QSO_W_drmag = QLF4Compl_dNdrmag/(scale_factor*QSO_dNdrmag)
QSO_W_drmag[QLF4Compl_dNdrmag == 0. ] = np.nan
QSO_W_drmag[np.isnan(QSO_W_drmag) ] = np.nan
QSO_W_drmag[np.isinf(QSO_W_drmag) ] = np.nan

if False:
ref_QSO_dNdrmag_ind = np.nanargmin(QSO_W_drmag)
scale_factor = QLF4Compl_dNdrmag[ref_QSO_dNdrmag_ind] / QSO_dNdrmag[ref_QSO_dNdrmag_ind]
QSO_W_drmag = QLF4Compl_dNdrmag / (scale_factor * QSO_dNdrmag)
scale_factor = QLF4Compl_dNdrmag[ref_QSO_dNdrmag_ind]/QSO_dNdrmag[ref_QSO_dNdrmag_ind]
QSO_W_drmag = QLF4Compl_dNdrmag/(scale_factor*QSO_dNdrmag)
else:
nw_QSO_target = n_QSO
QSO_W_drmag *= nw_QSO_target / np.sum(QSO_dNdrmag * QSO_W_drmag)
QSO_W_drmag *= nw_QSO_target/np.sum(QSO_dNdrmag*QSO_W_drmag)

QSO_W_drmag[QLF4Compl_dNdrmag == 0. ] = 1.
QSO_W_drmag[np.isnan(QSO_W_drmag) ] = 1.
QSO_W_drmag[np.isinf(QSO_W_drmag) ] = 1.

w_QSO_dNdrmag = QSO_dNdrmag * QSO_W_drmag
w_QSO_dNdrmag = QSO_dNdrmag*QSO_W_drmag
nw_QSO = np.sum(w_QSO_dNdrmag)

#------------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#***STARS rmag. DISTRIBUTION NORMALIZATION***

# Random generator seeding for reproducibility
rs = np.random.RandomState(int(initRndm))

list_STARS_sel_ind = []
tab2print = []
tab2print.append(["m_rmag", "M_rmag", "n_STARS_drmag", "n_sel_STARS_drmag", "n_QSO_drmag", "nw_QSO_drmag"])
n_STARS_QSO_ratio = n_STARS / n_QSO
n_STARS_QSO_ratio = n_STARS/n_QSO

for rmag_biNum in range(n_rmag_bin):
m_rmag = rmag_binVect[rmag_biNum]
Expand All @@ -284,10 +272,10 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
n_QSO_drmag = int(QSO_dNdrmag[rmag_biNum])

if NORM_MODE == "NORMALIZE_PER_RMAG_BINS":
n_STARS_QSO_ratio = n_STARS_drmag / n_QSO_drmag
n_STARS_QSO_ratio = n_STARS_drmag/n_QSO_drmag

if (n_STARS_QSO_ratio >= 1.):
n_sel_STARS_drmag = int(n_STARS_drmag / n_STARS_QSO_ratio)
n_sel_STARS_drmag = int(n_STARS_drmag/n_STARS_QSO_ratio)
STARS_rnd_sel_ind = rs.choice(n_STARS_drmag, n_sel_STARS_drmag, replace=False)
else:
# STARS_rnd_sel_ind = np.array([])
Expand All @@ -301,7 +289,7 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
str(n_QSO_drmag),
str(int(w_QSO_dNdrmag[rmag_biNum]))])

tab_title = "STARS / QSO TABLE"
tab_title = "STARS/QSO TABLE"
tab = DoubleTable(tab2print, tab_title)
tab.inner_row_border = True
tab.justify_columns = {0:'center', 1:'center', 2:'center', 3:'center',
Expand All @@ -317,7 +305,6 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
print()

#------------------------------------------------------------------------------

#***COMPUTE AND ADD COLORS***

color_names = ['g_r', 'r_z', 'g_z', 'g_W1', 'r_W1', 'z_W1', 'g_W2', 'r_W2', 'z_W2', 'W1_W2', 'r']
Expand Down Expand Up @@ -352,7 +339,6 @@ def MAG_ERR_Func(FLUX, FLUX_IVAR):
QSO_hdu = pyfits.BinTableHDU.from_columns(list(QSO_hdu.columns) + list_cols)

#------------------------------------------------------------------------------

#***STARS & QSO TRAINING FITS CATALOG STORING***

STARS_hdu.writeto(fpn_STARS_output, overwrite=True)
Expand Down

0 comments on commit fae8935

Please sign in to comment.