Skip to content

Commit

Permalink
Merge pull request #726 from desihub/ADM-more-MS
Browse files Browse the repository at this point in the history 
Update minor elements of the ELG/LRG code for the Main Survey
  • Loading branch information
@geordie666
geordie666 committed May 6, 2021
2 parents 21e9568 + 48322f4 commit e073dd3
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 109 deletions.
7 changes: 6 additions & 1 deletion doc/changes.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ desitarget Change Log
0.57.3 (unreleased)
-------------------

* BGS update. New faint limits to r< 20.175 in DECaLS and to r<20.22 for BASS/MzLS in order to get a target density of ~1400 [`PR #725`_].
* Update the ELG/LRG code for the Main Survey [`PR #726`_]. Includes:
* Deprecate the ``LRG_LOWDENS`` targeting bit. It was never used.
* Upweight 10% of the "filler" ELG sample to the LRG priority.
* New BGS faint limits to obtain a target density of ~1400 [`PR #725`_].
* new limits are r< 20.175 for DECaLS and r<20.22 for BASS/MzLS.
* Add utility functions ``decode/encode_negative_targetid(ra,dec,group)``
unique to at least 2 milliarcsec [`PR #724`_].
* Update the baseline LRG selection [`PR #723`_]. Changes from SV3 include:
Expand Down Expand Up @@ -45,6 +49,7 @@ desitarget Change Log
.. _`PR #723`: https://github.com/desihub/desitarget/pull/723
.. _`PR #724`: https://github.com/desihub/desitarget/pull/724
.. _`PR #725`: https://github.com/desihub/desitarget/pull/725
.. _`PR #726`: https://github.com/desihub/desitarget/pull/726

0.57.2 (2021-04-18)
-------------------
Expand Down
109 changes: 59 additions & 50 deletions py/desitarget/cuts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,41 @@ def MWS_too_bright(gaiagmag=None, zfibertotflux=None):
return too_bright


def random_fraction_of_trues(fraction, bool_array):
"""Return True for a random subset of array entries that are True.
Parameters
----------
fraction : :class:`float`
The fraction of the True entries to retain as True. Should be
between 0 and 1.
bool_array : :class:`array_like` or `bool`
A boolean array, or scalar.
Returns
-------
:class:`array_like`
The original `bool_array`, with a random `fraction` of ``True``
entries retained as ``True`` and the others set to ``False``.
If a scalar is passed, then a scalar is returned.
"""
# ADM check that the input fraction was between 0 and 1.
if not 0 <= fraction <= 1:
msg = "fraction must be between 0 and 1, not {}".format(fraction)
log.critical(msg)
raise ValueError(msg)

if np.isscalar(bool_array):
# ADM catch the corner case that a scalar was passed.
chosen = np.random.uniform(0, 1) < fraction
else:
# ADM create a random array with the correct fraction of Trues.
chosen = np.random.random(len(bool_array)) < fraction

# ADM return True for the subset in bool_array that was chosen.
return bool_array & chosen


def _gal_coords(ra, dec):
"""Shift RA, Dec to Galactic coordinates.
Expand Down Expand Up @@ -478,18 +513,18 @@ def isLRG_colors(gflux=None, rflux=None, zflux=None, w1flux=None,
lrg &= (gmag - w1mag > 2.9) | (rmag - w1mag > 1.8) # low-z cuts
lrg &= (
((rmag - w1mag > (w1mag - 17.14) * 1.8)
& (rmag - w1mag > (w1mag - 16.33) * 1.))
& (rmag - w1mag > (w1mag - 16.33) * 1.))
| (rmag - w1mag > 3.33)
) # double sliding cuts and high-z extension
) # double sliding cuts and high-z extension
else:
lrg &= zmag - w1mag > 0.8 * (rmag - zmag) - 0.6 # non-stellar cut
lrg &= zfibermag < 21.61 # faint limit
lrg &= (gmag - w1mag > 2.97) | (rmag - w1mag > 1.8) # low-z cuts
lrg &= (
((rmag - w1mag > (w1mag - 17.13) * 1.83)
& (rmag - w1mag > (w1mag - 16.31) * 1.))
& (rmag - w1mag > (w1mag - 16.31) * 1.))
| (rmag - w1mag > 3.4)
) # double sliding cuts and high-z extension
) # double sliding cuts and high-z extension

return lrg

Expand Down Expand Up @@ -898,11 +933,12 @@ def notinMWS_main_mask(gaia=None, gfracmasked=None, gnobs=None, gflux=None,

return mws


def isMWS_faint_colors(gflux=None, rflux=None, zflux=None, w1flux=None, w2flux=None,
pmra=None, pmdec=None, parallax=None, parallaxerr=None,
obs_rflux=None, objtype=None, paramssolved=None,
gaiagmag=None, gaiabmag=None, gaiarmag=None, gaiaaen=None,
primary=None, south=True):
pmra=None, pmdec=None, parallax=None, parallaxerr=None,
obs_rflux=None, objtype=None, paramssolved=None,
gaiagmag=None, gaiabmag=None, gaiarmag=None, gaiaaen=None,
primary=None, south=True):
"""Set of cuts to define a fainter extension to the MWS main sample.
(see, e.g., :func:`~desitarget.cuts.isMWS_main` for parameters).
"""
Expand Down Expand Up @@ -955,6 +991,7 @@ def isMWS_faint_colors(gflux=None, rflux=None, zflux=None, w1flux=None, w2flux=N

return faint_red, faint_blue


def isMWS_main_colors(gflux=None, rflux=None, zflux=None, w1flux=None, w2flux=None,
pmra=None, pmdec=None, parallax=None, parallaxerr=None,
obs_rflux=None, objtype=None, paramssolved=None,
Expand Down Expand Up @@ -2461,6 +2498,14 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux,
elg_south = elg_vlo_south | elg_lop_south
elg = elg_vlo | elg_lop

# ADM form a seed using zflux in case we parallelized by HEALPixel.
# SJB seeds must be within 0 - 2**32-1
uniqseed = int(np.mean(zflux)*1e5) % (2**32 - 1)
np.random.seed(uniqseed)
# ADM 10% of ELG_LOP and ELG_VLO targets need ELG_HIP set.
elg_hip = random_fraction_of_trues(0.1, elg_lop)
elg_hip |= random_fraction_of_trues(0.1, elg_vlo)

# ADM initially set everything to arrays of False for QSO selection.
# ADM zeroth element stores the northern targets bits (south=False).
qso_classes = [[tcfalse, tcfalse], [tcfalse, tcfalse]]
Expand Down Expand Up @@ -2527,36 +2572,8 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux,
bgs_wise = (bgs_wise_north & photsys_north) | (bgs_wise_south & photsys_south)

# ADM 20% of the BGS_FAINT sources need the BGS_FAINT_HIP bit set.
# ADM form a seed using zflux in case we parallelized by HEALPixel.
# SJB seeds must be within 0 - 2**32-1
# SJB np1.18 scalar vs. vector support, but note that HIP won't be
# set identically for vector vs. calling scalar N times.
percent = 0.2
tnecrep = int(1/percent)
uniqseed = int(np.mean(zflux)*1e5) % (2**32 - 1)
np.random.seed(uniqseed)
is_bgs_hip = None
if np.isscalar(bgs_faint):
if bgs_faint:
is_bgs_hip = np.random.uniform(0, 1) < percent
else:
w = np.where(bgs_faint)[0]
nbgsf = len(w)
if nbgsf > 0:
is_bgs_hip = np.random.choice(w, nbgsf//tnecrep, replace=False)

# ADM similarly, 10% of the ELG_LOP sources need the ELG_HIP bit set.
percent = 0.1
tnecrep = int(1/percent)
is_elg_hip = None
if np.isscalar(elg_lop):
if elg_lop:
is_elg_hip = np.random.uniform(0, 1) < percent
else:
w = np.where(elg_lop)[0]
nelglop = len(w)
if nelglop > 0:
is_elg_hip = np.random.choice(w, nelglop//tnecrep, replace=False)
# ADM note that the random seed is set after the ELG selection.
bgs_faint_hip = random_fraction_of_trues(0.2, bgs_faint)

# ADM initially set everything to arrays of False for the MWS selection
# ADM the zeroth element stores the northern targets bits (south=False).
Expand Down Expand Up @@ -2669,18 +2686,14 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux,
desi_target |= qso * desi_mask.QSO
desi_target |= qsohiz * desi_mask.QSO_HIZ

# ADM set fraction of the ELG_LOP/ELG_VLO targets to ELG_HIP.
desi_target |= elg_hip * desi_mask.ELG_HIP

# ADM Standards.
desi_target |= std_faint * desi_mask.STD_FAINT
desi_target |= std_bright * desi_mask.STD_BRIGHT
desi_target |= std_wd * desi_mask.STD_WD

# ADM set fraction of the ELG_LOP targets to ELG_HIP.
if is_elg_hip is not None:
if is_elg_hip is True:
desi_target |= desi_mask.ELG_HIP
else:
desi_target[is_elg_hip] |= desi_mask.ELG_HIP

# BGS targets, south.
bgs_target = bgs_bright_south * bgs_mask.BGS_BRIGHT_SOUTH
bgs_target |= bgs_faint_south * bgs_mask.BGS_FAINT_SOUTH
Expand All @@ -2697,11 +2710,7 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux,
bgs_target |= bgs_wise * bgs_mask.BGS_WISE

# ADM set fraction of the BGS_FAINT targets to BGS_FAINT_HIP.
if is_bgs_hip is not None:
if is_bgs_hip is True:
bgs_target |= bgs_mask.BGS_FAINT_HIP
else:
bgs_target[is_bgs_hip] |= bgs_mask.BGS_FAINT_HIP
bgs_target |= bgs_faint_hip * bgs_mask.BGS_FAINT_HIP

# ADM MWS main, nearby, and WD.
mws_target = mws_broad * mws_mask.MWS_BROAD
Expand Down
15 changes: 1 addition & 14 deletions py/desitarget/data/targetmask.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@ desi_mask:
- [ELG, 1, "ELG", {obsconditions: DARK}]
- [QSO, 2, "QSO", {obsconditions: DARK}]

#- ADM LRG sub-classes used in SV3 but not in the Main Survey.
- [LRG_LOWDENS, 3, "LRG selected using cuts that produce a lower (~600 per sq. deg.) target density", {obsconditions: DARK}]

#- ADM QSO sub-classes. Used in SV1 and SV2 but ultimately deprecated for the Main Survey.
- [QSO_HIZ, 4, "QSO selected using high-redshift Random Forest (informational bit)", {obsconditions: DARK}]

Expand All @@ -25,14 +22,12 @@ desi_mask:
- [QSO_NORTH, 10, "QSO cuts tuned for Bok/Mosaic data", {obsconditions: DARK}]
- [ELG_LOP_NORTH, 11, "ELG at standard (ELG) priority tuned for Bok/Mosaic data", {obsconditions: DARK}]
- [ELG_VLO_NORTH, 12, "Very-low priority ELG (filler) tuned for Bok/Mosaic data", {obsconditions: DARK}]
- [LRG_LOWDENS_NORTH, 13, "LRG cuts (lower density) tuned for Bok/Mosaic data", {obsconditions: DARK}]

- [LRG_SOUTH, 16, "LRG cuts tuned for DECam data", {obsconditions: DARK}]
- [ELG_SOUTH, 17, "ELG cuts tuned for DECam data", {obsconditions: DARK}]
- [QSO_SOUTH, 18, "QSO cuts tuned for DECam data", {obsconditions: DARK}]
- [ELG_LOP_SOUTH, 19, "ELG at standard (ELG) priority tuned for DECam data", {obsconditions: DARK}]
- [ELG_VLO_SOUTH, 20, "Very-low priority ELG (filler) tuned for DECam data", {obsconditions: DARK}]
- [LRG_LOWDENS_SOUTH, 21, "LRG cuts (lower density) tuned for DECam data", {obsconditions: DARK}]

#- Calibration targets
- [SKY, 32, "Blank sky locations", {obsconditions: DARK|GRAY|BRIGHT|BACKUP|TWILIGHT12|TWILIGHT18}]
Expand Down Expand Up @@ -270,23 +265,20 @@ priorities:
ELG_VLO: {UNOBS: 2999, MORE_ZGOOD: 2, DONE: 2, OBS: 1, DONOTOBSERVE: 0, MORE_MIDZQSO: 0}

# ADM Informational bits. Don't let them set priorities.
LRG_LOWDENS: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0, MORE_MIDZQSO: 0}
QSO_HIZ: SAME_AS_LRG_LOWDENS
QSO_HIZ: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0, MORE_MIDZQSO: 0}

# ADM don't prioritize a N/S target if it doesn't have other bits set.
LRG_NORTH: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0, MORE_MIDZQSO: 0}
ELG_NORTH: SAME_AS_LRG_NORTH
QSO_NORTH: SAME_AS_LRG_NORTH
ELG_LOP_NORTH: SAME_AS_LRG_NORTH
ELG_VLO_NORTH: SAME_AS_LRG_NORTH
LRG_LOWDENS_NORTH: SAME_AS_LRG_NORTH

LRG_SOUTH: SAME_AS_LRG_NORTH
ELG_SOUTH: SAME_AS_LRG_NORTH
QSO_SOUTH: SAME_AS_LRG_NORTH
ELG_LOP_SOUTH: SAME_AS_LRG_NORTH
ELG_VLO_SOUTH: SAME_AS_LRG_NORTH
LRG_LOWDENS_SOUTH: SAME_AS_LRG_NORTH

# ADM calibration and other non-standard targets.
BAD_SKY: {UNOBS: 0, OBS: 0, DONE: 0, MORE_ZWARN: 0, MORE_ZGOOD: 0, MORE_MIDZQSO: 0}
Expand Down Expand Up @@ -417,22 +409,17 @@ numobs:
ELG_HIP: 1
ELG_VLO: 1

# ADM LRG_LOWDENS is a purely informational bit.
LRG_LOWDENS: 0

# ADM don't observe a N/S target if it doesn't have other bits set.
LRG_NORTH: 0
ELG_NORTH: 0
QSO_NORTH: 0
ELG_LOP_NORTH: 0
ELG_VLO_NORTH: 0
LRG_LOWDENS_NORTH: 0
LRG_SOUTH: 0
ELG_SOUTH: 0
QSO_SOUTH: 0
ELG_LOP_SOUTH: 0
ELG_VLO_SOUTH: 0
LRG_LOWDENS_SOUTH: 0
BAD_SKY: 0

#- Standards and sky are treated specially; NUMOBS doesn't apply.
Expand Down
39 changes: 19 additions & 20 deletions py/desitarget/lyazcat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ def add_abs_data(zcat, coaddname):
* Z_ABS - The highest redshift of MgII absorption
* Z_ABS_CONF - The confidence value for this redshift.
Notes
-----
- The original function was written by Lucas Napolitano (LGN) and
Expand All @@ -365,7 +365,7 @@ def add_abs_data(zcat, coaddname):
first_line_wave = 2796.3543
second_line_wave = 2803.5315
rf_line_sep = second_line_wave - first_line_wave

# LGN Define hyperparameters
rf_err_margain = 0.50
kernel_smooth = 2
Expand All @@ -374,10 +374,10 @@ def add_abs_data(zcat, coaddname):
snr_threshold = 3.0
qi_min = 0.01
sim_fudge = 0.94

# LGN Intialize output array.
out_arr = []

# LGN Read the coadd file and find targetid.
specobj = read_spectra(coaddname)
redrockfile = coaddname.replace('coadd', 'redrock').replace('.fits', '.h5')
Expand Down Expand Up @@ -419,25 +419,25 @@ def add_abs_data(zcat, coaddname):
x_spc, y_flx, y_err = coadd_brz_cameras(wave_arr, flux_arr,
noise_arr)

# LGN Apply a gaussian smoothing kernel using hyperparameters
# LGN Apply a gaussian smoothing kernel using hyperparameters
# defined above.
smooth_yflx = convolve(y_flx, kernel)
# LGN Estimate the continuum using median filter.
continuum = medfilt(y_flx, med_filt_size)
# LGN Run the doublet finder.

# LGN Run the doublet finder.
residual = continuum - y_flx

# LGN Generate groups of data with positive residuals.
# LGN/EBL: The following is from a stackoverlow thread:
# https://stackoverflow.com/questions/3149440/python-splitting-list-based-on-missing-numbers-in-a-sequence
groups = []
for k, g in groupby(enumerate(np.where(residual>0)[0]), lambda x: x[0] - x[1]):
for k, g in groupby(enumerate(np.where(residual > 0)[0]), lambda x: x[0] - x[1]):
groups.append(list(map(itemgetter(1), g)))

# LGN Intialize the absorbtion line list.
absorb_lines = []

for group in groups:
# LGN Skip groups of 1 or 2 data vals, these aren't worthwhile
# peaks and cause fitting issues.
Expand All @@ -451,15 +451,15 @@ def add_abs_data(zcat, coaddname):
model = models.Gaussian1D(amplitude=np.nanmax(residual[group]),
mean=np.average(x_spc[group]))
fm = fitter(model=model, x=x_spc[group], y=residual[group])
#LGN Unpack the model fit data
# LGN Unpack the model fit data.
amp, cen, stddev = fm.parameters

absorb_lines.append([amp, cen, stddev, snr])

# LGN Extract the highest z feature and associated quality index (QI)
hz = 0
hz_qi = 0
# LGN This is particuarly poorly implemented, using range(len) so
# LGN This is particuarly poorly implemented, using range(len) so
# I can slice to higher redshift lines only more easily.
for counter in range(len(absorb_lines)):
line1 = absorb_lines[counter]
Expand All @@ -475,21 +475,21 @@ def add_abs_data(zcat, coaddname):
err_margain = rf_err_margain * (1 + ztemp)
# LGN for all lines at higher redshifts.
for line2 in absorb_lines[counter+1:]:
# LGN calculate error from expected line seperation
# LGN calculate error from expected line seperation
# given the redshift of the first line.
sep_err = np.abs(line2[1] - line1[1] - line_sep)
# LGN Keep if within error margains.
if sep_err < err_margain:
# LGN Calculate the QI.
# LGN S/N similarity of lines. sim_fudge is defined
# in the hyperparameters above and
# in the hyperparameters above and
# adjusts for the first line being larger,
# kind of a fudge, won't lie.
snr_sim = sim_fudge * line1[3] * line2[3]**(-1.0)
# LGN Rescale to peak at lines having exact same S/N.
if snr_sim > 1:
snr_sim = snr_sim**(-1.0)
# LGN seperation accuracy
# LGN seperation accuracy
# Is '1' if expected seperation = actual seperation.
# Decreases to 0 outside this.
sep_acc = (1 - sep_err) * err_margain**(-1.0)
Expand All @@ -498,10 +498,9 @@ def add_abs_data(zcat, coaddname):
hz = ztemp
hz_qi = qi


out_arr.append([targetid, hz, hz_qi])
# EBL Add the redshift and quality index for each targetid to the

# EBL Add the redshift and quality index for each targetid to the
# zcat file passed to the function.
out_arr = np.array(out_arr)
zcat_args, abs_args = match(zcat['TARGETID'], out_arr[0])
Expand Down
Loading

0 comments on commit e073dd3

Please sign in to comment.