From aa511762d4c4e34c7d2dbfda168a90893d378562 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 17 Jun 2021 13:49:50 -0700 Subject: [PATCH 1/6] workaround for SV3 files that have the data model for Main Survey fibermaps --- py/desitarget/mtl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 9b77dc63e..c131cb674 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1211,6 +1211,11 @@ def make_zcat_rr_backstop(zcatdir, tiles, obscon, survey): allzs.append(zz) # ADM read in all of the exposures in the fibermap. fm = fitsio.read(zbestfn, "FIBERMAP") + # ADM in the transition between SV3 and the Main Survey, the + # ADM fibermap data model changed. New columns may need to be + # ADM removed to concatenate old- and new-style fibermaps. + if "PLATE_RA" in fm.dtype.names: + fm = rfn.drop_fields(fm, ["PLATE_RA", "PLATE_DEC"]) # ADM recover the information for unique targets based on the # ADM first entry for each TARGETID. _, ii = np.unique(fm['TARGETID'], return_index=True) From 368c915450b8b8beae94755f4260c63e1ec2cf88 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 17 Jun 2021 13:56:46 -0700 Subject: [PATCH 2/6] fix read_mtl_ledger to read both SV3 and Main Survey files using full data model; this fix is also in PR #753 but I need it here to make progress --- py/desitarget/io.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 974ed339e..819043728 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -2604,6 +2604,10 @@ def read_mtl_ledger(filename, unique=True, isodate=None, from desitarget.mtl import mtldatamodel, survey_data_model # ADM allow for the full set of possible columns. mtldm = survey_data_model(mtldatamodel, survey="main") + # ADM need to include two columns that were briefly included as + # ADM part of the ledgers in version 1.0.0 of desitarget. + fulldescr = mtldm.dtype.descr + [('ZS', 'U2'), ('ZINFO', 'U8')] + mtldm = np.array([], dtype=fulldescr) # ADM the data model can differ depending on survey type. names, forms = [], [] with open(filename) as f: @@ -2613,12 +2617,6 @@ def read_mtl_ledger(filename, unique=True, isodate=None, iname, iform = [i+1 for i, stringy in enumerate(l) if "name" in stringy or "datatype" in stringy] name, form = l[iname][:-1], l[iform][:-1] - - # SB for backwards compatibility with mtl 1.0.0 format, - # SB which doesn't have Z_QN and has placeholder ZS,ZINFO - if name not in mtldm.dtype.names: - continue - names.append(name) if 'string' in form: forms.append(mtldm[name].dtype.str) From 6a2e743e688acf2c59ac9b37820c8655f91a4129 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 17 Jun 2021 16:13:46 -0700 Subject: [PATCH 3/6] for the Main Survey, change the location of the ZTILE (zdone=True/False) file from $ZCAT_DIR/tiles.csv to $MTL_DIR/../ops/tiles-specstatus.ecsv --- py/desitarget/mtl.py | 50 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index c131cb674..048da622d 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -252,15 +252,27 @@ def get_mtl_tile_file_name(secondary=False): return fn -def get_ztile_file_name(): +def get_ztile_file_name(survey='main'): """Convenience function to grab the name of the ZTILE file. + survey : :class:`str`, optional, defaults to "main" + To look up the correct ZTILE filename. Options are ``'main'`` and + ``'svX``' (where X is 1, 2, 3 etc.) for the main survey and + different iterations of SV, respectively. + Returns ------- :class:`str` The name of the ZTILE file. """ - fn = "tiles.csv" + if survey[:2] == 'sv': + fn = "tiles.csv" + elif survey == 'main': + fn = "tiles-specstatus.ecsv" + else: + msg = "Allowed 'survey' inputs are sv(X) or main, not {}".format(survey) + log.critical(msg) + raise ValueError(msg) return fn @@ -1011,7 +1023,7 @@ def tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey): Parameters ---------- zcatdir : :class:`str` - Full path to the "daily" directory that hosts redshift catalogs. + Full path to the directory that hosts redshift catalogs. mtltilefn : :class:`str` Full path to the file of tiles that have been processed by MTL. obscon : :class:`str` @@ -1028,9 +1040,27 @@ def tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey): :class:`~numpy.array` An array of tiles that have not yet been processed and written to the mtl tile file. + + Notes + ----- + - If `survey` is `'main'` the code assumes the file with zdone for + spectro tiles is `mtltilefn`/../../ops/tiles-specstatus.ecsv + (i.e. it is in the ops directory parallel to the mtl directory). + If `survey` is `'svX'` the code assumes it is `zcatdir`/tiles.csv. """ # ADM read in the ZTILE file. - ztilefn = os.path.join(zcatdir, get_ztile_file_name()) + ztilefn = get_ztile_file_name(survey=survey) + # ADM directory structure is different for SV and the Main Survey. + if survey[:2] == 'sv': + ztilefn = os.path.join(zcatdir, ztilefn) + elif survey == 'main': + opsdir = os.path.dirname(mtltilefn).replace("mtl", "ops") + ztilefn = os.path.join(opsdir, ztilefn) + else: + msg = "Allowed 'survey' inputs are sv(X) or main, not {}".format(survey) + log.critical(msg) + raise ValueError(msg) + tilelookup = Table.read(ztilefn) # ADM the ZDONE column is a string, convert to a Boolean. @@ -1289,9 +1319,9 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None, Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.) for the main survey and different iterations of SV, respectively. zcatdir : :class:`str`, optional, defaults to ``None`` - Full path to the "daily" directory that hosts redshift catalogs. - If this is ``None``, look up the redshift catalog directory from - the $ZCAT_DIR environment variable. + Full path to the directory that hosts redshift catalogs. If this + is ``None``, look up the redshift catalog directory from the + $ZCAT_DIR environment variable. mtldir : :class:`str`, optional, defaults to ``None`` Full path to the directory that hosts the MTL ledgers and the MTL tile file. If ``None``, then look up the MTL directory from the @@ -1320,6 +1350,10 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None, ----- - Assumes all of the relevant ledgers have already been made by, e.g., :func:`~desitarget.mtl.make_ledger()`. + - If `survey` is `'main'` the code assumes the file with the zdone + status for spectro tiles is `mtldir`/../ops/tiles-specstatus.ecsv + (i.e. it is in the ops directory parallel to the mtl directory). + If `survey` is `'svX'` the code assumes it is `zcatdir`/tiles.csv. """ # ADM first grab all of the relevant files. # ADM grab the MTL directory (in case we're relying on $MTL_DIR). @@ -1342,7 +1376,7 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None, # ADM grab the zcat directory (in case we're relying on $ZCAT_DIR). zcatdir = get_zcat_dir(zcatdir) # ADM And contruct the associated ZTILE filename. - ztilefn = os.path.join(zcatdir, get_ztile_file_name()) + ztilefn = os.path.join(zcatdir, get_ztile_file_name(survey=survey)) # ADM grab an array of tiles that are yet to be processed. tiles = tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey) From 42896d5e2b2dee776951708b90caf41de80b616f Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 17 Jun 2021 16:32:39 -0700 Subject: [PATCH 4/6] update the changes docs; code style --- doc/changes.rst | 11 +++++++++++ py/desitarget/lyazcat.py | 2 +- py/desitarget/mtl.py | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/changes.rst b/doc/changes.rst index e5255b00b..520b3bcb0 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,16 +5,27 @@ desitarget Change Log 1.2.0 (unreleased) ------------------ +* Update the data model for Main Survey MTL [`PR #755`_]. Includes: + * Read all types of MTL ledgers, including SV3 and Main Survey 1.0.0. + * Addresses `issue #749`_. + * Also part of `PR #753`_ but I needed it to make progress. + * Change the location of the "ZTILE" (zdone=True/False) file: + * For sv it was $ZCAT_DIR/tiles.csv + * For main it will be $MTL_DIR/../ops/tiles-specstatus.ecsv + * A workaround to process SV3 files that have Main-like fibermaps. * Add new ZWARN bits BAD_SPECQA and BAD_PETALQA [`PR #754`_]. * ``make_zqso_files`` more robust to individual tile,night,petal problems while still processing the others [`PR #752`_]. * Add final QSO decision logic to full Main Survey MTL loop [`PR #751`_]. * Make creation of zqso catalogs robust to missing columns [`PR #750`_]. +.. _`issue #749`: https://github.com/desihub/desitarget/issues/749 .. _`PR #750`: https://github.com/desihub/desitarget/pull/750 .. _`PR #751`: https://github.com/desihub/desitarget/pull/751 .. _`PR #752`: https://github.com/desihub/desitarget/pull/752 +.. _`PR #753`: https://github.com/desihub/desitarget/pull/753 .. _`PR #754`: https://github.com/desihub/desitarget/pull/754 +.. _`PR #755`: https://github.com/desihub/desitarget/pull/755 1.1.1 (2021-05-29) ------------------ diff --git a/py/desitarget/lyazcat.py b/py/desitarget/lyazcat.py index 5f0e377c3..d1263299f 100644 --- a/py/desitarget/lyazcat.py +++ b/py/desitarget/lyazcat.py @@ -646,7 +646,7 @@ def zcat_writer(zcat, outputdir, outputname, # ADM create the header and add the standard DESI dependencies. hdr = {} add_dependencies(hdr) - add_dependencies(hdr, module_names=['quasarnp',]) + add_dependencies(hdr, module_names=['quasarnp', ]) # ADM add the specific lyazcat dependencies hdr['QN_ADDED'] = qn_flag diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 048da622d..181a21051 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1341,8 +1341,8 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None, :class:`str` The name of the MTL tile file that was updated. :class:`str` - The name of the ZTILE file that was used to link TILEIDs to - observing conditions and to determine if tiles were "done". + Name of ZTILE file used to link TILEIDs to observing conditions + to determine if tiles were "done" (that they had zdone=True). :class:`~numpy.array` Information for the tiles that were processed. From abaf3626772a062d7ad1e547798eb676ef25894f Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 17 Jun 2021 16:59:02 -0700 Subject: [PATCH 5/6] update MTL_DIR in the manifest as we'll need the full surveyops trunk for the Main Survey --- etc/desitarget.module | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/desitarget.module b/etc/desitarget.module index fdba488c8..cf93b0ebd 100644 --- a/etc/desitarget.module +++ b/etc/desitarget.module @@ -77,7 +77,7 @@ setenv [string toupper $product] $PRODUCT_DIR # Add any non-standard Module code below this point. # setenv TARG_DIR $env(DESI_ROOT)/target/catalogs -setenv MTL_DIR $env(DESI_ROOT)/target/mtl +setenv MTL_DIR $env(DESI_ROOT)/target/surveyops/mtl setenv DUST_DIR $env(DESI_ROOT)/external/dust/v0_1 setenv CMX_DIR $env(DESI_ROOT)/target/cmx_files setenv SCND_DIR $env(DESI_ROOT)/target/secondary From bdba75f88f4f50c00eb90383ba97b6132bf7c18c Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sat, 19 Jun 2021 09:40:45 -0700 Subject: [PATCH 6/6] minor change to log the correct redshift tile file --- py/desitarget/mtl.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/py/desitarget/mtl.py b/py/desitarget/mtl.py index 181a21051..67a6e62c3 100644 --- a/py/desitarget/mtl.py +++ b/py/desitarget/mtl.py @@ -1375,12 +1375,19 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None, survey=survey, obscon=obscon, ender=form) # ADM grab the zcat directory (in case we're relying on $ZCAT_DIR). zcatdir = get_zcat_dir(zcatdir) - # ADM And contruct the associated ZTILE filename. - ztilefn = os.path.join(zcatdir, get_ztile_file_name(survey=survey)) # ADM grab an array of tiles that are yet to be processed. tiles = tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey) + # ADM contruct the ZTILE filename, for logging purposes. + ztilefn = get_ztile_file_name(survey=survey) + # ADM directory structure is different for SV and the Main Survey. + if survey[:2] == 'sv': + ztilefn = os.path.join(zcatdir, ztilefn) + elif survey == 'main': + opsdir = os.path.dirname(mtltilefn).replace("mtl", "ops") + ztilefn = os.path.join(opsdir, ztilefn) + # ADM stop if there are no tiles to process. if len(tiles) == 0: return hpdirname, mtltilefn, ztilefn, tiles