diff --git a/DMTN-049.tex b/DMTN-049.tex index 90b2d19..0480af5 100644 --- a/DMTN-049.tex +++ b/DMTN-049.tex @@ -13,7 +13,7 @@ \setDocStatus{draft} \setDocAbstract{ -This roadmap guides the Rubin Observatory Data Management (DM) team's efforts to engage with the scientific community of data-rights holders in order to validate and implement one or more existing photometric redshift (photo-$z$) estimators into the Data Release (DR) processing pipeline, and serve the resulting photo-$z$ data products for the DR {\tt Object} catalogs. +This roadmap guides the Rubin Observatory Data Management (DM) team's efforts to engage with the scientific community of data-rights holders in order to implement and validate one or more existing photometric redshift (photo-$z$) estimators into the Data Release (DR) processing pipeline, and serve the resulting photo-$z$ data products for the DR {\tt Object} catalogs. \medskip The DR {\tt Object} catalog photo-$z$ estimates will initially meet a set of {\it minimum scientific attributes} and serve the {\it widest variety of science applications}. @@ -26,12 +26,11 @@ \medskip \textbf{This roadmap is a living document} to guide this joint venture, and it will evolve over time to incorporate input from the science community. -\textbf{The primary milestone in this roadmap is DM's call for "Letters of Recommendation" from the science community regarding photo-$z$ estimators (deadline 2021-09-30).} -From these letters, DM will generate a shortlist of community-vetted photo-$z$ options that meet both the science community's needs for DR {\tt Object} photo-$z$ and the technical directives of the DM System. +In 2021, community input regarding the evaluation criteria for photo-$z$ estimators, and which kind of photo-$z$ estimator(s) should be selected, was gathered via DM's series of ``LSST Photo-$z$ Virtual Forums" and photo-$z$ ``Letters of Recommendation" (LOR). +This input has been incorporated into \S~\ref{sec:eval} and \ref{sec:lor}. \medskip -In support of this milestone, starting in early 2021, DM will host a series of "LSST Photo-z Virtual Forums", discussions open to all interested people, which will focus on defining the evaluation criteria for photo-$z$ estimators (\S~\ref{sec:eval}) and facilitating the development of "Letters of Recommendation" (\S~\ref{sec:lor}). -Further on in this roadmap DM will facilitate science community participation in a "Photo-$z$ Validation Cooperative" based on commissioning data for the shortlisted estimators (\S~\ref{sec:pzcoop}; dates TBD). +Further on in this roadmap DM will facilitate science community participation in a ``Photo-$z$ Validation Cooperative" based on commissioning data for the shortlisted estimators (\S~\ref{sec:pzcoop}; dates TBD). \medskip This roadmap includes possible in-kind contributions from international partnerships to {\tt Object} catalog photo-$z$ for DR1 and beyond, as well as potential processes of improving the DR {\tt Object} catalog photo-$z$ and/or federating user-generated photo-$z$ catalogs during Rubin operations. @@ -45,6 +44,7 @@ \addtohist{3.3}{2021-01-01}{Updates to Section 3: Technical Considerations.}{Jim Bosch, Leanne Guy} \addtohist{4.}{2021-01-20}{Released, merged to master branch.}{Melissa Graham, Jim Bosch, Leanne Guy} \addtohist{4.1}{2021-05-01}{Updated Evaluation Criteria and the call for Letters of Recommendation to incorporate community input (tickets/DM-29784). Merged to master branch.}{Melissa Graham} +\addtohist{4.2}{2022-01-XX}{Updated after the Letters of Recommendation process completed (tickets/DM-31945).}{Melissa Graham} } \begin{document} @@ -85,43 +85,49 @@ \section{Roadmap and Timeline}\label{sec:time} All items in this timeline reference \textbf{\textit{actions taken by DM}}. -{\bf 2021-02-01:} Write a summary of this proposed timeline. Advertise it broadly. +{\bf 2021-02-01:} Wrote a summary of this proposed timeline. Advertised it broadly. -{\bf 2021-02/04:} Hold three LSST Photo-$z$ Virtual Forums, one per month, which focus on ingesting input from the science community regarding the proposed evaluation criteria (\S~\ref{sec:eval}). Advertise the forums broadly. +{\bf 2021-02/04:} Held three ``LSST Photo-$z$ Virtual Forums", one per month, which focused on ingesting input from the science community regarding the proposed evaluation criteria (\S~\ref{sec:eval}). Advertised the forums broadly. -{\bf 2021-05-01:} Write a call for "Letters of Recommendation" from the science community. Advertise it broadly. +{\bf 2021-05-01:} Wrote a call for ``Letters of Recommendation" from the science community. Advertised it broadly. -{\bf 2021-05/08:} Hold four LSST Photo-$z$ Virtual Forums, one per month, which focus on supporting the science community's efforts to prepare "Letters of Recommendation" (\S~\ref{sec:lor}). Advertise the forums broadly. +{\bf 2021-05/08:} Held four ``LSST Photo-$z$ Virtual Forums", one per month, which focused on supporting the science community's efforts to prepare ``Letters of Recommendation" (\S~\ref{sec:lor}). Advertised the forums broadly. -{\bf 2021-09-30:} Deadline for "Letters of Recommendation" from the science community. +{\bf 2021-09-30:} Deadline for ``Letters of Recommendation" from the science community. -{\bf Before commissioning,} generate a shortlist of community-vetted estimators, and add the rationale for these choices to \S~\ref{ssec:lor_choice}. Advertise this update broadly. +{\bf 2022-01-30:} Incorporated additional scientific criteria from the ``Letters of Recommendation" into \S~\ref{sec:eval}. +Added a summary of the ``Letters" and DM's shortlist to \S~\ref{ssec:lor_choice}. -{\bf During commissioning,} facilitate community participation in a "Photo-$z$ Validation Cooperative", which will analyze photo-$z$ estimates from the shortlisted estimators based on commissioning data (\S~\ref{sec:pzcoop}). +{\bf Before commissioning,} identify community members of the shortlisted photo-$z$ teams to join the Rubin Commissioning Team and help to guide implementation and perform science validation for photo-$z$ estimators (see Section 3.2 of \cite{sitcomtn-010}). + +{\bf During commissioning,} DM and the photo-$z$ Commissioning Team members will facilitate community participation in a ``Photo-$z$ Validation Cooperative", which will analyze photo-$z$ estimates from DM's shortlisted estimators based on processed commissioning data released as Data Previews (DP) 1 and 2 (\S~\ref{sec:pzcoop}). The timeline for this effort is contingent on the commissioning schedule. -At the end of commissioning, based on the results of the "Photo-$z$ Validation Cooperative", DM will select one or more estimators to generate the photo-$z$ data products for DR1. +The science community is welcome to test non-shortlisted estimators with the data previews. +At the end of commissioning, based on the results of the ``Photo-$z$ Validation Cooperative", DM will select one or more estimators to generate the photo-$z$ data products for DR1. {\bf During operations}, the Data Production team will generate photo-$z$ for {\tt Object} catalogs using the software and data products delivered by the Construction-era DMS, and make available any and all supporting materials such as documentation or spectral templates. The Rubin Operations team may solicit and collect community feedback on the photo-$z$ performance, and return to any stage of this roadmap to update the attributes, algorithms, implementation, and/or validation of {\tt Object} catalog photo-$z$ for future data releases. Additionally (or instead of a DMS-provided photo-$z$), the Rubin Operations team might choose to federate a user-generated photo-$z$ catalog, as described in \S~\ref{ssec:time_ops_ugfed} below. -Several terms are used above, which are clarified here. \\ -$\bullet$ {\bf Advertise broadly} means posting in the Photo-$z$ Coordination Group at \url{Community.lsst.org} (\S~\ref{ssec:time_pzcoord}), sending emails to Science Collaborations and other lists of potentially interested individuals (e.g., authors of papers containing the terms LSST and photo-$z$), and including in Rubin newsletters. \\ -$\bullet$ The {\bf science community}, in the context of this document, refers to any individuals or groups of data rights holders who plan to use Rubin data products or services for science -- in particular, the {\tt Object} catalog photo-$z$. \\ -$\bullet$ {\bf LSST Photo-$z$ Virtual Forums} will be informal drop-in discussion sessions for Rubin staff and science community members, and held virtually at a variety of times so as to enable people from all timezones to attend at least one. \\ -$\bullet$ {\bf Ingesting input from the science community} means that this document will be updated to include contributions from the science community to (e.g., updating the proposed criteria in \S~\ref{sec:eval}). -Science community input will primarily be ingested from the LSST Photo-$z$ Virtual Forums and from written discussions in the Photo-$z$ Coordination Group at \url{Community.lsst.org} (\S~\ref{ssec:time_pzcoord}). +Several terms are used above, which are clarified here. +\begin{itemize} +\item {\bf Advertise broadly} means posting in the ``Photo-$z$ Coordination Group" at \url{Community.lsst.org} (\S~\ref{ssec:time_pzcoord}), sending emails to Science Collaborations and other lists of potentially interested individuals (e.g., authors of papers containing the terms LSST and photo-$z$), and including in Rubin newsletters. \\ +\item The {\bf science community}, in the context of this document, refers to any individuals or groups of data rights holders who plan to use Rubin data products or services for science -- in particular, the {\tt Object} catalog photo-$z$. \\ +\item {\bf ``LSST Photo-$z$ Virtual Forums"} will be informal drop-in discussion sessions for Rubin staff and science community members, and held virtually at a variety of times so as to enable people from all timezones to attend at least one. \\ +\item {\bf Ingesting input from the science community} means that this document will be updated to include contributions from the science community to (e.g., updating the proposed criteria in \S~\ref{sec:eval}). +Science community input will primarily be ingested from the ``LSST Photo-$z$ Virtual Forums" and from written discussions in the ``Photo-$z$ Coordination Group" at \url{Community.lsst.org} (\S~\ref{ssec:time_pzcoord}). +\end{itemize} \subsection{Photo-$z$ Coordination Group}\label{ssec:time_pzcoord} -To help all groups and individuals from across the Rubin community communicate about photo-$z$ for the LSST throughout this roadmap, a Photo-$z$ Coordination Group has been established as the ``Photometric Redshifts" sub-category in the ``Science" category at \url{Community.lsst.org}; see \url{ls.st/clo4381} for more information. +To help all groups and individuals from across the Rubin community communicate about photo-$z$ for the LSST throughout this roadmap, a ``Photo-$z$ Coordination Group" has been established as the ``Photometric Redshifts" sub-category in the ``Science" category at \url{Community.lsst.org}; see \url{ls.st/clo4381} for more information. This is not a Group that requires anyone to ``join", but rather it is a centralized location for questions, answers, and discussion about photo-$z$. All are encouraged to use this virtual space to raise questions or issues, report plans or progress, seek feedback on ideas, post summary notes from relevant meetings, or post topics and replies about any other LSST photo-$z$ related activities. Posts are welcome from anyone, including e.g., DM staff members, Rubin Operations team members, International Programs members involved with in-kind contributions, Science Collaboration members, and anyone interested in science with LSST photo-$z$. Active communication should help all parties to optimize and synchronize their efforts and reduce redundancy as progress is made along this roadmap to LSST photo-$z$. -There are many advantages of using \url{Community.lsst.org} for the Photo-$z$ Coordination Group. +There are many advantages of using \url{Community.lsst.org} for the ``Photo-$z$ Coordination Group". The platform is suitable for asynchronous conversations, which is important for our global community; it is easy to both search and browse, which allows fast access to information; topic features include threaded replies and cross-linking to enable discussion; markdown allows for nice formatting and embedding objects like images; and individuals may be 'mentioned' to facilitate collaboration. Furthermore, it is an open platform in which anyone may make an account and contribute, and offers user experience enhancements such as notifications. @@ -133,13 +139,13 @@ \subsection{Federating a User-Generated Photo-$z$ Catalog}\label{ssec:time_ops_u As a long-term approach to providing DR {\tt Object} catalog photo-$z$, federating a user-generated catalog might require providing the community team(s) with access to a small (e.g., $\sim10\%$) amount of a data release weeks to months in advance. This would enable the community team(s) to train and calibrate their photo-$z$ estimator, and would minimize any delay between data release and federation (which is desired for many science goals; Appendix \ref{ssec:use_none}). -Facilitating multiple teams sharing their photo-$z$ catalogs might involve hosting a "photo-$z$ server" within the Rubin Science Platform, similar to that of the Dark Energy Survey's Science Portal, an infrastructure for organizing input catalogs, training and running photo-$z$ estimators, and evaluating their output\footnote{A series of YouTube tutorials about the DES Science Portal are available at \url{https://www.youtube.com/playlist?list=PLGFEWqwqBauBIYa8H6KnZ4d-5ytM59vG2}.}, as described by \citet{2018A&C....25...58G}). +Facilitating multiple teams sharing their photo-$z$ catalogs might involve hosting a ``photo-$z$ server" within the Rubin Science Platform, similar to that of the Dark Energy Survey's Science Portal, an infrastructure for organizing input catalogs, training and running photo-$z$ estimators, and evaluating their output\footnote{A series of YouTube tutorials about the DES Science Portal are available at \url{https://www.youtube.com/playlist?list=PLGFEWqwqBauBIYa8H6KnZ4d-5ytM59vG2}.}, as described by \citet{2018A&C....25...58G}). \subsection{International Partnerships (In-Kind Contributions)}\label{ssec:time_inkind} As the process for International Partnerships with Rubin Observatory via in-kind contributions evolved, photometric redshifts emerged as an area in which (1) multiple teams propose to provide different types of contributions and (2) optimizing the LSST scientific returns on these contributions requires coordination across the Rubin community: the in-kind teams, Rubin staff, the Science Collaborations, and the broad community of data rights holders. -Thus, the stages of this roadmap and the Photo-$z$ Coordination Group (\S~\ref{ssec:time_pzcoord}) were developed with the in-kind contributions in mind, to create a timeline and communication methods that will assist with the development and integration of the in-kind program contributions related to photo-$z$. +Thus, the stages of this roadmap and the ``Photo-$z$ Coordination Group" (\S~\ref{ssec:time_pzcoord}) were developed with the in-kind contributions in mind, to create a timeline and communication methods that will assist with the development and integration of the in-kind program contributions related to photo-$z$. At the time of Version 4 of this document, the proposed contributions included, e.g., individuals' expertise; scientific analyses; software algorithms, tools, or infrastructure; computational resources; datasets or observing time. The proposals specify the recipients for these contributions as either the Rubin Observatory project (e.g., DM, commissioning) or the science community (e.g., a Science Collaboration, all data rights holders). @@ -147,11 +153,11 @@ \subsection{International Partnerships (In-Kind Contributions)}\label{ssec:time_ Here are a couple of examples to illustrate how in-kind teams would be involved in this roadmap. The participation level would vary between in-kind teams depending on their type of contribution, for example: \begin{itemize} -\item In-kind teams proposing to contribute photo-$z$ related directable effort or software packages to Rubin Observatory might be helping DM to evaluate the "Letters of Recommendation", or to set up infrastructure to run the "Photo-z Validation Cooperative". -\item In-kind teams proposing to contribute data or software to the Science Collaborations might be more focused on preparing "Letters of Recommendation", writing photo-$z$ algorithms, and then participating in the "Photo-z Validation Cooperative". +\item In-kind teams proposing to contribute photo-$z$ related directable effort or software packages to Rubin Observatory might be helping DM to set up infrastructure to run the ``Photo-$z$ Validation Cooperative". +\item In-kind teams proposing to contribute data or software to the Science Collaborations might have prepared ``Letters of Recommendation", be writing photo-$z$ algorithms, and planning to participate in the ``Photo-$z$ Validation Cooperative". \end{itemize} -As with all other groups and individuals interested in LSST photo-$z$, the in-kind teams should use the Photo-$z$ Coordination Group in the Community Forum for open discussions and Q\&A about their contributions, when possible (with the exception of things like staffing or other sensitive aspects, of course). +As with all other groups and individuals interested in LSST photo-$z$, the in-kind teams should use the ``Photo-$z$ Coordination Group" in the Community Forum for open discussions and Q\&A about their contributions, when possible (with the exception of things like staffing or other sensitive aspects, of course). Since this is a living document, this section can be updated as the role of the in-kind contributors clarifies and evolves after the review process concludes. @@ -159,7 +165,7 @@ \section{Technical Considerations from the DM System}\label{sec:dmcon} The following provides a set of {\it initial} technical considerations for potential photo-$z$ estimators to be implemented in the DM System. The goal is to provide the community with some initial technical boundaries and expectations. -Community input on the feasibility or challenges of these technical considerations is solicited, and this topic will be open for discussion during the LSST Photo-$z$ Virtual Forum series. +This topic was open for discussion during the LSST Photo-$z$ Virtual Forum series, and continued community input on the feasibility or challenges of these technical considerations is welcome at any time (\S~\ref{ssec:time_pzcoord}). The DM team endeavors to be clear about the nature of these technical issues, but the following is subject to change as the DM System evolves and as feedback from the community is considered. Adherence with these technical considerations is one of the {\it proposed} evaluation criteria in \S~\ref{sec:eval}. @@ -170,22 +176,19 @@ \section{Technical Considerations from the DM System}\label{sec:dmcon} As an indication, the computation of photometric redshifts should take on the order of milliseconds per object and not seconds per object. \textbf{Inputs and Outputs:} -Photo-$z$ estimators should require only catalog-level inputs and not require the use of pixels. +Photo-$z$ estimators should require only catalog-level LSST inputs and not require the use of pixels. Inputs may include all measurements in all filter bands for a given object in the DR {\tt Object} catalog. -The DM team will ensure that all measured quantities needed as inputs to photo-$z$ estimators are computed and included in the {\tt Object} catalog (See Appendix~\ref{ssec:dp_objvals}). +The DM team will ensure that all measured quantities needed as inputs to photo-$z$ estimators are computed and included in the {\tt Object} catalog (See Appendix~\ref{ssec:dp_objvals}). +This will include making corrections for MW dust using an external dust map prior to passing fluxes to the photo-$z$ estimator(s). The outputs currently foreseen and budgeted for, irrespective of algorithm, are defined in the LSST Data Products Definition Document (DPDD) \cite{LSE-163}, and are presented in Appendix \ref{sec:dp}. Exact schema definitions for the output user-facing data products are stored in the {\tt GitHub} repository {\tt sdm\_schemas} (\url{https://github.com/lsst/sdm\_schemas}). This repository represents the source of truth for the schema definitions for user-facing data products. A detailed description of schema management in DM is presented in \cite{dmtn-153}. -Community input on the currently defined defined inputs and outputs is solicited and this topic will be open for discussion during the LSST Photo-$z$ Virtual Forum series. - -Any input measured quantities or outputs that are needed by proposed photo-$z$ estimators and that are not part of the current DR {\tt Object} catalog baseline should be presented for discussion during the LSST Photo-$z$ Virtual Forum series and detailed in the "Letters of Recommendation" for photo-$z$ estimators. \textbf{Storage Constraints:} Additional storage may be required for auxiliary input data such as spectral templates, calibration data or training sets (See Appendix \ref{ssec:dp_calib}). Storage for outputs defined in the the DPDD \cite{LSE-163} has already been budgeted for by DM and is presented in Appendix~\ref{ssec:dp_objvals}. -"Letters of Recommendation" for photo-$z$ estimators should list all necessary auxiliary data together with an estimate of the storage required. \textbf{External Data Sets:} The assembly of vetted training sets (e.g., compilations of spectroscopic and many-band photometric redshifts) is a separate but related aspect with its own technical considerations. @@ -196,7 +199,7 @@ \section{Technical Considerations from the DM System}\label{sec:dmcon} Photo-$z$ estimators often require training in order to accurately relate the detailed properties and tendencies of the input photometric measurements to external data. This is rarely something that can be fully automated. Training of estimators such that they can be run at scale on the full {\tt Object} catalog with no human intervention requires domain-specific knowledge and is not within scope of the Rubin Operations team. -Data Management will work with the LSST science community to identify teams who will support the short-listed estimators through the Photo-$z$ Validation Cooperative during commissioning and beyond, into operations. +Data Management will work with the LSST science community to identify teams who will support the short-listed estimators through the ``Photo-$z$ Validation Cooperative" during commissioning and beyond, into operations. These community-lead teams would take the lead on ensuring that their estimators are provided pre-trained to run within the LSST science pipelines. This should include both software and auxiliary data that represents (and probably condenses) the results of any training procedure. @@ -215,7 +218,7 @@ \section{Technical Considerations from the DM System}\label{sec:dmcon} \textbf{Computational Processing Constraints:} Photo-$z$ estimators are expected to have a low-memory-footprint and to operate only on measurements in the LSST {\tt Object} catalog (See Appendix \ref{ssec:dp_objvals}). DM has budgeted for currently known algorithms and continues to revise estimates based on benchmarking of the codebase as development progresses, updating estimates for future processing based on current benchmarking results. -Shortlisted community-vetted photo-z estimators will be similarly benchmarked to enable DM to accurately revise budget estimates. +Shortlisted community-vetted photo-$z$ estimators will be similarly benchmarked to enable DM to accurately revise budget estimates. The DM sizing model through the end of the construction project is described in \cite{dmtn-135}. \textbf{Implementation Language:} @@ -223,17 +226,14 @@ \section{Technical Considerations from the DM System}\label{sec:dmcon} C++ is employed for computationally intensive operations such as direct pixel-level processing as well as for low-level primitive classes and data structures (e.g. images, PSF models, geometric regions), whereas high-level algorithms are written in Python. Most photo-$z$ estimators are expected to fall into the latter category. Estimators written in either C++ or Python should be easy to adapt to run in the LSST science pipelines. -For estimators not written in either C++ or Python, "Letters of Recommendation" should include a description of how the code would be adapted to run from a Python harness. - The effort required not only to integrate and run a photo-$z$ estimator as part of the Rubin Science Pipelines, but also to support it over the long term in the operations era will need to be understood. -Plans for long term maintenance of a photo-$z$ estimator should be addressed in the "Letters of Recommendation". For further details on Data Management's development practices, consult the DM Developer Guide \cite{DevGuide}. \section{Evaluation Criteria} \label{sec:eval} -The following set of proposed evaluation criteria will be used first as a guide for the Letters of Recommendation (\S~\ref{sec:lor}), and then to select which photo-$z$ estimator(s) will be used by DM to generate the {\tt Object} catalog photo-$z$. -These criteria were assembled by the authors and include input from the science community who attended the Photo-$z$ Virtual Forums in early 2021. +The following set of proposed evaluation criteria was used first as a guide for the ``Letters of Recommendation" (\S~\ref{sec:lor}), and will also be used by DM to select photo-$z$ estimator(s) to generate the {\tt Object} catalog photo-$z$. +These criteria were assembled by the authors and include input from the science community who attended the ``LSST Photo-$z$ Virtual Forums" or submitted science use-case ``Letters of Recommendation" in 2021. These criteria will continue to evolve -- and to become more detailed -- during the later stages of this Roadmap, which will further clarify what the community needs and what existing photo-$z$ estimators can deliver. Some supporting material that provides background motivation for these criteria can be found in Appendices \ref{sec:imp}, \ref{sec:dp} and \ref{sec:use}. These criteria would also apply to any user-generated catalogs to be federated with the {\tt Object} table (Appendix \ref{ssec:time_ops_ugfed}). @@ -246,11 +246,12 @@ \subsection{Scientific Utility} The development of such advanced photo-$z$ algorithms is an active research topic within the Dark Energy Science Collaboration \citep{2018arXiv180901669T}, and is a significant effort which the Rubin Observatory staff should not attempt to replicate. \item \textbf{Serve Diverse Science Needs -- } The selected photo-$z$ estimator(s) should serve as wide a variety of scientists as possible, especially those who would or could not generate custom photo-$z$ for themselves (Appendix \ref{sec:use}). -Photo-$z$ estimators that can return reliable results not just for galaxies but also for, e.g., AGN and quasars, and/or that can also classify stellar types, should be considered favorably. +At least one photo-$z$ estimator that can return reliable results not just for galaxies but also for AGN (and, e.g., quasars) should be selected. \item \textbf{Demonstrated Success -- } The selected photo-$z$ estimator(s) should have broad community adoption and demonstrated success with other wide-field optical surveys. It is especially beneficial if those past surveys overlap with the LSST Main Survey, to enable comparisons. -The selected photo-$z$ estimator(s) should have publicly accessible documentation (e.g., journal article, website, schema) that has been used by the science community. +\item \textbf{Accessibility --} The selected photo-$z$ estimator(s) should have publicly accessible documentation (e.g., journal article, website, schema) that has been used by the science community. +The selected photo-$z$ estimator(s) should have publicly accessible code, inputs, and metadata, so that all of this can be made available to the science community for reproducibility. Its output data products should be straightforward to understand and easy to access (Appendix \ref{ssec:dp_pz}). \end{itemize} @@ -275,117 +276,127 @@ \subsection{Scientific Performance} The selected photo-$z$ estimators should, at least initially, meet a minimum quality in order to serve the basic science needs of the community. As described below, it is likely that many existing photo-$z$ estimators can easily meet a fiducial minimum quality. Of course, from this fiducial minimum there is room for optimization. -During the Photo-$z$ Validation Cooperative, metrics to quantitatively evaluate the shortlisted estimators will be developed and used to inform the decision of which estimator to select to provide the {\tt Object} catalog photo-$z$. +During the ``Photo-$z$ Validation Cooperative", metrics to quantitatively evaluate the shortlisted estimators will be developed and used to inform the decision of which estimator to select to provide the {\tt Object} catalog photo-$z$. -\textbf{Proposed minimum performance targets --} +\begin{itemize} +\item \textbf{Proposed minimum performance targets --} Based on the science use-cases in Appendix \ref{sec:use}, the {\tt Object} catalog photo-$z$ could have a point-estimate accuracy of $\sim10\%$ and still meet the basic science needs. The photo-$z$ results should result in a standard deviation of $z_{\rm true}-z_{\rm phot}$ of $\sigma_z < 0.05(1+z_{\rm phot})$, and a catastrophic outlier fraction of $f_{\rm outlier} < 10\%$, over a redshift range of $0.0 < z_{\rm phot} < 2.0$ for galaxies with $i<25$ mag galaxies. Note that this preliminary proposed performance has been shown to be achievable for simulated LSST data with existing photo-$z$ estimators \citep[e.g.,][]{2018AJ....155....1G,2020MNRAS.499.1587S}. +\item \textbf{Robustness -- } Photo-$z$ estimators that are demonstrated to perform well with imperfect priors and/or incomplete training sets, especially at low-$z$, should be prioritized. +\end{itemize} \subsection{Technical Considerations} The selected photo-$z$ estimator(s) should meet the technical considerations described in Section \ref{sec:dmcon}. +Furthermore, the selected photo-$z$ estimator(s) should be able to meet these technical constraints and produce photo-$z$ for the Object catalog by the time of data release, as there are negative scientific impacts of delaying the addition of photo-$z$ to the Object table (\S~\ref{ssec:use_none}). + \section{Letters of Recommendation} \label{sec:lor} -\textbf{Deadline:} Sep 30 2021. +The original call for photo-$z$ ``Letters of Recommendation" (LOR) and the template provided with the call have been moved to Appendix~\ref{sec:orig_LOR}. -\textbf{Submission:} -In the spirit of open cooperation and collaboration towards our collective goal of generating LSST Object PZ that serve a wide variety of science goals, all letters will be public. -Letters should be submitted by creating a new topic in the "Science -- Photometric Redshifts" category at \url{Community.lsst.org}\footnote{Start at this link, \url{https://community.lsst.org/c/sci/photoz}, click "+New Topic" at upper right, enter a title and text (do not worry about tags), and if applicable upload a PDF using the up-arrow icon in the menu bar.}. -Use a title that starts with ``LOR" and provides a bit of detail, e.g., ``LOR for the CMNN PZ Estimator", ``LOR: PZ and Local Volume Science". -Short letters could be presented in the text body of the new topic; longer letters could be provided by uploading an accompanying PDF. +\subsection{Summary of the Letters Submitted (and DM's Shortlist)} \label{ssec:lor_choice} -\textbf{Introduction:} -The Rubin Observatory Data Management (DM) team is tasked with constructing LSST Science Pipelines that produce science-ready data products, and this includes photometric redshift (PZ) estimates for the LSST data release (DR) Object catalog (e.g., \url{ls.st/dpdd}). -As described in the LSST PZ Roadmap (\url{ls.st/dmtn-049}), DM will select one or more existing, community-vetted algorithms to generate Object PZ which, at least initially, meet a set of minimum scientific attributes and serve the widest variety of science applications. +The Rubin Observatory Data Management (DM) team sincerely thanks everyone who engaged with the PZ LOR process. +In total, there were 20 submissions: 19 LOR and 1 ``non-LOR" describing DESC's photo-$z$ activities. +Of the 19 LOR, 12 advocated for particular algorithms, 6 presented scientific use cases, and 1 was a notification of future software development (deep probabilistic networks). -\textbf{LOR Purpose:} -The Rubin science community has a considerable wealth of expertise in generating PZ catalogs and will be the primary users of the Object PZ data products. -These letters provide a formal opportunity for the science community to advocate for one or more photo-$z$ estimators that will meet their minimum scientific needs, and/or to define these minimum needs so that DM can consider them when shortlisting PZ estimators. +Of the 6 LOR describing scientific use cases: +\begin{itemize} +\item1 focused on galaxies +\item2 focused on dark energy +\item3 focused on active galactic nuclei +\end{itemize} +The scientific recommendations made by these 6 LOR are detailed in Appendix~\ref{ssec:use_scilor} and have been incorporated into \S~\ref{sec:eval}. -\textbf{LOR Writers:} -Any group or individual who would use the LSST Object PZ for their future scientific analyses, and/or are developers of potentially suitable PZ estimators, are encouraged to submit an letter. -The DM team is especially interested in hearing the needs of scientists who plan to use the LSST Object catalog but who would/could not generate custom PZ estimates. -LORs are not restricted to Rubin data rights holders. +Of the 12 LOR advocating for particular algorithms: +\begin{itemize} +\item 4 were machine-learning (ML) based codes (GPz; DEmP; PZFlow; DNF) +\item 3 were template-fitting (TF) based codes (LePhare; Phosphoros, BPZ) +\item 2 were hybrid ML+TF codes (Delight, ML-accelerated hierarchical SPS models) +\item 3 were for codes which performed "post-processing" to enhance PZ estimates (e.g., combine PZ estimates, recalibrate PDFs, refine outlier flags) +\end{itemize} -\textbf{LOR Scope:} -Letters could qualitatively (or quantitatively) recommend one or more specific PZ estimators (or type of estimator), but they do not have to: letters could instead recommend minimum attributes of the LSST PZ data product that would enable basic LSST science, or focus on describing the science that the LSST PZ data product should enable. -Discussions on the research and development of new or improved algorithms can be considered as beyond the scope of these letters, as the shortlist will only include currently existing PZ estimators. -New in-depth quantitative analyses of PZ estimator performance can also be considered as beyond the scope of the LORs because such analyses are the focus of the next stage of the PZ roadmap (see below). -That said, no letters will be rejected for extending beyond these scope boundaries, which are provided just to guide and inform the community's efforts. +The LORs for five of the PZ estimators demonstrated that the software was established and would (or would likely) be capable of meeting the scientific, performance, and technical aspects as described in the call for LORs: GPz, DEmP, DNF, LePhare, and BPZ. +The LORs for three of the PZ estimators which are still in development also demonstrated that the software would (or would likely) be capable of meeting these aspects: PZFlow, Delight, and Phosphoros. -\textbf{Guidelines for PZ LORs:} +\textbf{Thus, these eight PZ estimators comprise DM's formal initial shortlist} for potential codes to generate PZ for the LSST Objects catalog: GPz, DEmP, DNF, LePhare, BPZ, PZFlow, Delight, and Phosphoros. +Given that DM's ability to support the implementation and validation of PZ estimators during commissioning will be limited, the five more established estimators (GPz, DEmP, DNF, LePhare, BPZ) would be prioritized. +Further prioritization would be left to the discretion and expertise of the individuals handling the implementation (\S~\ref{ssec:pzcoop_ct}). + +The LOR for the in-development estimator based on ML-accelerated hierarchical SPS models was, due to the newness of the method, unclear about whether it would likely meet the technical aspects defined in \S~\ref{sec:dmcon}, but seems very promising. +The three LORs for the post-processing codes established their potential positive impacts. +Furthermore, the DM team is aware that additional PZ Estimators, such as those analyzed in \cite{2020MNRAS.499.1587S}, would likely be appropriate for the task of generating PZ for LSST Objects: e.g., ANNz2, EAZY, FlexZBoost, METAPhoR, SkyNet, and TPZ. + +%\textbf{In summary, the DM team would like to see the science community apply all of these PZ estimators (and post-processing codes) to commissioning data from the Data Previews during the Photo-$z$ Validation Cooperative (PVC) phase of the PZ Roadmap.} + + +\section{Photo-$z$ Validation Cooperative}\label{sec:pzcoop} + +As the scientific community has a considerable wealth of expertise in generating photo-$z$ catalogs, and will be the primary users of the photo-$z$ data product, they are the best suited to scientifically validate the {\tt Object} catalog photo-$z$. + +Towards this end the DM team plans to facilitate the ``Photo-$z$ Validation Cooperative", during which DM will focus on providing technical support for the implementation of the eight photo-$z$ estimators that were shortlisted based on the community's ``Letters of Recommendation" (\S~\ref{sec:lor}; but the community may also consider other estimators). + +As the DM team is very limited in terms of the support it can provide for implementation, several spots on the Rubin Commissioning Team are being made available to individuals with relevant expertise who commit to implementing the shortlisted PZ estimators, to using the unreleased commissioning data for early tests, and to helping guide the science community's validation efforts with the released commissioning data (\S~\ref{ssec:pzcoop_ct}). + +During the ``Photo-$z$ Validation Cooperative", the science community would focus on: \begin{itemize} -\item Keep scope in mind: to identify the minimum scientific attributes, a wide set of science applications, and established PZ estimators for the LSST Object PZ data products. -\item Follow the template. It's okay to skip some sections, but do not add new ones. -\item Make it short (1-3 pages ideally) -- a lot of detail is not needed at this time. -\item Be qualitative -- quantitative analyses will be the focus of the PZ Validation Cooperative. -\item Refer to DMTN-049 for more detail about the roadmap, evaluation criteria, and LOR. +\item assembling training sets and validation metrics, +\item helping to implement photo-$z$ estimators with the LSST Science Pipelines, +\item using the data previews to validate the estimators' performance, and +\item documenting, evaluating, and sharing the results. \end{itemize} -\textbf{Beyond the LORs:} -The Rubin DM team will use these letters to inform the evaluation criteria used to select PZ estimators, and to assemble a shortlist of viable PZ estimators to be evaluated quantitatively using Rubin Commissioning data during the ``PZ Validation Cooperative" phase of the PZ Roadmap. -DM might add viable PZ estimators to the shortlist if DM thinks that they meet the evaluation criteria and the community's scientific needs, even if they were not mentioned by any of the LORs. -Writing a letter is not required (and will not be construed as a commitment) to participate in the PZ Validation Cooperative. +These activities will inform DM's selection of photo-$z$ estimator(s) for the Object catalog, but more broadly would also enable the science community to prepare for photo-$z$ science with the LSST and to develop user-generated photo-$z$ catalogs. -\subsection{LOR Template} +A preliminary collection of photo-$z$ validation techniques are presented in Appendix \ref{sec:imp}, and a preliminary collection of training and calibration data in Appendix \ref{sec:dp}. +Types of commissioning data that might be most useful for scientific validation of photo-$z$ estimators are described in \S~\ref{ssec:pzcoop_commissioning}. +Documentation regarding the technical aspects of the implementation and validation process is considered beyond the scope of this document and will appear elsewhere in the future. -\textbf{Title:} E.g., ``LOR on behalf of X science", or ``LOR for the X PZ Estimator". \\ -\textbf{Contributors:} Names and affiliations of the letter writers. \\ -\textbf{Co-signers:} If applicable, the names and affiliations of co-signers. +The ``Photo-$z$ Coordination Group" will be the communications hub for the ``Photo-$z$ Validation Cooperative". -\begin{enumerate} -\item Summary Statement -- -\textit{Provide a short statement that introduces the writers' and the letter's main recommendation(s). -Include the citations for any software discussed. -} -\item Scientific Utility -- -Rubin DM seeks to understand LSST PZ-related science cases in order to ensure that the LSST Object PZ data products will be scientifically useful for a wide variety of communities, especially those which would/could not generate custom PZ. -\textit{Describe how you would use the PZ data products for your LSST science, or the LSST science enabled by the PZ estimator(s) you are recommending. -Examples from past experiences would be useful here. -} -\item Outputs -- -Rubin DM seeks to ensure that the selected LSST Object PZ estimator generates science-ready outputs that serve a wide variety of the community's minimum scientific needs. -\textit{If possible, describe the minimum set of PZ outputs that are required for your science, or the outputs generated by the PZ estimator(s) you are recommending. -E.g., full posteriors, point estimates, statistics (mode, mean, standard deviation, skewness, kurtosis), best-fit templates, and/or flags (e.g., quality, failure modes). -} -\item Performance -- -Rubin DM seeks a general understanding of the minimum quality of PZ estimates needed to meet the basic PZ-related science goals of the community. -\textit{If possible, describe the minimum PZ quality that would enable your LSST science (e.g., the minimum point-estimate error at intermediate redshifts, or whatever is relevant to your science goals), or the predicted minimum quality of the PZ estimator(s) you are recommending. -It is understood that this information might not be available at this time. -} -\item Technical Aspects -- -Rubin DM has a set of technical considerations for PZ estimators, regarding their scalability, inputs, outputs, language, external data sets, training, storage, and compute resources. -\textit{If possible -- and this is probably only possible for letters being prepared by PZ algorithm developers -- please briefly address the technical considerations described in Section 3 of the PZ Roadmap (ls.st/dmtn-049). -Details are not necessary, but an evaluation of the technical considerations at the level of ``will meet", ``will probably meet", ``probably will not meet", ``will not meet" would be most helpful at this time. -} -\end{enumerate} -\subsection{Shortlisted Photo-$z$ Estimator(s)} \label{ssec:lor_choice} +\subsection{Seeking Photo-$z$ Expertise for the Rubin Commissioning Team}\label{ssec:pzcoop_ct} -\textit{TBD: This section will summarize the "Letters of Recommendation" and describe which community-vetted photo-$z$ estimator(s) have been shortlisted for the photo-$z$ validation cooperative, and why.} +It is expected that \textit{most} of the science community participating in the ``Photo-$z$ Validation Cooperative" will use the commissioning data that is released as part of Data Preview 1 and 2 (DP1 and DP2), and will not need to formally join the Rubin Commissioning Team and have access to the unreleased commissioning data. -\section{Photo-$z$ Validation Cooperative}\label{sec:pzcoop} +However, as mentioned above the DM team is very limited in terms of the support it can provide for implementation, and furthermore there are some aspects of implementing and validating photo-$z$ estimators which would benefit from a connection with the unreleased commissioning data. -\textbf{This preliminary draft description of this proposed cooperative effort will be updated based on community input.} +Thus, Rubin Observatory is seeking to identify a small group of $\sim$5-10 individuals who have expertise with photo-$z$ estimators to join the Rubin Commissioning Team. -\textbf{Draft:} \\ -As the scientific community has a considerable wealth of expertise in generating photo-z catalogs, and will be the primary users of the photo-z data product, they are the best suited to scientifically validate the {\tt Object} catalog photo-$z$. -Towards this end the DM team plans to facilitate a "Photo-$z$ Validation Cooperative" to analyze photo-$z$ estimates based on LSST commissioning data and share results. -This is intended to be a collaborative endeavor between project and community to maximize the scientific utility of the {\tt Object} catalog photo-$z$ data products. +As a member of the Rubin Commissioning team, these individuals would be responsible for implementing the shortlisted photo-$z$ estimators and leading the science community's validation efforts during DP1 and DP2. +As mentioned in \S~\ref{ssec:lor_choice}, the five more established estimators (GPz, DEmP, DNF, LePhare, BPZ) should be prioritized for implementation, but any further prioritization or the order of implementation would be left to the discretion and expertise of DM and these individuals doing the implementation work. -It is anticipated that this process will start with open communication about what the science community needs to fully participate in the photo-$z$ co-op (e.g., commissioning data, access to tools, computational resources). -During this process, DM will focus on providing technical support for the validation of the photo-$z$ estimators using commissioning data, especially those shortlisted based on the community's "Letters of Recommendation" (\S~\ref{sec:lor}), but the community may also consider estimator(s) that were not shortlisted. -It is expected that the community will be focused on assembling training sets and validation metrics and applying them to the commissioning data, and documenting and sharing the results. -The mode of communication during this time (e.g., an extension of the "Photo-$z$ Virtual Forums") remains to be determined. +Example components of the implementation process which these individuals would be committing to are provided below (see also Appendix \ref{ssec:imp_imp}). -Types of commissioning data that might be most useful for scientific validation of photo-$z$ estimators are described in \S~\ref{ssec:pzcoop_commissioning}. -A preliminary collection of photo-$z$ validation techniques based are presented in Appendix \ref{sec:imp}, and a preliminary collection of training and calibration data in Appendix \ref{sec:dp}. +\begin{itemize} +\item adapting code language for the LSST environment +\item adapting input/output formats for the LSST catalogs +\item ensuring I/O meets storage constraints +\item ensuring processing meets computational constraints +\item assembling and storing external data sets +\item testing estimator training with the unreleased commissioning data +\item early science validation based on unreleased commissioning data +\item optimizing PZ estimator code to ensure scalability +\end{itemize} + +With the exception of items that require access to the unreleased commissioning data, many of the above overlap with activities that the broader science community will engage in as part of the ``Photo-$z$ Validation Cooperative" using the commissioning data released as Data Previews. +The difference is that the PZ-related Commissioning Team members would be committed to implementation and to providing leadership for the community in these tasks. + +{\bf Eligibility --} All Commissioning Team members must have Rubin data rights, as described in \cite{RDO-013}. +For individuals who are part of US/Chilean community, the Commissioning Announcement of Opportunity will be used to guide the process of identifying Commissioning Team members (e.g., Section 4 of \cite{sitcomtn-010}). +Individuals who are associated with an approved international program should consult their program managers to confirm the proposed work fits into the allocated contributions. + +DM will reach out to the shortlisted photo-$z$ estimator teams as one way to identify potential individuals who are interested in taking on this role with the Rubin Commissioning team, and anyone else from the science community who is interested in this opportunity to do photo-$z$ implementation during Commissioning is welcome to contact Melissa Graham (by March 15). + +Interested parties will be asked to prepare a short statement to establish specifically what they would contribute as Commissioning Team members (as individuals or teams) and how much time they can commit. +As with all Commissioning Team members, the minimum level of sustained effort should be about one day per week of sustained effort over a year (0.2 FTE) or at least one contiguous month of focused effort (0.1 FTE), or some reasonable interpolation between these ``sustained" and ``focused" thresholds. + +To enable these individuals to get started on implementation in advance of commissioning, if desired, DM has arranged for 5--10 accounts in the Rubin Science Platform (RSP) at the Interim Data Facility (IDF) during Data Preview 0 (DP0) to be reserved for them, as described in \citep{rtn-004}. -Documentation regarding the technical aspects of the implementation and validation process is considered beyond the scope of this document and will appear elsewhere in the future. \subsection{Potential Commissioning Data for Photo-$z$ Validation}\label{ssec:pzcoop_commissioning} @@ -411,6 +422,8 @@ \subsection{Potential Commissioning Data for Photo-$z$ Validation}\label{ssec:pz Data Preview 2 (DP2) will be based on data taken during the second and third phases of commissioning and processed with the Rubin Science Pipelines. DP2 is more likely to provide a preliminary data set for photo-$z$ validation and possibly some of the needed data for LSST photo-$z$ described in Appendix \ref{ssec:dp_calib}. Covering well-studied fields with DP2 data (e.g., COSMOS) would facilitate photo-$z$ validation; note that the effort to collect community input on the commissioning fields is underway\footnote{\url{https://community.lsst.org/t/community-input-to-the-on-sky-observing-strategy-during-commissioning}}. +As of early 2022, there had been several suggestions for fields that overlap reference datasets which would be suitable for the generation and validation of photo-$z$ estimates. +The Commissioning Team is taking this input into consideration and will work with the ``Photo-$z$ Coordination Group" to develop and announce the observing strategy for DP2 as soon as they can. As a side note for LSST users who are interested in doing science with commissioning data, it should not be assumed that any release of data products based on commissioning surveys will include photo-$z$ estimates. The release of any photo-$z$ catalogs prior to DR1 remains at the discretion of the DM team. @@ -442,7 +455,7 @@ \subsection{Example Validation Tests}\label{ssec:imp_val} Validation tests and quality assessment diagnostics will be necessary to ensure the results meet performance expectations (as defined in \S~\ref{sec:eval}). These lists are based in part on a brainstorming session during the LSST Project and Community Workshop's session on Photometric Redshifts on Aug 14 2019\footnote{E.g., slide 14 of \url{https://docs.google.com/presentation/d/1GEahvDQXIjSL4lLVjDlZHV5zpXLhGQfHwVtucs72Ajg/edit?usp=sharing}}. -Journal articles that demonstrate validation processes for photo-$z$ from multi-band wide-area surveys include {\it "DES science portal: Computing photometric redshifts"} \citep{2018A&C....25...58G}; {\it "Photometric redshifts for Hyper Suprime-Cam Subaru Strategic Program Data Release 1"} \citep{2018PASJ...70S...9T}; and {\it "On the realistic validation of photometric redshifts"} \citep{2017MNRAS.468.4323B}. +Journal articles that demonstrate validation processes for photo-$z$ from multi-band wide-area surveys include {\it ``DES science portal: Computing photometric redshifts"} \citep{2018A&C....25...58G}; {\it ``Photometric redshifts for Hyper Suprime-Cam Subaru Strategic Program Data Release 1"} \citep{2018PASJ...70S...9T}; and {\it ``On the realistic validation of photometric redshifts"} \citep{2017MNRAS.468.4323B}. {\bf Truth Comparisons --} Catalogs of true redshifts can be obtained by withholding some fraction of the training set or by cross-matching to external spectroscopic catalogs. Validation metrics should include at least those used to define the minimum performance of the LSST photo-$z$ for basic science needs (\S~\ref{sec:eval}). A list of potential metrics might include the following, and targets or limits on these metric values might apply to subsets in magnitude, color, or redshift: @@ -487,8 +500,8 @@ \subsection{Example Publications that Evaluate Photo-$z$ Estimator Performance}\ \begin{itemize} \item \citet{2010A&A...523A..31H} tested 18 different photo-$z$ codes on the same sets of simulated and real data and found no significantly outstanding method. -\item \citet{2013ApJ...775...93D} tested 11 different photo-$z$ codes on the CANDLES data set ($U$-band through infrared photometry) and also find that no method stands out as the "best'', and that most of the photo-$z$ codes underestimate their redshift errors. -\item \citet{2014MNRAS.445.1482S} used the science verification data (200 square degrees of $grizY$ photometry to a depth of $i_{AB}=24$ magnitudes) of the Dark Energy Survey (DES) to evaluate several photometric redshift estimators. They found that the Trees for Photo-$z$ code (TPZ; \citet{2013ascl.soft04011C}) provided the most accurate results with the highest redshift resolution, and that template-fitting methods also performed well -- especially with priors -- but that in general there was no clear "winner.'' +\item \citet{2013ApJ...775...93D} tested 11 different photo-$z$ codes on the CANDLES data set ($U$-band through infrared photometry) and also find that no method stands out as the ``best'', and that most of the photo-$z$ codes underestimate their redshift errors. +\item \citet{2014MNRAS.445.1482S} used the science verification data (200 square degrees of $grizY$ photometry to a depth of $i_{AB}=24$ magnitudes) of the Dark Energy Survey (DES) to evaluate several photometric redshift estimators. They found that the Trees for Photo-$z$ code (TPZ; \citet{2013ascl.soft04011C}) provided the most accurate results with the highest redshift resolution, and that template-fitting methods also performed well -- especially with priors -- but that in general there was no clear ``winner.'' \item \citet{2018PASJ...70S...9T} provides a comparative analysis of several photo-$z$ estimators applied to their data set from the HSC Strategic Program. Their website\footnote{\url{https://hsc-release.mtk.nao.ac.jp/doc/index.php/photometric-redshifts/}} provides comparative analysis plots for each of them. \item \citet{2020MNRAS.499.1587S} statistically compare the posterior probability distribution functions produced by 12 photo-$z$ estimators for a mock data set that is representative of the LSST, identifying some biases and shortfalls in both the produced PDFs and the evaluation methods used to analyze them. \end{itemize} @@ -506,7 +519,7 @@ \section{Appendix: Data Products Related to LSST Photo-$z$}\label{sec:dp} \subsection{Inputs to Photo-$z$ Estimators}\label{ssec:dp_objvals} It is important to ensure that all measured quantities needed by photometric redshift estimators are going to be computed and included in the {\tt Object} table. -Aside from the fluxes and/or apparent magnitudes and errors for each Rubin Observatory filter, which will be provided in the {\tt Object} catalog, the color properties in the {\tt Object} table might be used for photo-$z$. {\it "Colors of the object in 'standard seeing' (for example, the third quartile expected survey seeing in the i band, $\sim$0.9 arcsec) will be measured. These colors are guaranteed to be seeing-insensitive, suitable for estimation of photometric redshifts"} \citedsp{LSE-163}. In the {\tt Object} table the relevant elements are: +Aside from the fluxes and/or apparent magnitudes and errors for each Rubin Observatory filter, which will be provided in the {\tt Object} catalog, the color properties in the {\tt Object} table might be used for photo-$z$. {\it ``Colors of the object in 'standard seeing' (for example, the third quartile expected survey seeing in the i band, $\sim$0.9 arcsec) will be measured. These colors are guaranteed to be seeing-insensitive, suitable for estimation of photometric redshifts"} \citedsp{LSE-163}. In the {\tt Object} table the relevant elements are: \vspace{-15pt} \begin{itemize} \item \texttt{stdColor (float[5])} = {\it 'standard color', color of the object measured in 'standard seeing', suitable for photo-$z$} @@ -542,14 +555,14 @@ \subsection{Output Schema, Access Methods, and Documentation}\label{ssec:dp_pz} \item \texttt{photoZ\_pest (float[10])} = point estimates for the photometric redshift provided in {\tt photoZ} \end{itemize} -The exact point estimate quantities stored in the \texttt{photoZ\_pest} are to-be-determined, {\it "but likely candidates are the mode, mean, standard deviation, skewness, kurtosis, and 1\%, 5\%, 25\%, 50\%, 75\%, and 99\% points from cumulative distribution"} \citedsp{LSE-163}. +The exact point estimate quantities stored in the \texttt{photoZ\_pest} are to-be-determined, {\it ``but likely candidates are the mode, mean, standard deviation, skewness, kurtosis, and 1\%, 5\%, 25\%, 50\%, 75\%, and 99\% points from cumulative distribution"} \citedsp{LSE-163}. Flags that represent potential catastrophic outliers, failure modes, a photo-$z$ consistent with $z=0$, etc., could also be included. {\bf Access Methods --} The user experience is one of the proposed selection criteria for the LSST photo-$z$ estimator (\S~\ref{sec:eval}). Some examples of publicly released photo-$z$ catalogs which were prepared with a user experience that might be desirable for the LSST photo-$z$ include the Dark Energy Survey's Science Portal to serve photometric redshifts \cite{2018A&C....25...58G} and the Hyper SuprimeCam Subaru Strategic Program \cite{2018PASJ...70S...9T}\footnote{\url{https://hsc-release.mtk.nao.ac.jp/doc/index.php/photometric-redshifts/}}. -If the LSST photo-$z$ are not made available in either the {\tt Objects} table or in a federated or joinable catalog -- for example in the case where a community-generated photo-$z$ catalog is replacing the DMS-generated catalog (Appendix \ref{ssec:time_ops_ugfed}) -- and are instead made available via, e.g., a "photo-$z$ server" (as in \cite{2018A&C....25...58G}), then at least the {\tt Object} catalog ID of the most recent data release should be a queryable parameter. +If the LSST photo-$z$ are not made available in either the {\tt Objects} table or in a federated or joinable catalog -- for example in the case where a community-generated photo-$z$ catalog is replacing the DMS-generated catalog (Appendix \ref{ssec:time_ops_ugfed}) -- and are instead made available via, e.g., a ``photo-$z$ server" (as in \cite{2018A&C....25...58G}), then at least the {\tt Object} catalog ID of the most recent data release should be a queryable parameter. If the results of multiple estimators are generated, compressed, and stored in the {\tt Objects} table, then decompression should be straightforward for the user (Appendix \ref{ssec:dp_store}). @@ -567,10 +580,10 @@ \subsection{Storage and Compression}\label{ssec:dp_store} The stored values related to photo-$z$ are subject to the storage space allotted in the {\tt Objects} table as described in \S~\ref{sec:dp}. Both the posteriors and point estimates from several different photo-$z$ estimators could be compressed and stored in this allotted space. -Given the variety of use-cases and the fact that different photo-z estimators produce different results \citep{2020MNRAS.499.1587S}, the option to compute, compress, and store estimates from multiple estimators in the $2\times95$ float might be scientifically desirable. +Given the variety of use-cases and the fact that different photo-$z$ estimators produce different results \citep{2020MNRAS.499.1587S}, the option to compute, compress, and store estimates from multiple estimators in the $2\times95$ float might be scientifically desirable. Efficient $P(z)$ compression algorithms are in development, such as \citet{2014MNRAS.441.3550C} and \citet{2018AJ....156...35M}. -\citet{2014MNRAS.441.3550C} present an algorithm for sparse representation, for which {\it "an entire PDF can be stored by using a 4-byte integer per basis function''} and {\it "only ten to twenty points per galaxy are sufficient to reconstruct both the individual PDFs and the ensemble redshift distribution, $N(z)$, to an accuracy of 99.9\% when compared to the one built using the original PDFs computed with a resolution of $\delta z = 0.01$, reducing the required storage of two hundred original values by a factor of ten to twenty.''} +\citet{2014MNRAS.441.3550C} present an algorithm for sparse representation, for which {\it ``an entire PDF can be stored by using a 4-byte integer per basis function''} and {\it ``only ten to twenty points per galaxy are sufficient to reconstruct both the individual PDFs and the ensemble redshift distribution, $N(z)$, to an accuracy of 99.9\% when compared to the one built using the original PDFs computed with a resolution of $\delta z = 0.01$, reducing the required storage of two hundred original values by a factor of ten to twenty.''} \citet{2018AJ....156...35M} presents a {\tt Python} package for compressing one-dimensional posterior distribution functions (PDFs), demonstrates its performance on several types of photo-$z$ PDFs, and provides a set of recommendations for best practices which should be consulted when DM is making decisions on the DR photo-$z$ data products. However, compression (and decompression by users) will require extra computational resources, which should be estimated and considered, and decompression must be fast and easy for users. @@ -632,7 +645,7 @@ \subsubsection{Stars, Milky Way, and Local Volume}\label{sssec:use_sci_smwlv} LSST-provided {\tt Object} photo-$z$ could be used to reject compact extragalactic objects from stellar samples for population studies and/or spectroscopic follow-up campaigns. \subsubsection{Education and Public Outreach}\label{sssec:use_sci_epo} -The question {\it "how far away is it?"} is common to many EPO initiatives and the {\tt Object} catalog photo-$z$ will be used when preparing information for the public. +The question {\it ``how far away is it?"} is common to many EPO initiatives and the {\tt Object} catalog photo-$z$ will be used when preparing information for the public. EPO might also use photo-$z$ for, e.g., generating 3D graphics that visualize large volumes, or educational programs on the Hubble constant. For EPO purposes, high precision is not as important as outlier reduction for photo-$z$. @@ -652,18 +665,131 @@ \subsection{The Science Impacts of a Data Release Without {\tt Object} Photo-$z$ \subsection{Considerations for Maximizing Early Science}\label{ssec:use_LOY1} -Plans for early science with LSST are still in development and are strongly dependent on the outcomes of commissioning (\citeds{rtn-011}). -Given the uncertainties on the actual construction schedule and commissioning period, several different plans for early science are being considered. +Plans for early science with LSST are still in development and are strongly dependent on the outcomes of commissioning. +Given the uncertainties on the actual construction schedule and commissioning period, several different plans for early science are being considered (\citeds{rtn-011}). The actual state of science verification (SV) and system completeness at handover to Operations will determine the most appropriate course of action to take regarding early science. -In the event that SV is completely successful and the project moves quickly to the LSST cadence and DR1 (Plan "A"), estimators that meet the criteria defined in \S~\ref{sec:eval} and that have been fully validated during the commissioning period will be prioritized. -If, however, the commissioning SV period is cut short, an {\it Early Science Period} of $\approx$3\,\textendash\,6 months that is different to regular survey operations may be enacted (Plan "B"). +In the event that SV is completely successful and the project moves quickly to the LSST cadence and DR1 (Plan ``A"), estimators that meet the criteria defined in \S~\ref{sec:eval} and that have been fully validated during the commissioning period will be prioritized. +If, however, the commissioning SV period is cut short, an {\it Early Science Period} of $\approx$3\,\textendash\,6 months that is different to regular survey operations may be enacted (Plan ``B"). During this period, it is envisaged that estimators that meet the criteria defined in \S~\ref{sec:eval} could continue to work on validation during this {\it Early Science Period}, and that a decision would be taken on which estimator(s) to run for DR1 prior to reverting to the LSST cadence and DR1. -If a further shakedown of operations procedures and data taking is still required at handover to Operations (Plan "C"), the implications for photo-$z$ estimators will be reviewed at that time based on the actual state of the system to determine the best course of action for photo-$z$ estimators. +If a further shakedown of operations procedures and data taking is still required at handover to Operations (Plan ``C"), the implications for photo-$z$ estimators will be reviewed at that time based on the actual state of the system to determine the best course of action for photo-$z$ estimators. In any of these scenarios, estimators that will return the most accurate photo-$z$ as early in the survey as possible and meet the criteria in (\S~\ref{sec:dmcon}) may be prioritized. In the first year of LSST, it might be simpler to use a template-fitting photo-$z$ estimator and avoid potential issues related to computation resources and/or the need to train a machine learning model. Additionally, the likelihood that the large spectroscopic training sets needed for ML photo-$z$ estimators exist will continue to increase through the 2020s. However, if a machine learning estimator is applied for LSST DR1 and DR2, it should be a community-accepted estimator with demonstrated success in other surveys, preferably surveys that overlap the LSST footprint, as this will facilitate the characterization and validation of the LSST photo-$z$. +\subsection{A Summary of Scientific Recommendations from the LORs}\label{ssec:use_scilor} + +As described in \S~\ref{ssec:lor_choice}, of the 6 LOR describing scientific use cases: +\begin{itemize} +\item 1 focused on galaxies +\item 2 focused on dark energy +\item 3 focused on active galactic nuclei +\end{itemize} + +Recommendations that are within scope for the DM team (or were already in the plans or evaluation criteria; \S~\ref{sec:eval}) and are likely to be achieved: +\begin{itemize} +\item adopt a PZ estimator that provides PDFs, point estimates, SED/template fits, physical params, and flags +\item adopt at least one PZ estimator that can generate accurate PZ for galaxies with AGN +\item adopt a PZ estimator that performs well with imperfect priors / incomplete training sets (especially at low-z) +\item provide PZ in the Object catalog at the time of data release (i.e., no delay) +\item make all PZ-related code, inputs, metadata, etc. available (for reproducibility) +\item correct for MW dust before passing fluxes to the PZ estimator +\item provide users with a framework and tools to cross-match to non-LSST catalogs +\end{itemize} +Any of these recommendations which were not already part of the evaluation criteria in \S~\ref{sec:eval} have been added. + +Recommendations that present a challenge: +\begin{itemize} +\item incorporate non-LSST, non-optical photometry into the PZ estimates +\item cross-match the LSST Object catalogs with external data sets (e.g., an X-ray AGN catalog) and provide the results with flags and identifiers +\item release revised PZ data products every few months as new sky regions are observed +\item link Object identifiers between data releases (so users can get new host data for old alerts) +\item in the alert packets, provide host galaxy Object data (PZ, photometry, shape) and non-LSST survey data for host galaxies (spec-z, PZ, photometry) +\end{itemize} +These challenges are out of scope for the Rubin Construction project (and some violate the current Rubin data rights policy), and some items (such as providing cross-matched catalogs) are best left to the expertise of the science community. +However, just because an idea or data product presents a challenge, or is beyond the scope of Rubin construction and commissioning, does not mean it could never be accomplished. +Adopting and federating value-added user-generated data products, and updating algorithms in the LSST Science Pipelines, is an Operations-era (2024 and beyond) activity that will take input from the science community via the Rubin Users Committee, the Community Engagement Team, the Science Advisory Committee, and other such groups. +It is very useful to have these suggestions raised early on so that the DM team can help the science community prepare for Operations; all of the LOR providing these recommendations are much appreciated. + + +\section{Original Call for Photo-$z$ Letters of Recommendation}\label{sec:orig_LOR} + +\textbf{Deadline:} Sep 30 2021. + +\textbf{Submission:} +In the spirit of open cooperation and collaboration towards our collective goal of generating LSST Object PZ that serve a wide variety of science goals, all letters will be public. +Letters should be submitted by creating a new topic in the ``Science -- Photometric Redshifts" category at \url{Community.lsst.org}\footnote{Start at this link, \url{https://community.lsst.org/c/sci/photoz}, click ``+New Topic" at upper right, enter a title and text (do not worry about tags), and if applicable upload a PDF using the up-arrow icon in the menu bar.}. +Use a title that starts with ``LOR" and provides a bit of detail, e.g., ``LOR for the CMNN PZ Estimator", ``LOR: PZ and Local Volume Science". +Short letters could be presented in the text body of the new topic; longer letters could be provided by uploading an accompanying PDF. + +\textbf{Introduction:} +The Rubin Observatory Data Management (DM) team is tasked with constructing LSST Science Pipelines that produce science-ready data products, and this includes photometric redshift (PZ) estimates for the LSST data release (DR) Object catalog (e.g., \url{ls.st/dpdd}). +As described in the LSST PZ Roadmap (\url{ls.st/dmtn-049}), DM will select one or more existing, community-vetted algorithms to generate Object PZ which, at least initially, meet a set of minimum scientific attributes and serve the widest variety of science applications. + +\textbf{LOR Purpose:} +The Rubin science community has a considerable wealth of expertise in generating PZ catalogs and will be the primary users of the Object PZ data products. +These letters provide a formal opportunity for the science community to advocate for one or more photo-$z$ estimators that will meet their minimum scientific needs, and/or to define these minimum needs so that DM can consider them when shortlisting PZ estimators. + +\textbf{LOR Writers:} +Any group or individual who would use the LSST Object PZ for their future scientific analyses, and/or are developers of potentially suitable PZ estimators, are encouraged to submit an letter. +The DM team is especially interested in hearing the needs of scientists who plan to use the LSST Object catalog but who would/could not generate custom PZ estimates. +LORs are not restricted to Rubin data rights holders. + +\textbf{LOR Scope:} +Letters could qualitatively (or quantitatively) recommend one or more specific PZ estimators (or type of estimator), but they do not have to: letters could instead recommend minimum attributes of the LSST PZ data product that would enable basic LSST science, or focus on describing the science that the LSST PZ data product should enable. +Discussions on the research and development of new or improved algorithms can be considered as beyond the scope of these letters, as the shortlist will only include currently existing PZ estimators. +New in-depth quantitative analyses of PZ estimator performance can also be considered as beyond the scope of the LORs because such analyses are the focus of the next stage of the PZ roadmap (see below). +That said, no letters will be rejected for extending beyond these scope boundaries, which are provided just to guide and inform the community's efforts. + +\textbf{Guidelines for PZ LORs:} +\begin{itemize} +\item Keep scope in mind: to identify the minimum scientific attributes, a wide set of science applications, and established PZ estimators for the LSST Object PZ data products. +\item Follow the template. It's okay to skip some sections, but do not add new ones. +\item Make it short (1-3 pages ideally) -- a lot of detail is not needed at this time. +\item Be qualitative -- quantitative analyses will be the focus of the PZ Validation Cooperative. +\item Refer to DMTN-049 for more detail about the roadmap, evaluation criteria, and LOR. +\end{itemize} + +\textbf{Beyond the LORs:} +The Rubin DM team will use these letters to inform the evaluation criteria used to select PZ estimators, and to assemble a shortlist of viable PZ estimators to be evaluated quantitatively using Rubin Commissioning data during the ``PZ Validation Cooperative" phase of the PZ Roadmap. +DM might add viable PZ estimators to the shortlist if DM thinks that they meet the evaluation criteria and the community's scientific needs, even if they were not mentioned by any of the LORs. +Writing a letter is not required (and will not be construed as a commitment) to participate in the PZ Validation Cooperative. + +\subsection{LOR Template} + +\textbf{Title:} E.g., ``LOR on behalf of X science", or ``LOR for the X PZ Estimator". \\ +\textbf{Contributors:} Names and affiliations of the letter writers. \\ +\textbf{Co-signers:} If applicable, the names and affiliations of co-signers. + +\begin{enumerate} +\item Summary Statement -- +\textit{Provide a short statement that introduces the writers' and the letter's main recommendation(s). +Include the citations for any software discussed. +} +\item Scientific Utility -- +Rubin DM seeks to understand LSST PZ-related science cases in order to ensure that the LSST Object PZ data products will be scientifically useful for a wide variety of communities, especially those which would/could not generate custom PZ. +\textit{Describe how you would use the PZ data products for your LSST science, or the LSST science enabled by the PZ estimator(s) you are recommending. +Examples from past experiences would be useful here. +} +\item Outputs -- +Rubin DM seeks to ensure that the selected LSST Object PZ estimator generates science-ready outputs that serve a wide variety of the community's minimum scientific needs. +\textit{If possible, describe the minimum set of PZ outputs that are required for your science, or the outputs generated by the PZ estimator(s) you are recommending. +E.g., full posteriors, point estimates, statistics (mode, mean, standard deviation, skewness, kurtosis), best-fit templates, and/or flags (e.g., quality, failure modes). +} +\item Performance -- +Rubin DM seeks a general understanding of the minimum quality of PZ estimates needed to meet the basic PZ-related science goals of the community. +\textit{If possible, describe the minimum PZ quality that would enable your LSST science (e.g., the minimum point-estimate error at intermediate redshifts, or whatever is relevant to your science goals), or the predicted minimum quality of the PZ estimator(s) you are recommending. +It is understood that this information might not be available at this time. +} +\item Technical Aspects -- +Rubin DM has a set of technical considerations for PZ estimators, regarding their scalability, inputs, outputs, language, external data sets, training, storage, and compute resources. +\textit{If possible -- and this is probably only possible for letters being prepared by PZ algorithm developers -- please briefly address the technical considerations described in Section 3 of the PZ Roadmap (ls.st/dmtn-049). +Details are not necessary, but an evaluation of the technical considerations at the level of ``will meet", ``will probably meet", ``probably will not meet", ``will not meet" would be most helpful at this time. +} +\end{enumerate} + + + \end{document} diff --git a/DMTN049_timeline.pdf b/DMTN049_timeline.pdf deleted file mode 100644 index 88b4ee5..0000000 Binary files a/DMTN049_timeline.pdf and /dev/null differ diff --git a/appendix_evaluating_estimators.tex b/appendix_evaluating_estimators.tex deleted file mode 100644 index 5043151..0000000 --- a/appendix_evaluating_estimators.tex +++ /dev/null @@ -1,142 +0,0 @@ -\section{Appendix: Evaluating Photo-$z$ Estimators}\label{sec:eval} - -\textcolor{red}{This section could be entirely replaced with a citation to the comparative analysis work of Schmidt et al. (2019) when that paper is ready.} - -This section provides more in-depth examples of how to comparatively evaluate different photo-$z$ estimators, in support of the proposed selection criteria put forth in \S~\ref{sec:sel}. - - -\subsection{Lessons from Surveys That Compare Photo-$z$ Estimators}\label{ssec:eval_lit} - -\textbf{DESC photo-$z$ WG --} This science community is full engaged in the development of photo-$z$ routines and their optimization for LSST. -Their work is not reproduced here, \textcolor{red}{but we could add some citations.} - -\textbf{Relevant photo-$z$ testing papers --} \cite{2010A&A...523A..31H} tested 18 different photo-$z$ codes on the same sets of simulated and real data and found no significantly outstanding method. -\cite{2013ApJ...775...93D} test 11 different photo-$z$ codes on the CANDLES data set ($U$-band through infrared) and also find that no method stands out as the "best,'' and that there is a strong dependence of photo-$z$ accuracy on the SNR of the photometry (relevant for our tests at 1 year). -They also found that most of the photo-$z$ codes underestimate their redshift errors. - -\textbf{Lessons from DES --} \cite{2014MNRAS.445.1482S} use the science verification data (200 square degrees of $grizY$ photometry to a depth of $i_{AB}=24$ magnitudes) of the Dark Energy Survey (DES) to evaluate several photometric redshift estimators. -They found that the Trees for Photo-$z$ code (TPZ; \citealt{2013ascl.soft04011C}) provided the most accurate results with the highest redshift resolution, and that template-fitting methods also performed well -- especially with priors -- but that in general there was no clear "winner.'' - -\textbf{Lessons from SDSS --} \cite{2016MNRAS.460.1371B} describes the photo-$z$ adopted for the SDSS DR12. They first use an empirical technique with a large training set to estimate the redshift and it's error, and then fit SED templates with that redshift in order to obtain additional galaxy information such as $K$-correction and spectral type. (Note: \textit{They call it a hybrid technique, but the photo-$z$ sounds like it comes solely from the local linear regression, basically an interpolation in the color-redshift relation}.) - -\textbf{Lessons from the HSC Strategic Program --} \cite{2018PASJ...70S...9T} provides a comparative analysis of several photo-$z$ algorithms applied to their data set, and the program's website\footnote{\url{https://hsc-release.mtk.nao.ac.jp/doc/index.php/photometric-redshifts/}} provides comparative analysis plots for each of them. - -\textbf{Lessons from Lucy's work --} Summer student Lucy Halperin (UW 2016, with Melissa Graham) took what we call the "Brown'' catalog\footnote{We call it the Brown catalog because it uses the SEDs from \cite{2014ApJS..212...18B}} made by Sam Schmidt, with simulated 10-year LSST-like magnitude uncertainties, and ran it through 2 machine learning (ANNz and TPZ) and 2 template-fitting (LePhare and BPZ) photo-$z$ codes. All four returned sets of photo-$z$ with similar standard deviations and biases, but the template-fitting codes were more prone to failures and outliers. Lucy's work found that for template-fitting photo-$z$ codes, the choice of template SED set does make a significant difference in the results, particularly regarding photo-$z$ outliers -- however, this may have been particular to the use of the "Brown''-based galaxy catalog. - - -\subsection{A Preliminary Application of Algorithms to LSST-like Photometry}\label{ssec:eval_ex} - -We apply a couple of of-the-shelf estimators to simulated LSST photometry to demonstrate a few statistical techniques for comparing photometric redshifts. -We have used simulated galaxy catalogs (\S~\ref{sssec:eval_ex_cats}) and three different photo-$z$ estimators (\S~\ref{sssec:eval_ex_estimators}) to generate photometric redshifts. -We then apply a series of diagnostics to evaluate and compare their performance (\S~\ref{sssec:eval_ex_comp}). -Since this is mainly for {\it demonstrative} purposes, the estimators themselves have not been optimized to return the most accurate photo-$z$, and some minor mistakes have been left uncorrected. - -\subsubsection{Simulated Catalogs}\label{sssec:eval_ex_cats} - -We use a randomly chosen 30000 galaxy test subset of the \textsc{LC\_DEEP\_Gonzalez2014a} catalog, which is based on the Millennium simulation \citep{2005Natur.435..629S} and the galaxy formation models of \cite{2014MNRAS.439..264G} and constructed using the lightcone techniques described by \cite{2013MNRAS.429..556M}. We impose a limit on the true catalog redshift of $z<3.5$, and a limit on the apparent $i$-band magnitude of $i<25.5$, and furthermore require galaxies to be detected in the three filters $gri$. The latter requirement means that the test galaxies' apparent magnitude is brighter than a limit defined by a signal-to-noise ratio $<5$ in all three filters $gri$. This limit depends on the number of years of survey elapsed, and since we want to use the same set of test galaxies to analyze the algorithms' results early in the survey, we require this $gri$ non-detection with the expected limits after only 1 year of LSST. These restrictions mean that we end up with a catalog that has with fewer faint galaxies than will be in the LSST 10-year catalogs, and so the 10-year results we consider here are optimistic (but that's fine for our purposes). These restrictions are imposed prior to the random selection of 30000 test galaxies from the larger catalog. We then simulate 4 versions of the test galaxy catalog with errors appropriate for $1$, $2$, $5$, and $10$ years of LSST. We calculate galaxy magnitude uncertainties that are appropriate for the elapsed survey time, and observed photometry is simulated by adding a random scatter proportional to the uncertainties. - -In addition to the test set, we need a training set of galaxies for the machine-learning algorithm to serve as a spectroscopic redshift catalog. Spectroscopic data sets containing tens of thousands of galaxies down to $i>25$ and out to $z>3$ are certainly possible, e.g., the VIMOS Ultra Deep Survey (VUDS; \citealt{2015A&A...576A..79L}). Assuming that the LSST will cover a spectroscopic field like the VUDS to the full 10-year depth during commissioning or with a first-year deep drilling field, we use as our training set a sample of 30000 catalog galaxies with photometric uncertainties equivalent to a 10-year LSST. This training set has the same redshift and magnitude distribution and limits as the galaxy catalogs, which may not be the case for a real spectroscopic set. - -\subsubsection{Considered Photo-$z$ Estimators}\label{sssec:eval_ex_estimators} - -Here we've considered one template-fitting and one machine-learning photo-$z$ algorithm. Hybrid photo-z estimators attempt to mitigate the flaws of either process (e.g., the SDSS DR12 photo-$z$ estimator by \citet{2016MNRAS.460.1371B}, or the Gaussian Processes estimator described by \citet{2017ApJ...838....5L}). - -\textbf{Bayesian Photometric Redshifts} (BPZ; \citealt{2000ApJ...536..571B}) is a template-fitting algorithm with a magnitude prior\footnote{\url{http://www.stsci.edu/~dcoe/BPZ/}}. We use all default parameters, including the $i$-band for the magnitude prior, except that we supply the CFHTLS set of SED templates. This set is 66 SEDs that were used for the CFHTLS photo-$z$ paper and are from \cite{2006A&A...457..841I}, and they were interpolated from the CWW and Kinney models. - -%Note: run with 100 trees was 1h 47m. -\textbf{Trees for Photometric Redshifts} (TPZ; \citealt{2013ascl.soft04011C,2013MNRAS.432.1483C}) is a machine learning algorithm that uses prediction trees and a training set of galaxies with known redshifts. We use all the default parameters from the example, except we increase the number of trees from 4 to 10 (this was set low in the provided example to decrease run time). Since the number of realizations is 2, this is a total of 20 trees. As shown in \cite{2013MNRAS.432.1483C}, the bias and scatter of the resulting photo-$z$ improve the most as the number of trees is increased to 20, and continues to improve more mildly to 100, and then are not much improved beyond 100 trees (i.e., their Figure 9). We also set the maximum redshift to 3.5 and the number of redshift bins to 350. We include both magnitudes and colors and their uncertainties as attributes to be used in the prediction trees, as Lucy's work found that this led to better results. From the TPZ output files, we take as $z_\mathrm{phot}$ the mode of the redshift distribution instead of the mean because this is the peak of the distribution (most likely redshift). \textit{Note: I may have misunderstood how TPZ uses the photometric uncertainties. I thought it treated errors differently from other \texttt{Attributes}, but perhaps it uses them just the same. That makes sense, but means it is inappropriate to include photometric errors as {ttc Attributes} if the train and test sets have different photometric precision.} In case a training set with LSST photometric uncertainties at the level of a 10-year survey is not available from commissioning or a dedicated deep drilling survey by the end of year 1, we also simulate the photo-$z$ results with a training set that has the same level of photometric uncertainty as the test set. - -\textbf{Color-Matched Nearest-Neighbors} (CMNN; Graham et al. 2018) is a photo-$z$ estimator that uses the Mahalanobis distance in color-space to match a galaxy to a training set. We simulate photo-$z$ at 1, 2, 5, and 10 years using a test set of 20000 and a training set of 60000. Training set has the same photometric depth as the test set. - - -\subsubsection{Analysis and Results}\label{sssec:eval_ex_comp} - -In this section we demonstrate several analysis techniques for comparing the output of different photo-$z$ estimators, discussing each in turn below. - -\smallskip \noindent \textbf{The $z_\mathrm{true}$--$z_\mathrm{phot}$ Diagram} \\ -Figure \ref{fig:tzpz} shows the photo-$z$ results in $z_\mathrm{true}$--$z_\mathrm{phot}$ diagrams, which are a typical way to visually asses the output. Galaxies are plotted with a semi-transparent black dot so that density and clustering of points is clear, and galaxies that end up designated as "outliers'' are over-plotted with a more opaque red dot. Problems such as outlier structure from e.g., color-$z$ degeneracies, and quantization of $z_\mathrm{phot}$ is obvious in these kinds of diagrams, as well as a decent overall impression of the scatter and bias. Even though our runs with these estimators have not been optimized, we can make an "example'' assessment of these figures to compare the two estimators. -\textbf{BPZ:} Overall the results are quite poor even with the LSST 10-year predicted photometry, especially the amount of quantization at $z_\mathrm{phot}>1.5$. We find that the results are not improved if we remove the magnitude prior, or use a different SED template set such as those from \cite{2014ApJS..212...18B}. We do know from Lucy's work that the choice of SED template set has a significant impact on the results (this is actually widely known) -- in Lucy's work, the best results were achieved when we used the Brown SEDs with a galaxy catalog for which the photometry was simulated using those same SEDs. However, it's less straightforward to identify the "best'' template SEDs to use with the Euclid galaxy catalog (or real data for that matter). \cite{2014MNRAS.439..264G} describes the wide variety of stellar population spectral synthesis models they used, but it would take quite some work to get them all together into a single catalog to provide to BPZ. -\textbf{TPZ:} The results are quite poor 1 and 2 years, with a lot of quantization in the photo-$z$ and many outliers at low and high redshift, but are significantly improved at 10 years. It is very interesting that there is actually a large improvement if the training set does not have better photometric errors than the test set (i.e., compare the 1 and 2 year results in the third row to the second row), but this may just be related to how we've included the errors as \texttt{Attributes}. Either way, TPZ is sensitive to the provided training set, so an extended investigation into what would truly be a realistic 1 year spectroscopic training set for LSST should be done (e.g., different redshift distributions, different magnitude limits). Although TPZ appears to give better accuracy, we also need to ensure that it gives realistic precision for it's photo-$z$ results. - -\smallskip \noindent \textbf{Statistical Measures} \\ -The important statistical measures that are typically used to assess photo-$z$ results are based on the photo-$z$ error, $\Delta z_{(1+z)} = (z_\mathrm{spec}-z_\mathrm{phot})/(1+z_\mathrm{phot}$. We measure the robust standard deviation in $\Delta z_{(1+z)}$, $\sigma_{\Delta z_{(1+z)}}$ (i.e., "robust'' because it is the standard deviation of galaxies within the IQR); the robust bias, which is the mean deviation $\overline{\Delta z}_{(1+z)}$; and the fraction of outliers, $f_\mathrm{out}$, which is the fraction of galaxies with $|\Delta z_{(1+z)}|> 0.06$ and $>3\sigma_\mathrm{IQR}$ (i.e., must be greater than whichever constraint is larger). In the community sometimes the median deviation in $\Delta z_{(1+z)}$ over all galaxies is used instead of the mean deviation of galaxies within the IQR, but we find the two are comparable. In Figure \ref{fig:stats} we demonstrate a convenient way to statistically compare the results from multiple photo-$z$ estimators. In this case we are comparing the values of these statistical measures when the photo-$z$ estimators are run on galaxy catalogs simulated to represent the 1, 2, 5, and 10 year DRP from LSST (colored lines), for both BPZ (left) and TPZ (right). Different estimators for the same year could also be plotted in a single graph. From these statistical measures it is obvious, for example, that the photo-$z$ from TPZ outperform those from BPZ at all years. In Figure \ref{fig:stat_stat} we show examples of how to compare the statistical measures for the full catalog (i.e., $0.3 \leq z_\mathrm{phot} \leq 3.0$) for different photo-$z$ estimators by plotting, e.g., the fractions of failures versus the outliers, or the bias versus the standard deviation. - -\smallskip \noindent \textbf{Photo-$z$ Uncertainties, $\delta z_\mathrm{phot}$} \\ -In Figure \ref{fig:pzpze} we demonstrate a way to assess the photo-$z$ uncertainties, $\delta z_\mathrm{phot}$, that come out of the estimators: we plot $z_\mathrm{phot}$--$\delta z_\mathrm{phot}$ in the main axis, and above and to the side plot the distributions in $\delta z_\mathrm{phot}$, $z_\mathrm{phot}$, and for comparison, $z_\mathrm{true}$. With BPZ, we can see a strict floor in the photo-$z$ uncertainty that increases with redshift (i.e., the uncertainties are bogus, though this could be a fault of mine in running the code and not of the code itself). For both BPZ and TPZ we can see that in some cases the clumps causing a quantization in photo-$z$ also have high photo-$z$ uncertainty, suggesting that a simple cut on $\delta z_\mathrm{phot}$ could return a sample for which the photo-$z$ distribution matches the true distribution. However, there are other clumps in photo-$z$ that have a relatively low uncertainty. Overall, from these plots we could conclude that the TPZ algorithm returns a redshift distribution that is more similar to the true distribution. Another option here is to plot the photo-$z$ error ($\Delta z_{(1+z)}$). - -\smallskip \noindent \textbf{The Posterior Probability Density Function, $P(z)$} \\ -In Figure \ref{fig:zpdf} we plot examples of the posterior probability density functions output by the BPZ and TPZ algorithms for two test galaxies. One galaxy was chosen as a random representative of galaxies for which an inaccurate and imprecise photo-$z$ was returned from both BPZ and TPZ for all years (top panel of Figure \ref{fig:zpdf}). The other was chosen as a random representative of galaxies which experienced a large and consistent improvement in both the accuracy and precision of its photo-$z$ from year 1 to 10, for both BPZ and TPZ (bottom panel of Figure \ref{fig:zpdf}). These kind of plots demonstrate, for example, the quantization in the TPZ photo-$z$ in the PDFs (this may be related to a mistake in the TPZ input). - -\smallskip \noindent \textbf{Q-Q Plots} \\ -In Figure \ref{fig:qq} we show an example of a quantile-quantile plot using the true $vs.$ the photometric redshift. Each point represents the $z_\mathrm{true}$ and $z_\mathrm{phot}$ for a given quantile, and since the two distributions we are comparing have the same total number of objects (we've neglected any galaxies that have failed to return a photo-$z$), we're simply using $1/N$ as the quantiles. If the Q-Q plot is linear with a slope of 1, we would know the distributions of $z_\mathrm{phot}$ would match that of $z_\mathrm{true}$. In Figure \ref{fig:qq} we can see for both BPZ and TPZ that this is not the case, but that BPZ is worse. - -\clearpage - -\begin{figure*} -\begin{center} -\includegraphics[width=4.0cm]{figures/BPZ_Euclid_Y1_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_10Y1_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_1Y1_tzpz.png} -\includegraphics[width=4.0cm]{figures/CM_10Y1_tzpz.png} -\includegraphics[width=4.0cm]{figures/BPZ_Euclid_Y2_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_10Y2_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_2Y2_tzpz.png} -\includegraphics[width=4.0cm]{figures/CM_10Y2_tzpz.png} -\includegraphics[width=4.0cm]{figures/BPZ_Euclid_Y5_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_10Y5_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_5Y5_tzpz.png} -\includegraphics[width=4.0cm]{figures/CM_10Y5_tzpz.png} -\includegraphics[width=4.0cm]{figures/BPZ_Euclid_Y10_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_10Y10_tzpz.png} -\includegraphics[width=4.0cm]{figures/TPZ_Euclid_10Y10_tzpz.png} -\includegraphics[width=4.0cm]{figures/CM_10Y10_tzpz.png} -\caption{Examples of $z_\mathrm{true}$--$z_\mathrm{phot}$ plots for a variety of algorithms (by column), for 1, 2, 5, or 10 years of survey time elapsed (top to bottom). Galaxies that are statistical outliers are shown in red. \textbf{Left:} results for the BPZ algorithm. \textbf{Center-left:} results for the TPZ algorithm with a 10-year training set. \textbf{Center-right:} results for the TPZ algorithm with a co-evolving training set. \textbf{Right:} results for nearest-neighbors color-matching algorithm, with 50000 test galaxies and $10^6$ training set galaxies with co-evolving photometric errors. \label{fig:tzpz}} -\end{center} -\end{figure*} - -\begin{figure*} -\begin{center} -\includegraphics[width=6cm]{figures/BPZ_Euclid_IQRs.png} -\includegraphics[width=6cm]{figures/TPZ_Euclid_TFD_IQRs.png} -\includegraphics[width=6cm]{figures/BPZ_Euclid_bias.png} -\includegraphics[width=6cm]{figures/TPZ_Euclid_TFD_bias.png} -\includegraphics[width=6cm]{figures/BPZ_Euclid_fout.png} -\includegraphics[width=6cm]{figures/TPZ_Euclid_TFD_fout.png} -\caption{Examples of a statistical measures of the photo-$z$ results from BPZ (left) and TPZ with an evolving training set (right) for simulated catalogs at 1 to 10 years (line colors as in plot legends). From top to bottom we show the robust standard deviation from the IQR, the robust bias, and the fraction of outliers as a function of photo-$z$, with matched $x$- and $y$-axes to facilitate comparison between BPZ and TPZ. \label{fig:stats}} -\end{center} -\end{figure*} - -\begin{figure*} -\begin{center} -\includegraphics[width=8cm]{figures/stat_stat_fout_ffail.png} -\includegraphics[width=8cm]{figures/stat_stat_std_bias.png} -\caption{Examples of how to compare statistical measures over $0.3 \leq z_\mathrm{phot} \leq 3.0$ from different photo-$z$ estimators by plotting one against the other: fraction of failures and outliers (left), and the robust bias and standard deviation (right). In this case we're comparing the statistical measures for TPZ and BPZ from photometry simulated for the LSST at years 1, 2, 5, and 10 (legend). \label{fig:stat_stat}} -\end{center} -\end{figure*} - -\begin{figure*} -\begin{center} -\includegraphics[width=6cm,trim={1cm 1cm 1cm 0cm}, clip]{figures/zp_zpe_bpz_euclid_1_2.png} -\includegraphics[width=6cm,trim={1cm 1cm 1cm 0cm}, clip]{figures/zp_zpe_tpz_euclid_1_1_2.png} -\includegraphics[width=6cm,trim={1cm 1cm 1cm 0cm}, clip]{figures/zp_zpe_bpz_euclid_2_2.png} -\includegraphics[width=6cm,trim={1cm 1cm 1cm 0cm}, clip]{figures/zp_zpe_tpz_euclid_2_2_2.png} -\includegraphics[width=6cm,trim={1cm 1cm 1cm 0cm}, clip]{figures/zp_zpe_bpz_euclid_5_2.png} -\includegraphics[width=6cm,trim={1cm 1cm 1cm 0cm}, clip]{figures/zp_zpe_tpz_euclid_5_5_2.png} -\caption{Examples of plot to compare the photo-$z$ uncertainty ($\delta z_\mathrm{phot}$) between algorithms with $z_\mathrm{phot}$--$\delta z_\mathrm{phot}$ plots from the BPZ (left) and TPZ (right) estimators for simulated catalogs with photometric uncertainties at 1, 2, and 5 years of LSST (top to bottom). Red lines show the distribution of photo-$z$ errors; blue and green lines compare the distributions of true and photometric redshifts. \label{fig:pzpze}} -\end{center} -\end{figure*} - -\begin{figure*} -\begin{center} -\includegraphics[width=13cm]{figures/zpdf_g4.png} -\includegraphics[width=13cm]{figures/zpdf_g7.png} -\caption{Examples of the posterior probability density functions for two test galaxies in all of our simulations: BPZ (left) and TPZ (right) for photometric uncertainties like 1, 2, 5, and 10 years of LSST (rows from top to bottom). In the top panel we choose a galaxy that return inaccurate and imprecise photo-$z$ from all 8 trials, and in the bottom panel we choose a galaxy that experienced a large and consistent improvement in photo-$z$ accuracy and precision from 1 to 10 years with both estimators. \label{fig:zpdf}} -\end{center} -\end{figure*} - -\begin{figure*} -\begin{center} -\includegraphics[width=10cm]{figures/qq_BPZ_TPZ.png} -\caption{Example of a Q-Q plot, using $z_\mathrm{true}$ and $z_\mathrm{phot}$, from the BPZ and TPZ estimators.}\label{fig:qq} -\end{center} -\end{figure*} \ No newline at end of file diff --git a/local.bib b/local.bib index 49c81d8..b6a62f1 100644 --- a/local.bib +++ b/local.bib @@ -1,19 +1,3 @@ -@DocuShare{RTN-011, - author = {Leanne Guy and Robert Blum and {Ivezi{\'c}}, {\v{Z}}eljko }, - title = "{Plans for Early Science}", - year = 2020, - handle = {RTN-011}, - note = {Rubin Technical Note}, - url = {https://rtn-011.lsst.io}, } - -@DocuShare{DMTN-153, - author = {Colin Slater and William O'Mullane }, - title = {Schema Management in DM}, - year = 2019, - month = nov, - handle = {DMTN-153}, - url = {http://DMTN-153.lsst.io } } - @ARTICLE{2020MNRAS.499.1587S, author = {{Schmidt}, S.~J. and {Malz}, A.~I. and {Soo}, J.~Y.~H. and {Almosallam}, I.~A. and {Brescia}, M. and {Cavuoti}, S. and {Cohen-Tanugi}, J. and {Connolly}, A.~J. and {DeRose}, J. and {Freeman}, P.~E. and {Graham}, M.~L. and {Iyer}, K.~G. and {Jarvis}, M.~J. and {Kalmbach}, J.~B. and {Kovacs}, E. and {Lee}, A.~B. and {Longo}, G. and {Morrison}, C.~B. and {Newman}, J.~A. and {Nourbakhsh}, E. and {Nuss}, E. and {Pospisil}, T. and {Tranin}, H. and {Wechsler}, R.~H. and {Zhou}, R. and {Izbicki}, R. and {LSST Dark Energy Science Collaboration}}, title = "{Evaluation of probabilistic photometric redshift estimation approaches for The Rubin Observatory Legacy Survey of Space and Time (LSST)}",