DM-11325: Make prototype pipeline eups-compatible

.gitignore

@@ -1,4 +1,19 @@
-ingested/*
-calibingested/*
-data/*
-MasterCals/*
+*.o
+*.os
+*.pyc
+*.so
+*~
+.cache
+.sconf_temp
+.sconsign.dblite
+bin/*.py
+config.log
+doc/*.inc
+doc/*.tag
+doc/doxygen.conf
+doc/html
+doc/xml
+tests/.cache
+tests/.tests
+ups/*.cfgc
+version.py

README.md

@@ -1,10 +1,18 @@
-# decam_hits
+# ap_pipe
 
-This package contains the Prototype AP Pipeline and will eventually be 
-renamed `ap_pipe`.
+This package contains the Prototype AP Pipeline.
 
-At present, the main script is `decam_process.py`, which allows a
-user to run a portion of the AP Pipeline on data from the DECam HiTS survey.
+The main script is `ap_pipe.py`, which allows a user to run
+each step of the Prototype AP Pipeline on data from DECam.
+
+Individual functions in `ap_pipe.py` will be called by 
+[`ap_verify`](https://github.com/lsst-dm/ap_verify), but the 
+pipeline can also be run on its own without verification metrics using a dataset
+structured like [`ap_verify_hits2015`](https://github.com/lsst/ap_verify_hits2015).
 
 For more detailed documentation, including a tutorial, 
 please see [DMTN-039](https://dmtn-039.lsst.io).
+
+**Note**: the tutorial portion of the technote is not up-to-date with master.
+This will be remedied as part of [DM-11390](https://jira.lsstcorp.org/browse/DM-11390) 
+and/or [DM-11422](https://jira.lsstcorp.org/browse/DM-11422).

Sconstruct

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConstruct("ap_pipe")

doc/.gitignore

@@ -0,0 +1,7 @@
+_build
+py-api
+api
+html
+*.tag
+*.inc
+doxygen.conf

doc/Makefile

@@ -0,0 +1,64 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+	@echo "  dummy      to check syntax errors of document sources"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+	rm -rf py-api
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+ltd: html
+	ltd-mason-travis --html-dir $(BUILDDIR)/html
+
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: dummy
+dummy:
+	$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
+	@echo
+@echo "Build finished. Dummy builder generates no files."

doc/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.doc()

doc/conf.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+from documenteer.sphinxconfig.stackconf import build_package_configs
+
+import lsst.ap.pipe
+
+_g = globals()
+_g.update(build_package_configs(
+    project_name="ap_pipe",
+    copyright="2017 Association of Univerities for "
+              "Research in Astronomy, Inc.",
+    version=lsst.ap.pipe.version.__version__,
+    doxygen_xml_dirname=None))
+
+intersphinx_mapping['astropy'] = ('http://docs.astropy.org/en/stable', None)
+
+# DEBUG only
+automodsumm_writereprocessed = False

doc/requirements.txt

@@ -0,0 +1,5 @@
+Sphinx>=1.4.5
+astropy-helpers>=1.2
+documenteer>=0.1.9
+sphinx-rtd-theme==0.1.9
+ltd-mason==0.2.1

python/lsst/__init__.py

@@ -0,0 +1,27 @@
+# 
+# LSST Data Management System
+#
+# Copyright 2008-2017  AURA/LSST.
+# 
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the LSST License Statement and 
+# the GNU General Public License along with this program.  If not, 
+# see <https://www.lsstcorp.org/LegalNotices/>.
+#
+
+from __future__ import absolute_import
+
+import pkgutil, lsstimport
+__path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/ap/__init__.py

@@ -0,0 +1,27 @@
+# 
+# LSST Data Management System
+#
+# Copyright 2008-2017  AURA/LSST.
+# 
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the LSST License Statement and 
+# the GNU General Public License along with this program.  If not, 
+# see <https://www.lsstcorp.org/LegalNotices/>.
+#
+
+from __future__ import absolute_import
+
+import pkgutil, lsstimport
+__path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/ap/pipe/__init__.py

@@ -0,0 +1,24 @@
+# 
+# LSST Data Management System
+#
+# Copyright 2008-2017  AURA/LSST.
+# 
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the LSST License Statement and 
+# the GNU General Public License along with this program.  If not, 
+# see <https://www.lsstcorp.org/LegalNotices/>.
+#
+
+from __future__ import absolute_import

decam_process.py → python/lsst/ap/pipe/ap_pipe.py

@@ -23,9 +23,6 @@
 '''
 Process raw decam images with MasterCals from ingestion --> difference imaging
 
-TODO: Rename script and GitHub repo to ap_pipe (not decam_hits or decam_process).
-      This will be done in DM-11324.
-
 TODO: Update DMTN-039 to reflect the new user interface.
       I'm postponing this until DM-11422 and/or DM-11390 are complete.
 '''
@@ -78,7 +75,7 @@ def runPipelineAlone():
     Process raw decam images with MasterCals from ingestion --> difference imaging
 
     USAGE:
-    $ python decam_process.py -d dataset_root -o output_location -i "visit=12345, ccd=5"
+    $ python ap_pipe.py -d dataset_root -o output_location -i "visit=12345, ccd=5"
                                      '''))
     parser.add_argument('-d', '--dataset',
                         help="Location on disk of dataset_root, which contains subdirectories of raw data, calibs, etc.")

tests/SConscript

@@ -0,0 +1,3 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+scripts.BasicSConscript.tests()

ups/ap_pipe.build

@@ -0,0 +1,2 @@
+@LSST BUILD@ &&
+build_lsst @PRODUCT@ @VERSION@ @REPOVERSION@

ups/ap_pipe.cfg

@@ -0,0 +1,22 @@
+# -*- python -*-
+
+import lsst.sconsUtils
+
+# Dependencies that provide header files and or libraries should be included here.
+# Pure-Python dependencies do not need to be included.
+# Packages that use pybind11 or boost_tests should declare them as build dependencies.
+# Otherwise, the rules for which packages to list here are the same as those for
+# table files.
+dependencies = {
+}
+
+# For packages that build a C++ library and a Python module, the below should be sufficient.
+# Pure-Python packages should set headers=[], libs=[] (not libs=None). and hasSwigFiles=False.
+# For more information, see the sconsUtils Doxygen documentation.
+config = lsst.sconsUtils.Configuration(
+    __file__,
+    headers=[],
+    libs=[],
+    hasSwigFiles=False,
+    hasDoxygenInclude=False,
+)

ups/ap_pipe.table

@@ -0,0 +1,21 @@
+# For each dependency except python, list it here along with its minimum
+# version number.
+# Very common third-party packages (boost, python, swig, doxygen) 
+# and very low-level LSST packages can be assumed to be recursively
+# included by low-level LSST packages such as utils or daf_base.
+# Any other package whose interface is used should be listed explicitly
+# rather than assuming it will be included recursively.
+setupRequired(boost >= 1.47.0)
+setupRequired(utils >= 4.6.0.0)
+setupRequired(pex_exceptions >= 4.6.0.0)
+setupRequired(ndarray)
+
+setupRequired(obs_decam)
+
+# The following is boilerplate for all packages.
+# See Tech Note DMTN-001 for details on LSST_LIBRARY_PATH
+# Pure-Python packages only need the last line.
+#envPrepend(LD_LIBRARY_PATH, ${PRODUCT_DIR}/lib)
+#envPrepend(DYLD_LIBRARY_PATH, ${PRODUCT_DIR}/lib)
+#envPrepend(LSST_LIBRARY_PATH, ${PRODUCT_DIR}/lib)
+envPrepend(PYTHONPATH, ${PRODUCT_DIR}/python)