DM-15872: Incorporate AP documentation into pipelines.lsst.io

doc/conf.py

@@ -1,4 +1,7 @@
-#!/usr/bin/env python
+"""Sphinx configuration file for an LSST stack package.
+
+This configuration only affects single-package Sphinx documenation builds.
+"""
 
 from documenteer.sphinxconfig.stackconf import build_package_configs
 import lsst.ap.pipe

doc/index.rst

@@ -9,5 +9,4 @@ ap_pipe documentation preview
 .. toctree::
    :maxdepth: 1
 
-   ap_pipe/index
    lsst.ap.pipe/index

doc/lsst.ap.pipe/getting-started.rst

@@ -1,7 +1,9 @@
-.. _getting-started:
+.. py:currentmodule:: lsst.ap.pipe
+
+.. _ap-pipe-getting-started:
 
 ####################################
-Getting started with the AP Pipeline
+Getting started with the AP pipeline
 ####################################
 
 
@@ -10,7 +12,7 @@ Getting started with the AP Pipeline
 Installation
 ============
 
-`lsst.ap.pipe` is available from the `LSST Science Pipelines <https://pipelines.lsst.io/>`_.
+:doc:`lsst.ap.pipe <index>` is available from the `LSST Science Pipelines <https://pipelines.lsst.io/>`_.
 It is installed as part of the ``lsst_apps`` and ``lsst_distrib`` metapackages.
 
 
@@ -20,16 +22,16 @@ Ingesting data files
 ====================
 
 LSST-style image processing typically operates on Butler repositories and does not
-directly interface with data files. `lsst.ap.pipe` is no exception.
+directly interface with data files. :doc:`lsst.ap.pipe <index>` is no exception.
 The process of turning a set of raw data files and corresponding calibration
 products into a format the Butler understands is called ingestion. Ingestion
 can be somewhat camera-specific, and is outside the scope of the AP Pipeline.
 
 .. TODO: Cut or condense above paragraph and link to ingestion-related docs.
 
-A utility to ingest data before running `lsst.ap.pipe` 
-is available in :ref:`ap-verify-run-ingest`. However, this works
-only on datasets which adhere to the :ref:`ap-verify-datasets` format.
+A utility to ingest data before running :doc:`lsst.ap.pipe <index>`
+is available in :ref:`ap_verify <ap-verify-run-ingest>`. However, this works
+only on datasets which adhere to the :doc:`ap_verify dataset </modules/lsst.ap.verify/datasets>` format.
 Alternately, you may use a pre-
 ingested dataset or manually ingest files yourself following the directions
 for a given ``obs_`` package, e.g.,
@@ -62,7 +64,7 @@ For the AP Pipeline to successfully process data, the following is required:
   <https://community.lsst.org/t/creating-and-using-new-style-reference-catalogs/1523>`_
   ingested into a main Butler repository
 
-  - The reference catalogs must be in a directory called `ref_cats` with subdirectories
+  - The reference catalogs must be in a directory called :file:`ref_cats` with subdirectories
     for each catalog containing the appropriate catalog shards.
     We recommend using Pan-STARRS for photometry and gaia for astrometry.
     An example :ref:`config file <command-line-task-config-howto-configfile>` for using these two catalogs can be found in the `ap_verify_hits2015`_ repository.
@@ -72,7 +74,7 @@ For the AP Pipeline to successfully process data, the following is required:
   the command line at runtime
 
   - To check if this requirement has been satisfied, you can inspect the
-    `calibRegistry.sqlite3` created in this repository and ensure the information
+    :file:`calibRegistry.sqlite3` created in this repository and ensure the information
     in the flat, bias, and defect tables is accurate
 
 - **Template images** (of type ``deepCoadd`` by default) for difference imaging
@@ -84,7 +86,7 @@ For the AP Pipeline to successfully process data, the following is required:
 .. _ap_verify_hits2015: https://github.com/lsst/ap_verify_hits2015/
 
 A sample dataset from the `DECam HiTS survey <http://iopscience.iop.org/article/10.3847/0004-637X/832/2/155/meta>`_ 
-that works with ``ap_pipe`` in the :ref:`ap-verify-datasets` format
+that works with ``ap_pipe`` in the :doc:`/modules/lsst.ap.verify/datasets` format
 is available as `ap_verify_hits2015`_. However, this dataset must be
 ingested as described in :ref:`section-ap-pipe-ingesting-data-files`, and the reference
 catalog and defect files must be decompressed and extracted.

doc/lsst.ap.pipe/index.rst

@@ -1,3 +1,5 @@
+.. py:currentmodule:: lsst.ap.pipe
+
 .. _lsst.ap.pipe:
 
 ############
@@ -6,36 +8,43 @@ lsst.ap.pipe
 
 .. Paragraph that describes what this Python module does and links to related modules and frameworks.
 
-The ``lsst.ap.pipe`` module links together a set of common image processing
-tasks so that a user may run one Command-Line Task on a dataset of raw,
-ingested images rather than several. The Alert Production (AP) pipeline
-includes three key data processing Tasks for LSST Prompt Data Products:
-`~lsst.pipe.tasks.ProcessCcdTask` (which includes `~lsst.ip.isr.IsrTask`), 
-`~lsst.ip.diffim.ImageDifferenceTask`, and
-`~lsst.ap.associate.AssociationTask`.
-
-.. Add subsections with toctree to individual topic pages.
+The ``lsst.ap.pipe`` module links together a set of common image processing tasks so that a user may run one Command-Line Task on a dataset of raw, ingested images rather than several.
+The Alert Production (AP) pipeline includes three key data processing Tasks for LSST Prompt Data Products: `~lsst.pipe.tasks.ProcessCcdTask` (which includes `~lsst.ip.isr.IsrTask`),  `~lsst.ip.diffim.ImageDifferenceTask`, and `~lsst.ap.associate.AssociationTask`.
 
 Overview
 ========
 
 .. toctree::
    :maxdepth: 1
 
-   pipeline-overview.rst
-
+   pipeline-overview
+
+.. .. _lsst.ap.pipe-using:
 
-Running the AP Pipeline
-=======================
+Using lsst.ap.pipe
+==================
 
 .. toctree::
    :maxdepth: 1
 
-   getting-started.rst
-   pipeline-tutorial.rst
+   getting-started
+   pipeline-tutorial
+
+.. _lsst.ap.pipe-contributing:
+
+Contributing
+============
+
+``lsst.ap.pipe`` is developed at https://github.com/lsst/ap_pipe.
+You can find Jira issues for this module under the `ap_pipe <https://jira.lsstcorp.org/issues/?jql=project%20%3D%20DM%20AND%20component%20%3D%20ap_pipe>`_ component.
+
+.. If there are topics related to developing this module (rather than using it), link to this from a toctree placed here.
 
+.. _lsst.ap.pipe-pyapi:
 
 Python API reference
 ====================
 
 .. automodapi:: lsst.ap.pipe
+   :no-main-docstr:
+   :no-inheritance-diagram:

doc/lsst.ap.pipe/pipeline-overview.rst

@@ -1,16 +1,18 @@
+.. py:currentmodule:: lsst.ap.pipe
+
 .. _pipeline-overview:
 
 ###########################
-Overview of the AP Pipeline
+Overview of the AP pipeline
 ###########################
 
 :doc:`lsst.ap.pipe <index>` is a data processing pipeline for Prompt Data Products.
 It is a Command-Line Task which operates on ingested raw data in a Butler repository.
 It also requires appropriate calibration products and templates. As it runs,
-`lsst.ap.pipe.ApPipeTask` generates calibrated exposures, difference images,
+`ApPipeTask` generates calibrated exposures, difference images,
 difference image source catalogs, and a source association database.
 
-The initial motivation for `lsst.ap.pipe`, information about one of the original test datasets,
+The initial motivation for :doc:`lsst.ap.pipe <index>`, information about one of the original test datasets,
 and an outdated tutorial are available in `DMTN-039 <https://dmtn-039.lsst.io>`_.
 
 The AP Pipeline calls three main tasks and their associated subtasks:
@@ -19,18 +21,18 @@ The AP Pipeline calls three main tasks and their associated subtasks:
    `lsst.pipe.tasks.CharacterizeImageTask`, and `lsst.pipe.tasks.CalibrateTask`
    to perform image reduction as well as photometric and astrometric calibration;
 2. `~lsst.pipe.tasks.ImageDifferenceTask`, which uses many utilities from
-   `lsst.ip.diffim`; and
+   :doc:`lsst.ip.diffim </modules/lsst.ip.diffim/index>`; and
 3. `~lsst.ap.associate.AssociationTask`, which makes a catalog of
    Difference Image Analysis (DIA) Objects from the DIASources created
    during image differencing.
 
-In practice, `lsst.ap.pipe` is often discussed in the context of `lsst.ap.verify`.
-The former is responsible for running the AP Pipeline. The latter uses `lsst.ap.pipe`
+In practice, :doc:`lsst.ap.pipe <index>` is often discussed in the context of :doc:`lsst.ap.verify </modules/lsst.ap.verify/index>`.
+The former is responsible for running the AP Pipeline. The latter uses :doc:`lsst.ap.pipe <index>`
 to verify the output.
 
-`ap_pipe` is entirely written in Python. Key contents include:
+:doc:`ap_pipe <index>` is entirely written in Python. Key contents include:
 
-- `~lsst.ap.pipe.ApPipeTask`: a `~lsst.pipe.base.CmdLineTask` for running the entire AP Pipeline
-- `~lsst.ap.pipe.ApPipeConfig`: a config for customizing ``ApPipeTask`` for a particular dataset's needs.
+- `ApPipeTask`: a `~lsst.pipe.base.CmdLineTask` for running the entire AP Pipeline
+- `ApPipeConfig`: a config for customizing ``ApPipeTask`` for a particular dataset's needs.
   Supported observatory packages should provide a :ref:`config override file <command-line-task-config-howto-obs>` that does most of the work.
 

doc/lsst.ap.pipe/pipeline-tutorial.rst

@@ -1,8 +1,10 @@
+.. py:currentmodule:: lsst.ap.pipe
+
 .. _pipeline-tutorial:
 
-####################
-AP Pipeline Tutorial
-####################
+#######################
+Running the AP pipeline
+#######################
 
 Setup
 =====
@@ -54,11 +56,10 @@ Your directory structure should look something like
 
 .. _section-ap-pipe-command-line:
 
-AP Pipeline on the command line
+AP pipeline on the command line
 ===============================
 
-The executable to run for the AP Pipeline (`lsst.ap.pipe.ApPipeTask`) is
-in ``ap_pipe/bin/ap_pipe.py``.
+The executable to run for the AP Pipeline (`ApPipeTask`) is in ``ap_pipe/bin/ap_pipe.py``.
 
 To process your ingested data, run
 
@@ -73,7 +74,7 @@ This example command only processes observations that have a
 :ref:`dataId<subsection-ap-pipe-previewing-dataIds>`
 corresponding to visit 123456 and ccdnum 42 in with a filter called g.
 
-`lsst.ap.pipe` supports ``dataId`` parsing, e.g., ``ccdnum=3^6..12`` will process
+:doc:`lsst.ap.pipe <index>` supports ``dataId`` parsing, e.g., ``ccdnum=3^6..12`` will process
 ``ccdnums`` 3, 6, 7, 8, 9, 10, 11, and 12.
 
 .. note::
@@ -94,7 +95,7 @@ somewhere in ``repo``.
 
 .. note::
 
-   If you are using the default (SQLite) association database, you must :ref:`configure <command-line-task-config-howto>` the database location, or ``ap_pipe`` will not run.
+   If you are using the default (SQLite) association database, you must :doc:`configure </modules/lsst.pipe.base/command-line-task-config-howto>` the database location, or ``ap_pipe`` will not run.
    The location is a path to a new or existing database file to be used for source associations (including associations with previously known objects, if the database already exists).
    In the examples above, it is configured with the ``-c`` option, but a personal config file may be more convenient if you intend to run ``ap_pipe`` many times.
 
@@ -211,7 +212,7 @@ Interpreting the results
 .. warning:: 
 
    The format of the ``ap_association`` Prompt Product Database is rapidly evolving. For
-   the latest information on how to interface with it, see `lsst.ap.associate`.
+   the latest information on how to interface with it, see :doc:`lsst.ap.association </modules/lsst.ap.association/index>`.
 
 Try these python commands to make some initial plots of your
 newly processed data. You can also use the Butler to display

doc/manifest.yaml

@@ -1,15 +1,12 @@
 # Documentation manifest.
 
-# Name of this package
-# Also the name of the package documentation subdirectory.
-package: "ap_pipe"
-
 # List of names of Python modules in this package.
 # For each module there is a corresponding module doc subdirectory.
 modules:
   - "lsst.ap.pipe"
 
 # Name of the static content directories (subdirectories of `_static`).
 # Static content directories are usually named after the package.
-statics:
-  - "_static/ap_pipe"
+# Most packages do not need a static content directory (leave commented out).
+# statics:
+#   - "_static/ap_pipe"

python/lsst/ap/pipe/ap_pipe.py

@@ -82,9 +82,6 @@ class ApPipeTask(pipeBase.CmdLineTask):
     ``ApPipeTask`` processes raw DECam images from basic processing through
     source association. Other observatories will be supported in the future.
 
-    A tutorial for using ``ApPipeTask`` is available in
-    [DMTN-039](http://dmtn-039.lsst.io).
-
     ``ApPipeTask`` can be run from the command line, but it can also be called
     from other pipeline code. It provides public methods for executing each
     major step of the pipeline by itself.
@@ -212,7 +209,7 @@ def runDiffIm(self, sensorRef, templateIds=None):
 
         Parameters
         ----------
-        sensorRef: `lsst.daf.persistence.ButlerDataRef`
+        sensorRef : `lsst.daf.persistence.ButlerDataRef`
             Data reference for multiple dataset types, both input and output.
         templateIds : `list` of `dict`, optional
             A list of parsed data IDs for templates to use. Only used if
@@ -277,7 +274,7 @@ def _setupDatabase(configurable):
 
     Parameters
     ----------
-    configurable: `lsst.pex.config.ConfigurableInstance`
+    configurable : `lsst.pex.config.ConfigurableInstance`
         A ConfigurableInstance with a database-managing class in its ``target``
         field. The API of ``target`` must expose a ``create_tables`` method
         taking no arguments.