Update docs build and packaging setup

.github/workflows/sphinx.yml

@@ -0,0 +1,56 @@
+name: "Sphinx: Build"
+
+on: push
+
+jobs:
+  build:
+    name: (${{ matrix.os }}, Miniforge)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest"]
+    permissions:
+      contents: write
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: pipeline
+          environment-file: environment.yml
+          channel-priority: strict
+          use-only-tar-bz2: true
+          show-channel-urls: true
+          miniforge-version: latest
+      - run: conda info
+      - run: conda list
+      - run: conda config --show
+      - run: conda env list
+      - name: Pre-Build Setup (os misc. latex, git)
+        run: sudo apt-get update -y && sudo apt-get install -y git xvfb imagemagick poppler-utils latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
+      - name: Pre-build Setup (python/sphinx)
+        run: python -m pip install .[docs] && mkdir -p ~/.casa/data
+      - name: Build HTML
+        working-directory: ./docs
+        run: make html_docs
+      - name: Build PDF
+        working-directory: ./docs
+        run: make latexpdf
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-artifact
+          path: |
+            docs/_build/html/
+            docs/_build/latex/taskdocs.pdf
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/update-docs-build-and-packaging-setup'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/_build

.gitignore

@@ -72,14 +72,17 @@ instance/
 
 # Sphinx documentation
 docs/_build/**
-docs/source/_apidoc/**
 docs/source/_autosummary/**
-docs/source/_taskdocs/**
+docs/source/_automodapi/**
 
 # CASA log files
-casa-*.log
+**/casa-*.log
 docs/casa-*.log
 docs/source/casa-*.log
+*.last
+
+# Slurm
+**/*.out
 
 # PyBuilder
 target/
@@ -164,3 +167,31 @@ tags
 
 # ignore _version.py, created by setup.py
 _version.py
+docs/source/references/pipeline-resolved.bib
+
+**/.DS_Store
+
+# Ruff cache
+.ruff_cache
+
+# data etc.
+**/*.zarr
+**/*.ms
+**/*.tar
+**/*.zip
+**/*.fits
+**/*.im
+**/*.image
+**/*.model
+**/*.residual
+**/*.mask
+**/*.pbcor
+**/*.sumwt
+**/*.flux
+**/*.weight
+**/*.sigma
+**/*.cube
+**/*.moment
+**/*.moment0
+**/*.moment1
+**/*.moment2

.mailmap

@@ -1,3 +1,4 @@
+# git shortlog -se
 Ajay Vibhute <avibhute@nrao.edu> Ajay Vibhute <avibhute@cvpost125.cv.nrao.edu>
 Akira Yoshino <yoshino.akira@nao.ac.jp>
 Brian Kent <bkent@nrao.edu> brian.kent <brian.kent@ac9c0da5-131f-0410-9009-a9c195f183e6>
@@ -15,8 +16,11 @@ Laszlo Szucs <lszucs@nrao.edu> László Szűcs <laszlo.szucs@mpe.mpg.de>
 Laszlo Szucs <lszucs@nrao.edu> László Szűcs <laszlo.szucs@nrao.edu>
 Luke Maud <lmaud@eso.org> lmaud <lmaud@eso.org>
 Remy Indebetouw <rindebet@nrao.edu> <remy@virginia.edu>
+Remy Indebetouw <rindebet@nrao.edu> <indebetouw@gmail.com>
 Rui Xue <rxue@nrao.edu> <rx.astro@gmail.com>
+Rui Xue <rxue@nrao.edu> <r-xue@users.noreply.github.com>
 Shawn Booth <sbooth@nrao.edu>
+Shawn Booth <sbooth@nrao.edu> <stb5585@gmail.com>
 Stewart Williams <stewart.williams@stfc.ac.uk> Stewart Williams <16945415+bravostuzero@users.noreply.github.com>
 Stewart Williams <stewart.williams@stfc.ac.uk> Stewart Williams <swilliam@cvpost002.cv.nrao.edu>
 Stewart Williams <stewart.williams@stfc.ac.uk> Stewart Williams <swilliam@cvpost008.cv.nrao.edu>

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.10
+    hooks:
+      - id: ruff # Run the linter.
+        types_or: [python, pyi]
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format # Run the formatter.
+        types_or: [python, pyi]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: no-commit-to-branch
+        args: ["--pattern", "main|^release/.*"]

.readthedocs.yaml

@@ -0,0 +1,26 @@
+version: 2
+
+# Build tools and environment setup
+build:
+  os: ubuntu-24.04
+  tools:
+    python: miniconda-latest # use miniconda-latest before miniforge3 becomes available.
+  apt_packages:
+    - git
+    - xvfb
+    - imagemagick
+    - poppler-utils
+    - latexmk
+    - texlive-latex-recommended
+    - texlive-latex-extra
+    - texlive-fonts-recommended
+  # Build commands for HTML and PDF
+  commands:
+    - conda env update --file=environment.yml
+    - conda run -n pipeline python -m pip install .[docs]
+    - mkdir -p ~/.casa/data
+    - cd docs && conda run -n pipeline make html_docs
+    - cd docs && conda run -n pipeline make latexpdf
+    - mkdir --parents _readthedocs/html/ _readthedocs/pdf/
+    - cp --recursive docs/_build/html/* _readthedocs/html/
+    - cp --recursive docs/_build/latex/*.pdf _readthedocs/pdf/

.ruff.toml

@@ -0,0 +1,22 @@
+line-length = 120
+indent-width = 4
+
+[lint]
+# Enable pydocstyle (`D`) rules, in addition to the defaults.
+select = ["E4", "E7", "E9", "F", "D"]
+preview = true
+
+[lint.pycodestyle]
+max-doc-length = 140
+max-line-length = 120
+
+[lint.pydocstyle]
+convention = "google"
+
+[format]
+preview = true
+exclude = ["*.pyi"]
+docstring-code-format = true
+docstring-code-line-length = 140
+quote-style = "single"
+skip-magic-trailing-comma = false

conftest.py

@@ -13,7 +13,7 @@ def pytest_addoption(parser):
 
     nologfile_help = r"""
         Do not create CASA log files, equivalent to 'casa --nologfile'.
-        Please note that if you're using regression-tester.py, casa logfiles 
+        Please note that if you're using regression_tester.py, casa logfiles 
         will still be generated within individual test "working/" directories and 
         appear in test weblogs. In general, this option is only recommended when 
         manually/frequently running unit tests, to keep your local repo clean.

docs/Makefile

@@ -4,7 +4,7 @@
 #	export pythonbin=${casa_dir}/bin/python3
 PYTHON := $(if $(pythonbin),$(pythonbin),python3)
 
-SPHINXOPTS    = -v -j auto
+SPHINXOPTS    = -v # -j auto
 SPHINXBUILD   = ${PYTHON} -msphinx
 #SPHINXOPTS    = -v -j auto -b coverage
 SPHINXPROJ    = pipeline_docs
@@ -40,19 +40,17 @@ taskapidoc:
 	@sphinx-apidoc -o ./source/_apidoc/ ../pipeline ../pipeline/recipes/*.py ../pipeline/run*pipeline.py ../pipeline/domain/*.py ../pipeline/extern/*.py ../pipeline/infrastructure/*.py ../pipeline/qa/*.py ../pipeline/h*/heuristics/*.py ../pipeline/h*/tasks/*.py ../pipeline/h*/templates/*.py ../pipeline/h*/environment/*.py
 
 
-taskdocs:
-	@rm -rf ${SOURCEDIR}/_taskdocs
-	@${PYTHON} ./source/pipeline_tasks/create_docs.py --outdir ${SOURCEDIR}/_taskdocs --srcdir ../
-
 html_docs:
-	@make taskdocs
-	@make apidoc
-	@rm -rf ${BUILDDIR}/html ${BUILDDIR}/doctrees ${SOURCEDIR}/_autosummary
+	@rm -rf ${SOURCEDIR}/_apidoc
+	# @make apidoc
+	@rm -rf ${BUILDDIR}/html ${BUILDDIR}/doctrees 
+	@rm -rf ${SOURCEDIR}/_autosummary*
+	@rm -rf ${SOURCEDIR}/_automodapi
 	@make html
 	@rm -rf casa-*.log ${SOURCEDIR}/casa-*log
 
 pdf_taskdocs:
-	@make taskdocs
 	@rm -rf ${BUILDDIR}/latex ${BUILDDIR}/doctrees
+	@rm -rf ${SOURCEDIR}/_autosummary*
 	@make latexpdf
 	@rm -rf casa-*.log ${SOURCEDIR}/casa-*log

docs/README.rst

@@ -0,0 +1,39 @@
+Pipeline
+========
+*The operational data processing pipelines for ALMA, VLA, and the NRO 45m telescope.*
+
+|Sphinx Build|
+
+.. important::
+
+  This is a **demonstration** of the new internal documentation framework for the Pipeline development.
+  It is designed to provide *up-to-date and version-controlled* supplementary technical information to complement the official Pipeline documentatios from the `ALMA <https://almascience.nrao.edu/processing/science-pipeline>`_ and `VLA <https://science.nrao.edu/facilities/vla/data-processing>`_ science portals.
+  Pipeline development is a collaborative effort led by `NRAO`_, `ESO`_ (in partnership with `UKATC`_ and `MPIfR`_), and `NAOJ`_.
+
+
+Official Repository
+--------------------
+
+The official public repository is accessible here:  
+- `Open Bitbucket @ NRAO - PIPE <https://open-bitbucket.nrao.edu/projects/PIPE>`_
+
+Highlights
+----------
+
+`This demo <pipe-docs.readthedocs.io>`_ is automatically generated from `a development branch <https://open-bitbucket.nrao.edu/projects/PIPE/repos/pipeline/browse?at=refs%2Fheads%2FPIPE-1669-run-dev-pipeline-with-modular-casa6>`_ of the official repository and remains under active restructuring
+Some sections are highlighted below:
+
+  * `Past Pipeline Releases <https://pipe-docs.readthedocs.io/en/latest/releases.html#>`_
+  * `Pipeline Dependencies <https://pipe-docs.readthedocs.io/en/latest/dependencies.html>`_
+  * `Running a Pipeline Session in Python <https://pipe-docs.readthedocs.io/en/latest/modular.html>`_
+  * The latest Pipeline task reference manual: `PDF <https://pipe-docs.readthedocs.io/_/downloads/en/latest/pdf/>`_ `HTML <https://pipe-docs.readthedocs.io/en/latest/apisummary.html#pipeline-tasks-autosummary>`_
+
+.. _NRAO: http://www.nrao.edu  
+.. _ESO: https://www.eso.org  
+.. _UKATC: https://www.ukatc.stfc.ac.uk  
+.. _MPIfR: https://www.mpifr-bonn.mpg.de  
+.. _NAOJ: https://www.nao.ac.jp  
+
+.. |Sphinx Build| image:: https://github.com/r-xue/pipeline/actions/workflows/sphinx.yml/badge.svg  
+ :target: https://github.com/r-xue/pipeline/actions/workflows/sphinx.yml  
+ :alt: Sphinx: Build