DM-34105: Clean up dependencies

.github/workflows/build.yaml

@@ -0,0 +1,58 @@
+name: build_and_test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: 3.8
+          channels: conda-forge,defaults
+          channel-priority: strict
+          show-channel-urls: true
+
+      - name: Update pip/wheel infrastructure
+        shell: bash -l {0}
+        run: |
+          conda install -y -q pip wheel
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        run: |
+          pip install -r requirements.txt
+
+      # We have two cores so we can speed up the testing with xdist
+      - name: Install pytest packages
+        shell: bash -l {0}
+        run: |
+          conda install -y -q \
+            flake8 \
+            pytest pytest-flake8 pytest-xdist pytest-openfiles pytest-cov
+
+      - name: List installed packages
+        shell: bash -l {0}
+        run: |
+          conda list
+          pip list -v
+
+      - name: Build and install
+        shell: bash -l {0}
+        run: |
+          pip install -v -e .
+
+      - name: Run tests
+        shell: bash -l {0}
+        run: |
+          pytest -r a -v -n 3 --open-files --cov=lsst.ctrl.mpexec --cov=tests --cov-report=xml --cov-report=term --cov-branch
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v2
+        with:
+          file: ./coverage.xml

doc/changes/DM-34105.feature.rst

@@ -0,0 +1 @@
+Remove dependency on the ``obs_base`` and ``afw`` packages. Now only depends on ``pipe_base`` and ``daf_butler`` (along with ``pex_config`` and ``utils``).

python/lsst/__init__.py

@@ -21,6 +21,9 @@
 
 import pkgutil
 
-import lsstimport
+try:
+    import lsstimport
+except ImportError:
+    pass
 
 __path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/ctrl/__init__.py

@@ -21,6 +21,9 @@
 
 import pkgutil
 
-import lsstimport
+try:
+    import lsstimport
+except ImportError:
+    pass
 
 __path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -20,7 +20,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import click
-import lsst.obs.base.cli.opt as obsBaseOpts
+import lsst.pipe.base.cli.opt as pipeBaseOpts
 from lsst.daf.butler.cli.opt import config_file_option, config_option, options_file_option
 from lsst.daf.butler.cli.utils import MWCtxObj, catch_and_exit, option_section, unwrap
 
@@ -64,7 +64,7 @@ def _collectActions(ctx, **kwargs):
         ctrlMpExecOpts.delete_option.name(),
         config_option.name(),
         config_file_option.name(),
-        obsBaseOpts.instrument_option.name(),
+        pipeBaseOpts.instrument_option.name(),
     ):
         kwargs.pop(pipelineAction)
 

python/lsst/ctrl/mpexec/cli/opt/optionGroups.py

@@ -32,7 +32,7 @@
 
 import click
 import lsst.daf.butler.cli.opt as dafButlerOpts
-import lsst.obs.base.cli.opt as obsBaseOpts
+import lsst.pipe.base.cli.opt as pipeBaseOpts
 from lsst.daf.butler.cli.utils import OptionGroup, option_section, unwrap
 
 from . import options as ctrlMpExecOpts
@@ -65,7 +65,7 @@ def __init__(self):
             ctrlMpExecOpts.order_pipeline_option(),
             ctrlMpExecOpts.save_pipeline_option(),
             ctrlMpExecOpts.pipeline_dot_option(),
-            obsBaseOpts.instrument_option(help=instrumentOptionHelp, metavar="instrument", multiple=True),
+            pipeBaseOpts.instrument_option(help=instrumentOptionHelp, metavar="instrument", multiple=True),
         ]
 
 

python/lsst/ctrl/mpexec/cli/utils.py

@@ -25,7 +25,7 @@
 
 from lsst.daf.butler.cli.opt import config_file_option, config_option
 from lsst.daf.butler.cli.utils import MWCommand
-from lsst.obs.base.cli.opt import instrument_option
+from lsst.pipe.base.cli.opt import instrument_option
 
 from .opt import delete_option, task_option
 

python/lsst/ctrl/mpexec/cmdLineFwk.py

@@ -39,15 +39,16 @@
 from typing import Iterable, Optional, Tuple
 
 import lsst.pex.config as pexConfig
+import lsst.pex.config.history as pexConfigHistory
 
 # -----------------------------
 #  Imports for other modules --
 # -----------------------------
 from lsst.daf.butler import Butler, CollectionSearch, CollectionType, Registry
 from lsst.daf.butler.registry import MissingCollectionError, RegistryDefaults
-from lsst.obs.base import Instrument
 from lsst.pipe.base import (
     GraphBuilder,
+    Instrument,
     Pipeline,
     PipelineDatasetTypes,
     QuantumGraph,
@@ -882,7 +883,7 @@ def _showConfigHistory(self, pipeline, showArgs):
                     hconfig, cname
                 ):
                     print(f"### Configuration field for task `{taskDef.label}'")
-                    print(pexConfig.history.format(hconfig, cname))
+                    print(pexConfigHistory.format(hconfig, cname))
                     found = True
 
         if not found:

python/lsst/ctrl/mpexec/examples/calexpToCoaddTask.py

@@ -3,8 +3,6 @@
 
 import logging
 
-import lsstDebug
-from lsst.afw.image import ExposureF
 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
 from lsst.pipe.base import connectionTypes as cT
 
@@ -16,13 +14,13 @@ class CalexpToCoaddTaskConnections(PipelineTaskConnections, dimensions=("skymap"
         name="calexp",
         dimensions=["instrument", "visit", "detector"],
         multiple=True,
-        storageClass="ExposureF",
+        storageClass="StructuredDataList",
         doc="DatasetType for the input image",
     )
     coadd = cT.Output(
         name="deepCoadd_calexp",
         dimensions=["skymap", "tract", "patch", "band"],
-        storageClass="ExposureF",
+        storageClass="StructuredDataList",
         doc="DatasetType for the output image",
     )
 
@@ -52,12 +50,11 @@ def run(self, calexp):
         #     import lsstDebug
         #     lsstDebug.Info('lsst.ctrl.mpexec.examples.calexpToCoaddTask').display = True  # noqa: W505
         #
-        if lsstDebug.Info(__name__).display:
-            _LOG.info("%s: display enabled", __name__)
-
-        # output data, scalar in this case
-        data = ExposureF(100, 100)
+        # if lsstDebug.Info(__name__).display:
+        #     _LOG.info("%s: display enabled", __name__)
 
+        # output data, scalar in this case, simple list for example.
+        data = list(range(100))
         # attribute name of struct is the same as a config field name
         return Struct(coadd=data)
 

python/lsst/ctrl/mpexec/examples/rawToCalexpTask.py

@@ -3,7 +3,6 @@
 
 import logging
 
-from lsst.afw.image import ExposureF
 from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections, Struct
 from lsst.pipe.base import connectionTypes as cT
 
@@ -21,7 +20,9 @@ class RawToCalexpTaskConnections(PipelineTaskConnections, dimensions=("instrumen
     output = cT.Output(
         name="calexp",
         dimensions=["instrument", "visit", "detector"],
-        storageClass="ExposureF",
+        # Ordinarily this would be an ExposureF for a calexp
+        # but to minimize dependencies in this example use a simple list.
+        storageClass="StructuredDataList",
         doc="Output dataset type for this task",
     )
 
@@ -54,8 +55,8 @@ def run(self, input):
 
         _LOG.info("executing %s: input=%s", self.getName(), input)
 
-        # result, scalar in this case, just create 100x100 image
-        data = ExposureF(100, 100)
+        # result, scalar in this case, just create simple list.
+        data = list(range(100))
 
         # attribute name of struct is the same as a config field name
         return Struct(output=data)

python/lsst/ctrl/mpexec/simple_pipeline_executor.py

@@ -26,9 +26,8 @@
 from typing import Any, Iterable, Iterator, List, Optional, Type, Union
 
 from lsst.daf.butler import Butler, CollectionType, Quantum
-from lsst.obs.base import Instrument
 from lsst.pex.config import Config
-from lsst.pipe.base import GraphBuilder, Pipeline, PipelineTask, QuantumGraph, TaskDef
+from lsst.pipe.base import GraphBuilder, Instrument, Pipeline, PipelineTask, QuantumGraph, TaskDef
 
 from .preExecInit import PreExecInit
 from .singleQuantumExecutor import SingleQuantumExecutor

python/lsst/ctrl/mpexec/singleQuantumExecutor.py

@@ -38,10 +38,10 @@
 
 from lsst.daf.butler import DatasetRef, DatasetType, FileDataset, NamedKeyDict, Quantum
 from lsst.daf.butler.core.logging import ButlerLogRecordHandler, ButlerLogRecords, ButlerMDC, JsonLogFormatter
-from lsst.obs.base import Instrument
 from lsst.pipe.base import (
     AdjustQuantumHelper,
     ButlerQuantumContext,
+    Instrument,
     InvalidQuantumError,
     NoWorkFound,
     RepeatableQuantumError,

requirements.txt

@@ -0,0 +1,9 @@
+click
+astropy
+psutil
+pydantic
+networkx
+git+https://github.com/lsst/daf_butler@main#egg=daf_butler
+git+https://github.com/lsst/utils@main#egg=lsst_utils
+git+https://github.com/lsst/pipe_base@main#egg=pipe_base
+git+https://github.com/lsst/pex_config@main#egg=pex_config

setup.cfg

@@ -1,7 +1,46 @@
+[metadata]
+name = ctrl_mpexec
+description = Pipeline execution infrastructure for the Rubin Science Pipelines
+author = Rubin Observatory Data Management
+url = https://github.com/lsst/resources
+classifiers =
+    Intended Audience :: Science/Research
+    License :: OSI Approved ::  GNU General Public License v3 or later (GPLv3+)
+    Operating System :: OS Independent
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.8
+    Topic :: Scientific/Engineering :: Astronomy
+
+[options]
+zip_safe = True
+package_dir=
+    =python
+packages=find:
+setup_requires =
+  setuptools >=46.0
+install_requires =
+  lsst_utils @ git+https://github.com/lsst/utils@main
+  daf_butler @ git+https://github.com/lsst/daf_butler@main
+  pex_config @ git+https://github.com/lsst/pex_config@main
+  pipe_base @ git+https://github.com/lsst/pipe_base@main
+  click
+  astropy
+  pydantic
+  networkx
+  psutil
+tests_require =
+  pytest >= 3.2
+  flake8 >= 3.7.5
+  pytest-flake8 >= 1.0.4
+  pytest-openfiles >= 0.5.0
+
+[options.packages.find]
+where=python
+
 [flake8]
 max-line-length = 110
 max-doc-length = 79
-ignore = E133, E226, E228, N802, N803, N806, N812, N815, N816, W503, E203
+ignore = E203, W503, N802, N803, N806, N812, N815, N816
 exclude =
   bin,
   doc,
@@ -11,4 +50,4 @@ exclude =
 
 [tool:pytest]
 addopts = --flake8
-flake8-ignore = E133 E226 E228 N802 N803 N806 N812 N815 N816 W503 E203
+flake8-ignore = E203 W503 N802 N803 N806 N812 N815 N816

setup.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+# This file is part of ctrl_mpexec.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+
+import os.path
+
+from setuptools import setup
+
+version = "0.1.0"
+with open("./python/lsst/ctrl/mpexec/version.py", "w") as f:
+    print(
+        f"""
+__all__ = ("__version__", )
+__version__ = '{version}'""",
+        file=f,
+    )
+
+# read the contents of our README file
+this_directory = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(this_directory, "README.rst"), encoding="utf-8") as f:
+    long_description = f.read()
+
+setup(version=version, long_description=long_description, long_description_content_type="text/x-rst")

tests/test_cmdLineFwk.py

@@ -51,8 +51,14 @@
 from lsst.daf.butler import Config, DataCoordinate, DatasetRef, DimensionUniverse, Quantum, Registry
 from lsst.daf.butler.core.datasets.type import DatasetType
 from lsst.daf.butler.registry import RegistryConfig
-from lsst.obs.base import Instrument
-from lsst.pipe.base import Pipeline, PipelineTaskConfig, PipelineTaskConnections, QuantumGraph, TaskDef
+from lsst.pipe.base import (
+    Instrument,
+    Pipeline,
+    PipelineTaskConfig,
+    PipelineTaskConnections,
+    QuantumGraph,
+    TaskDef,
+)
 from lsst.pipe.base.graphBuilder import DatasetQueryConstraintVariant as DQCVariant
 from lsst.pipe.base.tests.simpleQGraph import (
     AddTask,