Merge branch 'master' into fix/refactor-lineage-class

theislab · Apr 21, 2022 · 84fc4ea · 84fc4ea
2 parents 0bd9e42 + ac6efec
commit 84fc4ea
Show file tree

Hide file tree

Showing 11 changed files with 168 additions and 109 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -53,6 +53,10 @@ jobs:
                 -   os: macos-latest
                     python: 3.8
                     use_slepc: false
+        env:
+            # make sure to pin this in tox.ini as well
+            PC_VERSION: 3.15.4  # PETSc version
+            SC_VERSION: 3.15.2  # SLEPc version
 
         steps:
         -   uses: actions/checkout@v2
@@ -61,12 +65,23 @@ jobs:
             with:
                 python-version: ${{ matrix.python }}
 
-        -   name: Install dependencies
-            env:
-                USE_SLEPC: ${{ matrix.use_slepc }}
+        -   name: Install Linux dependencies
+            if: runner.os == 'Linux'
             run: |
                 ./.scripts/ci/install_dependencies.sh
 
+        -   name: Export PETSc/SLEPc environment variables
+            if: ${{ runner.os == 'Linux' && matrix.use_slepc == true }}
+            run: |
+                echo "PETSC_DIR=$HOME/petsc-$PC_VERSION" >> $GITHUB_ENV
+                echo "SLEPC_DIR=$HOME/slepc-$SC_VERSION" >> $GITHUB_ENV
+                echo "PETSC_ARCH=arch-$RUNNER_OS-c-opt" >> $GITHUB_ENV
+
+        -   name: Install PETSc/SLEPc dependencies
+            if: ${{ runner.os == 'Linux' && matrix.use_slepc == true }}
+            run: |
+                ./.scripts/ci/install_petsc_linux.sh
+
         -   name: Get pip cache dir
             id: pip-cache
             run: |

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,7 +7,7 @@ default_stages:
 minimum_pre_commit_version: 2.9.3
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
     -   id: black
         additional_dependencies: [toml]
@@ -28,7 +28,7 @@ repos:
     -   id: pretty-format-yaml
         args: [--autofix, --indent, '4']
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.2.0
     hooks:
     -   id: detect-private-key
     -   id: check-merge-conflict
@@ -87,6 +87,6 @@ repos:
     -   id: rst-directive-colons
     -   id: rst-inline-touching-normal
 -   repo: https://github.com/PyCQA/doc8
-    rev: 0.10.1
+    rev: 0.11.1
     hooks:
     -   id: doc8
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,20 +1,23 @@
 version: 2
 
+build:
+    os: ubuntu-20.04
+    tools:
+        python: '3.9'
+    apt_packages:
+    -   pandoc
+
 sphinx:
     builder: html
     configuration: docs/source/conf.py
     fail_on_warning: false
 
-formats:
--   htmlzip
-
-build:
-    image: latest
-
 python:
-    version: 3.8
     install:
     -   method: pip
         path: .
         extra_requirements:
         -   docs
+
+formats:
+-   htmlzip
diff --git a/.scripts/ci/install_dependencies.sh b/.scripts/ci/install_dependencies.sh
@@ -1,23 +1,15 @@
 #!/usr/bin/env bash
 
-set -ev
+set -euo pipefail
 
-if [[ "$RUNNER_OS" == "Linux" ]]; then
-    echo "Installing APT dependencies"
-    # https://github.com/yarnpkg/yarn/issues/7866
-    curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-    # R-related
-    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
-    sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
+echo "Fetching Yarn GPG key"
+# https://github.com/yarnpkg/yarn/issues/7866
+curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
 
-    sudo apt update -y
-    sudo apt install libopenblas-base r-base r-base-dev r-cran-mgcv -y
+echo "Installing R"
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
+sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
 
-    if [[ "$USE_SLEPC" == "true" ]]; then
-        echo "Installing PETSc/SLEPc dependencies"
-        sudo apt install gcc gfortran libopenmpi-dev libblas-dev liblapack-dev petsc-dev slepc-dev -y
-    fi
-elif [[ "$RUNNER_OS" != "macOS" ]]; then
-    echo "Invalid OS for dependencie installations: $OS"
-    exit 42
-fi
+echo "Installing APT dependencies"
+sudo apt update -y
+sudo apt install libopenblas-base r-base r-base-dev r-cran-mgcv -y
diff --git a/.scripts/ci/install_petsc_linux.sh b/.scripts/ci/install_petsc_linux.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function install_petsc {
+    echo "Installing PETSc; version: '$PC_VERSION', dir: '$PETSC_DIR' with arch: '$PETSC_ARCH'"
+    curl -O "https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-lite-$PC_VERSION.tar.gz"
+    tar -xzf "petsc-lite-$PC_VERSION.tar.gz"
+    pushd "petsc-$PC_VERSION"
+    ./configure --with-cc=mpicc --with-fc=0 --with-cxx=mpicxx --with-debugging=0 --with-mpi=1
+    make all
+    make check
+    popd
+}
+
+function install_slepc {
+    echo "Installing SLEPc; version: '$SC_VERSION', dir: '$SLEPC_DIR'"
+    curl -O "https://slepc.upv.es/download/distrib/slepc-$SC_VERSION.tar.gz"
+    tar -xzf "slepc-$SC_VERSION.tar.gz"
+    pushd "slepc-$SC_VERSION"
+    ./configure
+    make all
+    make check
+    popd
+}
+
+echo "Installing PETSc/SLEPc dependencies"
+sudo apt-get update -y
+sudo apt-get install gcc gfortran libopenmpi-dev libblas-dev liblapack-dev -y
+
+pushd "$HOME"
+install_petsc
+install_slepc
+popd
diff --git a/cellrank/tl/kernels/_transport_map_kernel.py b/cellrank/tl/kernels/_transport_map_kernel.py
@@ -62,6 +62,76 @@ def _compute_tmap(
         and :attr:`anndata.AnnData.var_names` correspond to subsets of observations from :attr:`adata`.
         """
 
+    @d.get_sections(base="tmk_thresh", sections=["Parameters"])
+    def _threshold_transport_maps(
+        self,
+        tmaps: Dict[Pair_t, AnnData],
+        threshold: Threshold_t,
+        copy: bool = False,
+    ) -> Optional[Mapping[Pair_t, AnnData]]:
+        """
+        Remove small non-zero values from :attr:`transition_matrix`.
+
+        Parameters
+        ----------
+        threshold
+            How to remove small non-zero values from the transition matrix. Valid options are:
+
+                - `'auto'` - find the maximum threshold value which will not remove every non-zero value from any row.
+                - `'auto_local'` - same as above, but done for each transport separately.
+                - :class:`float` - value in `[0, 100]` corresponding to a percentage of non-zeros to remove in each
+                  transport map.
+
+            Rows where all values are removed will have uniform distribution and a warning will be issued.
+        copy
+            Whether to return a copy of thresholded ``tmaps`` or modify inplace.
+
+        Returns
+        -------
+        If ``copy = True``, returns a thresholded transport maps. Otherwise, modifies ``tmaps`` inplace.
+        """
+        if threshold == "auto":
+            thresh = min(
+                adata.X[i].max() for adata in tmaps.values() for i in range(adata.n_obs)
+            )
+            logg.info(f"Using automatic `threshold={thresh}`")
+        elif threshold == "auto_local":
+            logg.info("Using automatic `threshold` for each time point pair separately")
+        elif not (0 <= threshold <= 100):
+            raise ValueError(
+                f"Expected `threshold` to be in `[0, 100]`, found `{threshold}`.`"
+            )
+
+        for key, adata in tmaps.items():
+            if copy:
+                adata = adata.copy()
+            tmat = adata.X
+
+            if threshold == "auto_local":
+                thresh = min(tmat[i].max() for i in range(tmat.shape[0]))
+                logg.debug(f"Using `threshold={thresh}` at `{key}`")
+            elif isinstance(threshold, (int, float)):
+                thresh = np.percentile(tmat.data, threshold)
+                logg.debug(f"Using `threshold={thresh}` at `{key}`")
+
+            tmat = csr_matrix(tmat, dtype=tmat.dtype)
+            tmat.data[tmat.data < thresh] = 0.0
+            zeros_mask = np.where(np.asarray(tmat.sum(1)).squeeze() == 0)[0]
+            if np.any(zeros_mask):
+                logg.warning(
+                    f"After thresholding, `{len(zeros_mask)}` row(s) of transport at `{key}` are forced to be uniform"
+                )
+                for ix in zeros_mask:
+                    start, end = tmat.indptr[ix], tmat.indptr[ix + 1]
+                    size = end - start
+                    tmat.data[start:end] = np.ones(size, dtype=tmat.dtype) / size
+
+            # after `zeros_mask` has been handled, to have access to removed row indices
+            tmat.eliminate_zeros()
+            tmaps[key] = AnnData(tmat, obs=adata.obs, var=adata.var, dtype=tmat.dtype)
+
+        return tmaps if copy else None
+
     @d.get_sections(base="tmk_tmat", sections=["Parameters"])
     @d.dedent
     @inject_docs(st=SelfTransitions)
@@ -99,11 +169,14 @@ def compute_transition_matrix(
         conn_kwargs
             Keyword arguments for :func:`scanpy.pp.neighbors` when using ``self_transitions`` use
             :class:`cellrank.tl.kernels.ConnectivityKernel`. Can contain `'density_normalize'` for
-            :meth:`cellrank.kernels.ConnectivityKernel.compute_transition_matrix`.
+            :meth:`cellrank.tl.kernels.ConnectivityKernel.compute_transition_matrix`.
 
         Returns
         -------
-        Self and updates :attr:`transition_matrix` and :attr:`params`.
+        Self and updates the following attributes:
+
+            - :attr:`transition_matrix` - transition matrix.
+            - :attr:`transport_maps` - transport maps between consecutive time points.
         """
         cache_params = dict(kwargs)
         cache_params["threshold"] = threshold
@@ -312,76 +385,6 @@ def _validate_tmaps(
 
         return tmaps
 
-    @d.get_sections(base="tmk_thresh", sections=["Parameters"])
-    def _threshold_transport_maps(
-        self,
-        tmaps: Dict[Pair_t, AnnData],
-        threshold: Threshold_t,
-        copy: bool = False,
-    ) -> Optional[Mapping[Pair_t, AnnData]]:
-        """
-        Remove small non-zero values from :attr:`transition_matrix`.
-
-        Parameters
-        ----------
-        threshold
-            How to remove small non-zero values from the transition matrix. Valid options are:
-
-                - `'auto'` - find the maximum threshold value which will not remove every non-zero value from any row.
-                - `'auto_local'` - same as above, but done for each transport separately.
-                - :class:`float` - value in `[0, 100]` corresponding to a percentage of non-zeros to remove in each
-                  transport map.
-
-            Rows where all values are removed will have uniform distribution and a warning will be issued.
-        copy
-            Whether to return a copy of thresholded ``tmaps`` or modify inplace.
-
-        Returns
-        -------
-        If ``copy = True``, returns a thresholded transport maps. Otherwise, modifies ``tmaps`` inplace.
-        """
-        if threshold == "auto":
-            thresh = min(
-                adata.X[i].max() for adata in tmaps.values() for i in range(adata.n_obs)
-            )
-            logg.info(f"Using automatic `threshold={thresh}`")
-        elif threshold == "auto_local":
-            logg.info("Using automatic `threshold` for each time point pair separately")
-        elif not (0 <= threshold <= 100):
-            raise ValueError(
-                f"Expected `threshold` to be in `[0, 100]`, found `{threshold}`.`"
-            )
-
-        for key, adata in tmaps.items():
-            if copy:
-                adata = adata.copy()
-            tmat = adata.X
-
-            if threshold == "auto_local":
-                thresh = min(tmat[i].max() for i in range(tmat.shape[0]))
-                logg.debug(f"Using `threshold={thresh}` at `{key}`")
-            elif isinstance(threshold, (int, float)):
-                thresh = np.percentile(tmat.data, threshold)
-                logg.debug(f"Using `threshold={thresh}` at `{key}`")
-
-            tmat = csr_matrix(tmat, dtype=tmat.dtype)
-            tmat.data[tmat.data < thresh] = 0.0
-            zeros_mask = np.where(np.asarray(tmat.sum(1)).squeeze() == 0)[0]
-            if np.any(zeros_mask):
-                logg.warning(
-                    f"After thresholding, `{len(zeros_mask)}` row(s) of transport at `{key}` are forced to be uniform"
-                )
-                for ix in zeros_mask:
-                    start, end = tmat.indptr[ix], tmat.indptr[ix + 1]
-                    size = end - start
-                    tmat.data[start:end] = np.ones(size, dtype=tmat.dtype) / size
-
-            # after `zeros_mask` has been handled, to have access to removed row indices
-            tmat.eliminate_zeros()
-            tmaps[key] = AnnData(tmat, obs=adata.obs, var=adata.var, dtype=tmat.dtype)
-
-        return tmaps if copy else None
-
     def _tmat_to_adata(
         self, t1: Numeric_t, t2: Numeric_t, tmat: Union[np.ndarray, spmatrix, AnnData]
     ) -> AnnData:

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -260,6 +260,7 @@ def __repr__(self) -> str:
     "doc_module": "cellrank",
     "download_all_examples": False,
     "pypandoc": True,  # convert rST to md when downloading notebooks
+    "plot_gallery": "'True'",  # https://github.com/sphinx-gallery/sphinx-gallery/issues/913
 }
 
 # -- Options for HTML output -------------------------------------------------

diff --git a/docs/source/installation.rst b/docs/source/installation.rst
@@ -29,7 +29,7 @@ Development Version
 ~~~~~~~~~~~~~~~~~~~
 To stay up-to-date with the newest version, run::
 
-    pip install git+https://github.com/theislab/cellrank@dev
+    pip install git+https://github.com/theislab/cellrank@master
 
 Dependencies
 ~~~~~~~~~~~~

diff --git a/docs/source/release/changelog/850.misc.rst b/docs/source/release/changelog/850.misc.rst
@@ -0,0 +1 @@
+Fix PETSc/SLEPc in CI for Linux. For now, macOS PETSc/SLEPc remains disabled.
diff --git a/docs/source/release/notes-dev.rst b/docs/source/release/notes-dev.rst
@@ -1,4 +1,4 @@
-CellRank dev (2022-03-21)
+CellRank dev (2022-04-07)
 =========================
 
 Features
@@ -59,3 +59,6 @@ Miscellaneous
 
 - Ignore :mod:`pygam` deprecation warnings.
   `#798 <https://github.com/theislab/cellrank/pull/798>`__
+
+- Fix PETSc/SLEPc in CI for Linux. For now, macOS PETSc/SLEPc remains disabled.
+  `#850 <https://github.com/theislab/cellrank/pull/850>`__
diff --git a/tox.ini b/tox.ini
@@ -107,12 +107,19 @@ platform =
 deps =
     adjustText
     linux: rpy2>=3.3.0
-    py{37,38}: git+https://github.com/broadinstitute/wot@master
+    py37: git+https://github.com/broadinstitute/wot@master
+    # https://github.com/theislab/cellrank/pull/850
+    slepc: numpy==1.21.0
+    # we manually install PETSc/SLEPc on the CI from source
+    slepc: mpi4py>=3.0.3
+    slepc: petsc==3.15.4
+    slepc: petsc4py==3.15.1
+    slepc: slepc==3.15.2
+    slepc: slepc4py==3.15.1
 extras =
     test
-    slepc: krylov
-    py{37,38}: external
-passenv = TOXENV CI CODECOV_* GITHUB_ACTIONS PYTEST_FLAGS
+    py37: external
+passenv = TOXENV CI CODECOV_* GITHUB_ACTIONS PYTEST_FLAGS PETSC_* SLEPC_*
 usedevelop = true
 commands =
     python -m pytest --cov --cov-append --cov-report=term-missing --cov-config={toxinidir}/tox.ini --ignore docs/ {posargs:-vv} {env:PYTEST_FLAGS:}