Merge pull request #128 from scottgigante-immunai/scottgigante/rfunct…

…ion/traceback

Print traceback on RFunction error

.github/workflows/pre-commit.yml

@@ -26,14 +26,20 @@ jobs:
           path: ~/.cache/pre-commit
           key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}-
 
-      - uses: pre-commit/action@v2.0.0
+      - name: Install system dependencies
+        run: |
+          pip install --upgrade pip wheel setuptools
+
+      - name: Run pre-commit
+        uses: pre-commit/action@v2.0.0
         continue-on-error: true
 
       - name: Commit files
         run: |
           if [[ `git status --porcelain --untracked-files=no` ]]; then
             git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
             git config --local user.name "github-actions[bot]"
+            git checkout -- .github/workflows
             git commit -m "pre-commit" -a
           fi
 
@@ -42,3 +48,6 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           branch: ${{ github.ref }}
+
+      - name: Check pre-commit
+        uses: pre-commit/action@v2.0.0

.github/workflows/run_tests.yml

@@ -9,42 +9,6 @@ on:
       - '*'
 
 jobs:
-  run_linter:
-    runs-on: ${{ matrix.config.os }}
-    if: "!contains(github.event.head_commit.message, 'ci skip')"
-
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-        - {name: 'current', os: ubuntu-latest, python: '3.8' }
-
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.6.0
-      with:
-        access_token: ${{ github.token }}
-
-    - uses: actions/checkout@v2
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.config.python }}
-
-    - name: Install tools
-      run: |
-        python -m pip install --upgrade pip
-        pip install -U wheel setuptools
-        pip install -U black flake8 hacking
-    - name: Lint with Black
-      run: |
-        black . --check --diff
-
-    - name: Lint with flake8
-      run: |
-        flake8 scprep
-
   run_tester:
     runs-on: ${{ matrix.config.os }}
     if: "!contains(github.event.head_commit.message, 'ci skip')"
@@ -55,10 +19,10 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {name: '3.9', os: ubuntu-latest, python: '3.9', r: 'release' }
-        - {name: '3.8', os: ubuntu-latest, python: '3.8', r: 'release' }
-        - {name: '3.7', os: ubuntu-latest, python: '3.7', r: 'release' }
-        - {name: '3.6', os: ubuntu-latest, python: '3.6', r: 'release' }
+        - {name: '3.9', os: ubuntu-latest, python: '3.9', r: 'release'}
+        - {name: '3.8', os: ubuntu-latest, python: '3.8', r: 'release'}
+        - {name: '3.7', os: ubuntu-latest, python: '3.7', r: 'release'}
+        - {name: '3.6', os: ubuntu-latest, python: '3.6', r: 'oldrel'}
 
     steps:
     - name: Cancel Previous Runs
@@ -77,40 +41,50 @@ jobs:
         sudo apt-get install -y pandoc gfortran libblas-dev liblapack-dev libedit-dev llvm-dev libcurl4-openssl-dev ffmpeg libhdf5-dev
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.config.python }}
 
     - name: Set up R
       id: setup-r
-      uses: r-lib/actions/setup-r@v1
+      uses: r-lib/actions/setup-r@v2
       with:
         r-version: ${{ matrix.config.r }}
 
     - name: Cache Python packages
-      uses: actions/cache@v2
+      uses: pat-s/always-upload-cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{runner.os}}-pip-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
         restore-keys: ${{runner.os}}-pip-${{ env.pythonLocation }}-
 
     - name: Cache R packages
-      uses: actions/cache@v2
+      uses: pat-s/always-upload-cache@v2
       if: startsWith(runner.os, 'Linux')
       with:
         path: ${{ env.RENV_PATHS_ROOT }}
         key: ${{ runner.os }}-renv-${{ steps.setup-r.outputs.installed-r-version }}-${{ hashFiles('**/renv.lock') }}
         restore-keys: |
           ${{ runner.os }}-renv-${{ steps.setup-r.outputs.installed-r-version }}-
 
-    - name: Install package & dependencies
+    - name: Install python tools
       run: |
         python -m pip install --upgrade pip
         pip install --upgrade wheel setuptools
+
+    - name: Set up rpy2
+      run: |
+        pip install rpy2
+        echo "LD_LIBRARY_PATH=$(python -m rpy2.situation LD_LIBRARY_PATH):${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+    - name: Install scprep
+      id: install-python-packages
+      run: |
         pip install --upgrade .[test]
         python -c "import scprep"
 
     - name: Install R packages
+      id: install-r-packages
       run: |
         if (!requireNamespace("renv", quietly = TRUE)) install.packages("renv")
         renv::restore()
@@ -120,13 +94,6 @@ jobs:
         renv::install("github::dynverse/dynwrap")
       shell: Rscript {0}
 
-    - name: Install package & dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -U wheel setuptools
-        pip install -U .[test]
-        python -c "import scprep"
-
     - name: Run tests
       run: nose2 -vvv
 

.gitignore

@@ -15,3 +15,4 @@ doc/source/examples/*.png
 .syncthing.*
 
 .DS_Store
+.idea

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
       - id: black
         args: ['--target-version=py36']

scprep/_lazyload.py

@@ -24,7 +24,7 @@
     "rpy2": [
         {"robjects": ["numpy2ri", "pandas2ri", "packages", "vectors", "conversion"]},
         "rinterface",
-        {"rinterface_lib": ["callbacks"]},
+        {"rinterface_lib": ["callbacks", "embedded"]},
     ],
     "h5py": [],
     "tables": [],

scprep/normalize.py

@@ -76,7 +76,7 @@ def library_size_normalize(data, rescale=10000, return_library_size=False):
             # dense data
             data = data.to_numpy()
 
-    calc_libsize = sparse.issparse(data) and (return_library_size or data.nnz > 2 ** 31)
+    calc_libsize = sparse.issparse(data) and (return_library_size or data.nnz > 2**31)
     rescale, libsize = _get_scaled_libsize(data, rescale, calc_libsize)
 
     if libsize is not None:

scprep/plot/marker.py

@@ -79,7 +79,7 @@ def _cluster_tissues(tissue_names, cluster_names, tissue_labels, cluster_labels,
         tissue_features.append(np.concatenate(tissue_data))
     tissue_features = np.array(tissue_features)
     # normalize
-    tissue_features = tissue_features / np.sqrt(np.sum(tissue_features ** 2))
+    tissue_features = tissue_features / np.sqrt(np.sum(tissue_features**2))
     tissues_order = hierarchy.leaves_list(hierarchy.linkage(tissue_features))
     return tissues_order
 
@@ -103,7 +103,7 @@ def _cluster_markers(
                 marker_features.append(np.concatenate([s[marker_idx], c[marker_idx]]))
             marker_features = np.array(marker_features)
             # normalize
-            marker_features = marker_features / np.sqrt(np.sum(marker_features ** 2))
+            marker_features = marker_features / np.sqrt(np.sum(marker_features**2))
             marker_group_order = hierarchy.leaves_list(
                 hierarchy.linkage(marker_features)
             )

scprep/plot/scree.py

@@ -63,7 +63,7 @@ def scree_plot(
     >>> scprep.plot.scree_plot(singular_values, cumulative=True)
     """
     with temp_fontsize(fontsize):
-        explained_variance = singular_values ** 2
+        explained_variance = singular_values**2
         explained_variance = explained_variance / explained_variance.sum() * 100
         if cumulative:
             explained_variance = np.cumsum(explained_variance)

scprep/run/r_function.py

@@ -2,6 +2,8 @@
 from .._lazyload import rpy2
 from . import conversion
 
+import functools
+
 
 def _console_warning(s, log_fn):
     s = s.strip()
@@ -58,6 +60,16 @@ def __exit__(self, type, value, traceback):
         self.set(self.previous_warning)
 
 
+@functools.lru_cache(None)
+def setup_rlang():
+    rpy2.robjects.r(
+        """
+    if (!require('rlang')) install.packages('rlang')
+    options(error = rlang::entrace)
+    """
+    )
+
+
 class RFunction(object):
     """Run an R function from Python.
 
@@ -88,6 +100,7 @@ def __init__(self, args="", setup="", body="", cleanup=True, verbose=1):
 
     @utils._with_pkg(pkg="rpy2", min_version="3.0")
     def _build(self):
+        setup_rlang()
         if self.setup != "":
             with _ConsoleWarning(self.verbose - 1):
                 rpy2.robjects.r(self.setup)
@@ -121,7 +134,23 @@ def __call__(self, *args, rpy_cleanup=None, rpy_verbose=None, **kwargs):
         args = [conversion.py2rpy(a) for a in args]
         kwargs = {k: conversion.py2rpy(v) for k, v in kwargs.items()}
         with _ConsoleWarning(rpy_verbose):
-            robject = self.function(*args, **kwargs)
+            try:
+                robject = self.function(*args, **kwargs)
+            except rpy2.rinterface_lib.embedded.RRuntimeError as e:
+                # Attempt to capture the traceback from R.
+                # Credit: https://stackoverflow.com/a/40002973
+                try:
+                    r_traceback = rpy2.robjects.r(
+                        "format(rlang::last_trace(), simplify='none', fields=TRUE)"
+                    )[0]
+                except Exception as traceback_exc:
+                    r_traceback = (
+                        "\n(an error occurred while getting traceback "
+                        f"from R){traceback_exc}"
+                    )
+                e.args = (f"{e.args[0]}\n{r_traceback}",)
+                raise
+
             robject = conversion.rpy2py(robject)
             if rpy_cleanup:
                 rpy2.robjects.r("rm(list=ls())")
@@ -131,7 +160,7 @@ def __call__(self, *args, rpy_cleanup=None, rpy_verbose=None, **kwargs):
 
 _install_bioconductor = RFunction(
     args="package = character(), site_repository = character(), update = FALSE, "
-    "version = BiocManager::version()",
+    'type="binary", version = BiocManager::version()',
     body="""
         if (!require('BiocManager')) install.packages("BiocManager")
         ask <- !update
@@ -142,7 +171,7 @@ def __call__(self, *args, rpy_cleanup=None, rpy_verbose=None, **kwargs):
           for (pkg in package) {
             if (update || !require(pkg, character.only = TRUE)) {
               BiocManager::install(pkg, site_repository=site_repository,
-                                   update=update, ask=ask, version=version)
+                                   update=update, ask=ask, version=version, type=type)
             }
           }
         }
@@ -151,7 +180,12 @@ def __call__(self, *args, rpy_cleanup=None, rpy_verbose=None, **kwargs):
 
 
 def install_bioconductor(
-    package=None, site_repository=None, update=False, version=None, verbose=True
+    package=None,
+    site_repository=None,
+    update=False,
+    type="binary",
+    version=None,
+    verbose=True,
 ):
     """Install a Bioconductor package.
 
@@ -163,14 +197,17 @@ def install_bioconductor(
     update : boolean, optional (default: False)
         When False, don't attempt to update old packages.
         When True, update old packages automatically.
+    type : {"binary", "source", "both"}, optional (default: "binary")
+        Which package version to install if a newer version is available as source.
+        "both" tries source first and uses binary as a fallback.
     version : string, optional (default: None)
         Bioconductor version to install, e.g., version = "3.8".
         The special symbol version = "devel" installs the current 'development' version.
         If None, installs from the current version.
     verbose : boolean, optional (default: True)
         Install script verbosity.
     """
-    kwargs = {"update": update, "rpy_verbose": verbose}
+    kwargs = {"update": update, "rpy_verbose": verbose, "type": type}
     if package is not None:
         kwargs["package"] = package
     if site_repository is not None:
@@ -182,15 +219,16 @@ def install_bioconductor(
 
 _install_github = RFunction(
     args="""repo=character(), lib=.libPaths()[1], dependencies=NA,
-            update=FALSE, repos='http://cran.rstudio.com',
+            update=FALSE, type="binary",
             build_vignettes=FALSE, force=FALSE, verbose=TRUE""",
     body="""
         quiet <- !verbose
 
-        if (!require('remotes', quietly=TRUE)) install.packages('remotes')
-        remotes::install_github(repo=repo,
+        if (!require('remotes', quietly=TRUE)) install.packages('remotes', lib=lib)
+        library(remotes)
+        install_github(repo=repo,
                          lib=lib, dependencies=dependencies,
-                         upgrade=update, repos=repos,
+                         upgrade=update,
                          build_vignettes=build_vignettes,
                          force=force, quiet=quiet)
 
@@ -207,7 +245,7 @@ def install_github(
     lib=None,
     dependencies=None,
     update=False,
-    repos="http://cran.us.r-project.org",
+    type="binary",
     build_vignettes=False,
     force=False,
     verbose=True,
@@ -233,16 +271,17 @@ def install_github(
         "ask" prompts the user for which out of date packages to upgrade.
         For non-interactive sessions "ask" is equivalent to "always".
         TRUE and FALSE also accepted, correspond to "always" and "never" respectively.
-    repos: string, optional (default: "http://cran.us.r-project.org"):
-        R package repository.
+    type : {"binary", "source", "both"}, optional (default: "binary")
+        Which package version to install if a newer version is available as source.
+        "both" tries source first and uses binary as a fallback.
     build_vignettes: boolean, optional (default: False)
         Builds Github vignettes.
     force: boolean, optional (default: False)
         Forces installation even if remote state has not changed since previous install.
     verbose: boolean, optional (default: True)
         Install script verbosity.
     """
-    kwargs = {}
+    kwargs = {"type": type}
     if lib is not None:
         kwargs["lib"] = lib
     if dependencies is not None:
@@ -251,7 +290,6 @@ def install_github(
     _install_github(
         repo=repo,
         update=update,
-        repos=repos,
         build_vignettes=build_vignettes,
         force=force,
         verbose=verbose,

scprep/run/slingshot.py

@@ -212,7 +212,7 @@ def Slingshot(
     ...     ax.plot(curve[:,0], curve[:,1], c='black')
     """
     if seed is None:
-        seed = np.random.randint(2 ** 16 - 1)
+        seed = np.random.randint(2**16 - 1)
     if distance is not None:
         raise NotImplementedError("distance argument not currently implemented")
     np.random.seed(seed)