Make scipy an optional dependency (#2398)

* Vendor reduced version of scipy's KDE, adapt existing KDE code * Adapt violinplot and stripplot to not require scipy * Remove vendored kde docstring examples * Fix missing import * Remove more unhelpful doctests * Note that cumulative kdeplot requres scipy * Make scipy optional in matrix module * Replace scipy iqr calls and adapt old distplot code * Standardize flag for scipy availability * Define extra dependencies in setup file * Rework CI workflow for new dependency strategy * Pass extras_require to setuptools * Remove previously deprecated utility functions * Remove scipy from categorical tests * Remove scipy from regression module * Remove scipy from categorical * Remove scipy from distribution tests * Protect scipy-using code in matrix tests * Remove scipy from distributions tests (again) * Fix docs about scipy requirements * Update installing docs * Update release notes * Cover some missing-scipy-triggered errors in tests * Update README and Remove requirements file
mwaskom · Dec 23, 2020 · f1852b5 · f1852b5
1 parent 0a24478
commit f1852b5
Show file tree

Hide file tree

Showing 21 changed files with 618 additions and 280 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -25,8 +25,7 @@ jobs:
       - name: Install seaborn
         run: |
           python -m pip install --upgrade pip
-          pip install `cat ci/deps_latest.txt ci/utils.txt`
-          pip install .
+          pip install .[all] -r ci/utils.txt
 
       - name: Install doc tools
         run: |
@@ -47,20 +46,24 @@ jobs:
 
         python: [3.7.x, 3.8.x, 3.9.x]
         target: [test]
+        install: [all]
         deps: [latest]
         backend: [agg]
 
         include:
           - python: 3.7.x
             target: unittests
+            install: all
             deps: pinned
             backend: agg
           - python: 3.9.x
             target: unittests
-            deps: minimal
+            install: light
+            deps: latest
             backend: agg
           - python: 3.9.x
             target: test
+            install: all
             deps: latest
             backend: tkagg
 
@@ -77,8 +80,9 @@ jobs:
       - name: Install seaborn
         run: |
           python -m pip install --upgrade pip
-          pip install `cat ci/deps_${{ matrix.deps }}.txt ci/utils.txt`
-          pip install .
+          if [[ ${{matrix.install}} == 'all' ]]; then EXTRAS='[all]'; fi
+          if [[ ${{matrix.deps }} == 'pinned' ]]; then DEPS='-r ci/deps_pinned.txt'; fi
+          pip install .$EXTRAS $DEPS -r ci/utils.txt
 
       - name: Cache datastes
         run: python ci/cache_test_datasets.py

diff --git a/README.md b/README.md
@@ -29,20 +29,29 @@ Dependencies
 
 Seaborn supports Python 3.7+ and no longer supports Python 2.
 
-Installation requires [numpy](https://numpy.org/), [scipy](https://www.scipy.org/), [pandas](https://pandas.pydata.org/), and [matplotlib](https://matplotlib.org/). Some functions will optionally use [statsmodels](https://www.statsmodels.org/) if it is installed.
+Installation requires [numpy](https://numpy.org/), [pandas](https://pandas.pydata.org/), and [matplotlib](https://matplotlib.org/). Some functions will optionally use [scipy](https://www.scipy.org/) and/or [statsmodels](https://www.statsmodels.org/) if they are available.
 
 
 Installation
 ------------
 
-The latest stable release (and older versions) can be installed from PyPI:
+The latest stable release (and required dependencies) can be installed from PyPI:
 
     pip install seaborn
 
+It is also possible to include the optional dependencies:
+
+    pip install seaborn[all]
+
 You may instead want to use the development version from Github:
 
-    pip install git+https://github.com/mwaskom/seaborn.git#egg=seaborn
+    pip install git+https://github.com/mwaskom/seaborn.git
+
+Seaborn is also available from Anaconda and can be installed with conda:
+
+    conda install seaborn
 
+Note that the main anaconda repository typically lags PyPI in adding new releases.
 
 Testing
 -------

diff --git a/ci/deps_latest.txt b/ci/deps_latest.txt
diff --git a/ci/deps_minimal.txt b/ci/deps_minimal.txt
diff --git a/ci/deps_pinned.txt b/ci/deps_pinned.txt
@@ -1,5 +1,5 @@
 numpy~=1.16.0
-scipy~=1.2.0
 pandas~=0.23.0
 matplotlib~=3.0.0
+scipy~=1.2.0
 statsmodels~=0.9.0
diff --git a/doc/installing.rst b/doc/installing.rst
@@ -9,12 +9,17 @@ Installing and getting started
 
    <div class="col-md-9">
 
-Official releases of seaborn can be installed from `PyPI
-<https://pypi.org/project/seaborn/>`_::
+Official releases of seaborn can be installed from `PyPI <https://pypi.org/project/seaborn/>`_:
 
     pip install seaborn
 
-The library is also included as part of the `Anaconda <https://repo.anaconda.com/>`_ distribution::
+The basic invocation of `pip` will install seaborn and, if necessary, its mandatory dependencies.
+It is possible to include optional dependencies that give access to a few advanced features:
+
+    pip install seaborn[all]
+
+The library is also included as part of the `Anaconda <https://repo.anaconda.com/>`_ distribution,
+and it can be installed with `conda`:
 
     conda install seaborn
 
@@ -26,15 +31,11 @@ Supported Python versions
 
 - Python 3.7+
 
-Required dependencies
+Mandatory dependencies
 ^^^^^^^^^^^^^^^^^^^^^^
 
-If not already present, these libraries will be downloaded when you install seaborn.
-
 - `numpy <https://numpy.org/>`__
 
-- `scipy <https://www.scipy.org/>`__
-
 - `pandas <https://pandas.pydata.org/>`__
 
 - `matplotlib <https://matplotlib.org>`__
@@ -44,7 +45,9 @@ Optional dependencies
 
 - `statsmodels <https://www.statsmodels.org/>`__, for advanced regression plots
 
-- `fastcluster <https://pypi.org/project/fastcluster/>`__, for clustering large matrices
+- `scipy <https://www.scipy.org/>`__, for clustering matrices and some advanced options
+
+- `fastcluster <https://pypi.org/project/fastcluster/>`__, faster clustering of large matrices
 
 Quickstart
 ~~~~~~~~~~

diff --git a/doc/releases/v0.12.0.txt b/doc/releases/v0.12.0.txt
@@ -2,4 +2,8 @@
 v0.12.0 (Unreleased)
 --------------------
 
-- Following `NEP29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_, dropped support for Python 3.6 and bumped the minimally-supported versions of the library dependencies.
+- Made `scipy` an optional dependency and added `pip install seaborn[all]` as a method for ensuring the availability of compatible `scipy` and `statsmodels` libraries. This has a few minor implications for existing code, which are explained in the Github pull request (:pr:`2398`).
+
+- Following `NEP29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_, dropped support for Python 3.6 and bumped the minimally-supported versions of the library dependencies.
+
+- Removed several previously-deprecated utility functions (`iqr`, `percentiles`, `pmf_hist`, and `sort_df`).
diff --git a/requirements.txt b/requirements.txt
diff --git a/seaborn/_statistics.py b/seaborn/_statistics.py
@@ -26,7 +26,12 @@ class instantiation.
 """
 from numbers import Number
 import numpy as np
-from scipy import stats
+try:
+    from scipy.stats import gaussian_kde
+    _no_scipy = False
+except ImportError:
+    from .external.kde import gaussian_kde
+    _no_scipy = True
 
 from .utils import _check_argument
 
@@ -61,7 +66,7 @@ def __init__(
         clip : pair of numbers None, or a pair of such pairs
             Do not evaluate the density outside of these limits.
         cumulative : bool, optional
-            If True, estimate a cumulative distribution function.
+            If True, estimate a cumulative distribution function. Requires scipy.
 
         """
         if clip is None:
@@ -74,6 +79,9 @@ def __init__(
         self.clip = clip
         self.cumulative = cumulative
 
+        if cumulative and _no_scipy:
+            raise RuntimeError("Cumulative KDE evaluation requires scipy")
+
         self.support = None
 
     def _define_support_grid(self, x, bw, cut, clip, gridsize):
@@ -129,7 +137,7 @@ def _fit(self, fit_data, weights=None):
         if weights is not None:
             fit_kws["weights"] = weights
 
-        kde = stats.gaussian_kde(fit_data, **fit_kws)
+        kde = gaussian_kde(fit_data, **fit_kws)
         kde.set_bandwidth(kde.factor * self.bw_adjust)
 
         return kde

diff --git a/seaborn/categorical.py b/seaborn/categorical.py
@@ -1,18 +1,26 @@
 from textwrap import dedent
 from numbers import Number
+import warnings
 import colorsys
+from functools import partial
+
 import numpy as np
-from scipy import stats
 import pandas as pd
+try:
+    from scipy.stats import gaussian_kde
+    _no_scipy = False
+except ImportError:
+    from .external.kde import gaussian_kde
+    _no_scipy = True
+
 import matplotlib as mpl
 from matplotlib.collections import PatchCollection
 import matplotlib.patches as Patches
 import matplotlib.pyplot as plt
-import warnings
 
 from ._core import variable_type, infer_orient, categorical_order
 from . import utils
-from .utils import remove_na
+from .utils import remove_na, _normal_quantile_func
 from .algorithms import bootstrap
 from .palettes import color_palette, husl_palette, light_palette, dark_palette
 from .axisgrid import FacetGrid, _facet_docs
@@ -662,7 +670,7 @@ def estimate_densities(self, bw, cut, scale, scale_hue, gridsize):
 
     def fit_kde(self, x, bw):
         """Estimate a KDE for a vector of data with flexible bandwidth."""
-        kde = stats.gaussian_kde(x, bw)
+        kde = gaussian_kde(x, bw)
 
         # Extract the numeric bandwidth from the KDE object
         bw_used = kde.factor
@@ -942,7 +950,7 @@ def draw_box_lines(self, ax, data, support, density, center):
         """Draw boxplot information at center of the density."""
         # Compute the boxplot statistics
         q25, q50, q75 = np.percentile(data, [25, 50, 75])
-        whisker_lim = 1.5 * stats.iqr(data)
+        whisker_lim = 1.5 * (q75 - q25)
         h1 = np.min(data[data >= (q25 - whisker_lim)])
         h2 = np.max(data[data <= (q75 + whisker_lim)])
 
@@ -1099,7 +1107,7 @@ def __init__(self, x, y, hue, data, order, hue_order,
             jlim = float(jitter)
         if self.hue_names is not None and dodge:
             jlim /= len(self.hue_names)
-        self.jitterer = stats.uniform(-jlim, jlim * 2).rvs
+        self.jitterer = partial(np.random.uniform, low=-jlim, high=+jlim)
 
     def draw_stripplot(self, ax, kws):
         """Draw the points onto `ax`."""
@@ -1120,7 +1128,7 @@ def draw_stripplot(self, ax, kws):
 
                 # Plot the points in centered positions
                 cat_pos = np.ones(strip_data.size) * i
-                cat_pos += self.jitterer(len(strip_data))
+                cat_pos += self.jitterer(size=len(strip_data))
                 kws.update(c=palette[point_colors])
                 if self.orient == "v":
                     ax.scatter(cat_pos, strip_data, **kws)
@@ -1138,7 +1146,7 @@ def draw_stripplot(self, ax, kws):
                     # Plot the points in centered positions
                     center = i + offsets[j]
                     cat_pos = np.ones(strip_data.size) * center
-                    cat_pos += self.jitterer(len(strip_data))
+                    cat_pos += self.jitterer(size=len(strip_data))
                     kws.update(c=palette[point_colors])
                     if self.orient == "v":
                         ax.scatter(cat_pos, strip_data, **kws)
@@ -1846,7 +1854,7 @@ def _lv_box_ends(self, vals):
         elif self.k_depth == 'proportion':
             k = int(np.log2(n)) - int(np.log2(n * p)) + 1
         elif self.k_depth == 'trustworthy':
-            point_conf = 2 * stats.norm.ppf((1 - self.trust_alpha / 2)) ** 2
+            point_conf = 2 * _normal_quantile_func((1 - self.trust_alpha / 2)) ** 2
             k = int(np.log2(n / point_conf)) + 1
         else:
             k = int(self.k_depth)  # allow having k as input

diff --git a/seaborn/distributions.py b/seaborn/distributions.py
@@ -11,7 +11,6 @@
 import matplotlib.transforms as tx
 from matplotlib.colors import to_rgba
 from matplotlib.collections import LineCollection
-from scipy import stats
 
 from ._core import (
     VectorPlotter,
@@ -34,6 +33,7 @@
 )
 from .palettes import color_palette
 from .external import husl
+from .external.kde import gaussian_kde
 from ._decorators import _deprecate_positional_args
 from ._docstrings import (
     DocstringComponents,
@@ -2395,7 +2395,8 @@ def _freedman_diaconis_bins(a):
     a = np.asarray(a)
     if len(a) < 2:
         return 1
-    h = 2 * stats.iqr(a) / (len(a) ** (1 / 3))
+    iqr = np.subtract.reduce(np.nanpercentile(a, [75, 25]))
+    h = 2 * iqr / (len(a) ** (1 / 3))
     # fall back to sqrt(a) bins if iqr is 0
     if h == 0:
         return int(np.sqrt(a.size))
@@ -2642,7 +2643,7 @@ def pdf(x):
         gridsize = fit_kws.pop("gridsize", 200)
         cut = fit_kws.pop("cut", 3)
         clip = fit_kws.pop("clip", (-np.inf, np.inf))
-        bw = stats.gaussian_kde(a).scotts_factor() * a.std(ddof=1)
+        bw = gaussian_kde(a).scotts_factor() * a.std(ddof=1)
         x = _kde_support(a, bw, gridsize, cut, clip)
         params = fit.fit(a)
         y = pdf(x)