Merge branch 'master' into dev

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,69 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Standalone code to reproduce the error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Actual behavior**
+Please include the full traceback of any errors
+
+**System information:**
+
+Output of `MAGIC.__version__`:
+
+```
+If you are running MAGIC in R or Python, please run magic.__version__ and paste the results here.
+
+You can do this with `python -c 'import magic; print(magic.__version__)'`
+```
+
+Output of `pd.show_versions()`:
+
+<details>
+
+```
+If you are running MAGIC in R or Python, please run pd.show_versions() and paste the results here.
+
+You can do this with `python -c 'import pandas as pd; pd.show_versions()'`
+```
+
+</details>
+
+Output of `sessionInfo()`:
+
+<details>
+
+```
+If you are running MAGIC in R, please run sessionInfo() and paste the results here.
+
+You can do this with `R -e 'library(Rmagic); sessionInfo()'`
+```
+
+</details>
+
+Output of `reticulate::py_disc()`:
+
+<details>
+
+```
+If you are running MAGIC in R, please run `reticulate::py_discover_config(required_module = "magic")` and paste the results here.
+
+You can do this with `R -e 'reticulate::py_discover_config(required_module = "magic")'`
+```
+
+</details>
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask questions about MAGIC
+title: ''
+labels: question
+assignees: ''
+
+---
+
+

python/magic/magic.py

@@ -40,6 +40,13 @@ class MAGIC(BaseEstimator):
     applied to single-cell RNA sequencing data, as described in
     van Dijk et al, 2018 [1]_.
 
+    The algorithm implemented here has changed primarily in two ways
+    compared to the algorithm described in [1]_. Firstly, we use
+    the adaptive kernel described in Moon et al, 2019 [2]_ for
+    improved stability. Secondly, data diffusion is applied
+    in the PCA space, rather than the data space, for speed and
+    memory improvements.
+
     Parameters
     ----------
 
@@ -65,6 +72,12 @@ class MAGIC(BaseEstimator):
         n_pca < 20 allows neighborhoods to be calculated in
         roughly log(n_samples) time.
 
+    solver : str, optional, default: 'exact'
+        Which solver to use. "exact" uses the implementation described
+        in van Dijk et al. (2018). "approximate" uses a faster implementation
+        that performs imputation in the PCA space and then projects back to the
+        gene space. Note, the "approximate" solver may return negative values.
+
     knn_dist : string, optional, default: 'euclidean'
         Distance metric for building kNN graph. Recommended values:
         'euclidean', 'cosine'. Any metric from `scipy.spatial.distance` can be
@@ -134,10 +147,14 @@ class MAGIC(BaseEstimator):
     .. [1] Van Dijk D *et al.* (2018),
         *Recovering Gene Interactions from Single-Cell Data Using Data Diffusion*,
         `Cell <https://www.cell.com/cell/abstract/S0092-8674(18)30724-4>`__.
+
+    .. [2] Moon, van Dijk, Wang, Gigante *et al.* (2019),
+        *Visualizing Structure and Transitions in High-Dimensional Biological Data*,
+        `Nature Biotechnology (in press)`__.
     """
 
     def __init__(self, knn=10, knn_max=None, decay=2, t='auto', n_pca=100,
-                 knn_dist='euclidean', n_jobs=1, random_state=None,
+                 solver='exact', knn_dist='euclidean', n_jobs=1, random_state=None,
                  verbose=1, k=None, a=None):
         if k is not None:
             knn = k
@@ -151,7 +168,7 @@ def __init__(self, knn=10, knn_max=None, decay=2, t='auto', n_pca=100,
         self.knn_dist = knn_dist
         self.n_jobs = n_jobs
         self.random_state = random_state
-
+        self.solver = solver
         self.graph = None
         self.X = None
         self.X_magic = None
@@ -191,6 +208,8 @@ def _check_params(self):
                            decay=self.decay)
         utils.check_if_not('auto', utils.check_positive, utils.check_int,
                            t=self.t)
+        utils.check_in(['exact', 'approximate'],
+                         solver=self.solver)
         if not callable(self.knn_dist):
             utils.check_in(['euclidean', 'cosine', 'correlation',
                             'cityblock', 'l1', 'l2', 'manhattan', 'braycurtis',
@@ -301,6 +320,10 @@ def set_params(self, **params):
             del params['knn_dist']
 
         # parameters that don't change the embedding
+        if 'solver' in params and params['solver'] != self.solver:
+            self.solver = params['solver']
+            reset_imputation = True
+            del params['solver']
         if 'n_jobs' in params:
             self.n_jobs = params['n_jobs']
             self._set_graph_params(n_jobs=params['n_jobs'])
@@ -633,8 +656,12 @@ def _impute(self, data, t_max=20, plot=False, ax=None,
         X_magic : array-like, shape=[n_samples, n_pca]
             Imputed data
         """
+
         if not isinstance(data, graphtools.base.Data):
-            data = graphtools.base.Data(data, n_pca=self.n_pca)
+            if self.solver == 'approximate':
+                data = graphtools.base.Data(data, n_pca=self.n_pca)
+            elif self.solver == 'exact':
+                data = graphtools.base.Data(data, n_pca=None)
         data_imputed = scprep.utils.toarray(data.data_nu)
 
         if data_imputed.shape[1] > max_genes_compute_t:

python/magic/version.py

@@ -1 +1 @@
-__version__ = "1.5.8a0"
+__version__ = "1.5.8"

python/test/test.py

@@ -22,7 +22,7 @@ def test_scdata():
     scdata_norm = scprep.transform.sqrt(scdata_norm)
     assert scdata.shape == scdata_norm.shape
     np.random.seed(42)
-    magic_op = magic.MAGIC(t='auto', a=20, k=10)
+    magic_op = magic.MAGIC(t='auto', decay=20, knn=10, verbose=False)
     str_gene_magic = magic_op.fit_transform(
         scdata_norm, genes=['VIM', 'ZEB1'])
     int_gene_magic = magic_op.fit_transform(
@@ -39,6 +39,20 @@ def test_scdata():
     dremi = magic_op.knnDREMI("VIM", "ZEB1", plot=True)
     np.testing.assert_allclose(dremi, 1.573619, atol=0.0000005)
 
+    # Testing exact vs approximate solver
+    magic_op = magic.MAGIC(t='auto', decay=20, knn=10, solver='exact', verbose=False)
+    data_imputed_exact = magic_op.fit_transform(scdata_norm)
+    assert np.all(data_imputed_exact >= 0)
+
+    magic_op = magic.MAGIC(t='auto', decay=20, knn=10, solver='approximate', verbose=False)
+    #magic_op.set_params(solver='approximate')
+    data_imputed_apprx = magic_op.fit_transform(scdata_norm)
+    # make sure they're close-ish
+    assert np.allclose(data_imputed_apprx, data_imputed_exact, atol=0.05)
+    # make sure they're not identical
+    assert np.any(data_imputed_apprx != data_imputed_exact)
+
+
 
 def test_anndata():
     try:
@@ -47,7 +61,8 @@ def test_anndata():
         # anndata not installed
         return
     scdata = anndata.read_csv("../data/test_data.csv")
-    fast_magic_operator = magic.MAGIC(t='auto', a=None, k=10)
+    fast_magic_operator = magic.MAGIC(t='auto', solver='approximate',
+            decay=None, knn=10, verbose=False)
     sc_magic = fast_magic_operator.fit_transform(
         scdata, genes="all_genes")
     assert np.all(sc_magic.var_names == scdata.var_names)