let fpgrowth and fpmax work directly with sparse input

docs/sources/CHANGELOG.md

@@ -23,6 +23,7 @@ The CHANGELOG for the current development version is available at
 
 - Improve the runtime performance for the `apriori` frequent itemset generating function when `low_memory=True`. Setting `low_memory=False` (default) is still faster for small itemsets, but `low_memory=True` can be much faster for large itemsets and requires less memory.  Also, input validation for  `apriori`, ̀ fpgrowth` and `fpmax` takes a significant amount of time when input pandas DataFrame is large; this is now dramatically reduced when input contains boolean values (and not zeros/ones), which is the case when using `TransactionEncoder`. ([#619](https://github.com/rasbt/mlxtend/pull/619) via [Denis Barbier](https://github.com/dbarbier))
 - Add support for newer sparse pandas DataFrame for frequent itemset algorithms. Also, input validation for  `apriori`, ̀ fpgrowth` and `fpmax` runs much faster on sparse DataFrame when input pandas DataFrame contains integer values. ([#621](https://github.com/rasbt/mlxtend/pull/621) via [Denis Barbier](https://github.com/dbarbier))
+- Let `fpgrowth` and `fpmax` directly work on sparse DataFrame, they were previously converted into dense Numpy arrays. ([#622](https://github.com/rasbt/mlxtend/pull/622) via [Denis Barbier](https://github.com/dbarbier))
 
 ##### Bug Fixes
 - Fixes a bug in `mlxtend.plotting.plot_pca_correlation_graph` that caused the explaind variances not summing up to 1. Also, improves the runtime performance of the correlation computation and adds a missing function argument for the explained variances (eigenvalues) if users provide their own principal components. ([#593](https://github.com/rasbt/mlxtend/issues/593) via [Gabriel Azevedo Ferreira](https://github.com/Gabriel-Azevedo-Ferreira))

mlxtend/frequent_patterns/fpcommon.py

@@ -6,27 +6,59 @@
 
 
 def setup_fptree(df, min_support):
-    itemsets = df.values
-    num_items = itemsets.shape[1]       # number of unique items
-    num_itemsets = itemsets.shape[0]    # number of itemsets in the database
+    num_itemsets = len(df.index)        # number of itemsets in the database
+
+    is_sparse = False
+    if hasattr(df, "to_coo"):
+        # SparseDataFrame with pandas < 0.24
+        if df.size == 0:
+            itemsets = df.values
+        else:
+            itemsets = df.to_coo().tocsr()
+            is_sparse = True
+    elif hasattr(df, "sparse"):
+        # DataFrame with SparseArray (pandas >= 0.24)
+        if df.size == 0:
+            itemsets = df.values
+        else:
+            itemsets = df.sparse.to_coo().tocsr()
+            is_sparse = True
+    else:
+        # dense DataFrame
+        itemsets = df.values
 
     # support of each individual item
-    item_support = np.sum(itemsets, axis=0) / float(num_itemsets)
+    # if itemsets is sparse, np.sum returns an np.matrix of shape (1, N)
+    item_support = np.array(np.sum(itemsets, axis=0) / float(num_itemsets))
+    item_support = item_support.reshape(-1)
 
-    items = [item for item in range(
-        num_items) if item_support[item] >= min_support]
+    items = np.nonzero(item_support >= min_support)[0]
 
     # Define ordering on items for inserting into FPTree
-    items.sort(key=lambda x: item_support[x])
-    rank = {item: i for i, item in enumerate(items)}
+    indices = item_support[items].argsort()
+    rank = {item: i for i, item in enumerate(items[indices])}
+
+    if is_sparse:
+        # Ensure that there are no zeros in sparse DataFrame
+        itemsets.eliminate_zeros()
 
     # Building tree by inserting itemsets in sorted order
-    # Hueristic for reducing tree size is inserting in order
+    # Heuristic for reducing tree size is inserting in order
     #   of most frequent to least frequent
     tree = FPTree(rank)
     for i in range(num_itemsets):
-        itemset = [item for item in np.where(
-            itemsets[i, :])[0] if item in rank]
+        if is_sparse:
+            # itemsets has been converted to CSR format to speed-up the line
+            # below.  It has 3 attributes:
+            #  - itemsets.data contains non null values, shape(#nnz,)
+            #  - itemsets.indices contains the column number of non null
+            #    elements, shape(#nnz,)
+            #  - itemsets.indptr[i] contains the offset in itemset.indices of
+            #    the first non null element in row i, shape(1+#nrows,)
+            nonnull = itemsets.indices[itemsets.indptr[i]:itemsets.indptr[i+1]]
+        else:
+            nonnull = np.where(itemsets[i, :])[0]
+        itemset = [item for item in nonnull if item in rank]
         itemset.sort(key=rank.get, reverse=True)
         tree.insert_itemset(itemset)
 

mlxtend/frequent_patterns/fpgrowth.py

@@ -70,11 +70,10 @@ def fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0):
         colname_map = {idx: item for idx, item in enumerate(df.columns)}
 
     tree, _ = fpc.setup_fptree(df, min_support)
-
-    minsup = math.ceil(min_support * len(df.values))  # min support as count
+    minsup = math.ceil(min_support * len(df.index))  # min support as count
     generator = fpg_step(tree, minsup, colname_map, max_len, verbose)
 
-    return fpc.generate_itemsets(generator, len(df.values), colname_map)
+    return fpc.generate_itemsets(generator, len(df.index), colname_map)
 
 
 def fpg_step(tree, minsup, colnames, max_len, verbose):

mlxtend/frequent_patterns/tests/test_fpbase.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 from numpy.testing import assert_array_equal
+from scipy.sparse import csr_matrix
 from mlxtend.utils import assert_raises
 from mlxtend.preprocessing import TransactionEncoder
 import pandas as pd
@@ -179,6 +180,17 @@ def test_with_fill_values(fill_value):
         test_with_fill_values(0)
         test_with_fill_values(False)
 
+    def test_sparse_with_zero(self):
+        res_df = self.fpalgo(self.df)
+        ary2 = self.one_ary.copy()
+        ary2[3, :] = 1
+        sparse_ary = csr_matrix(ary2)
+        sparse_ary[3, :] = self.one_ary[3, :]
+        sdf = pd.SparseDataFrame(sparse_ary, columns=self.df.columns,
+                                 default_fill_value=0)
+        res_df2 = self.fpalgo(sdf)
+        compare_dataframes(res_df2, res_df)
+
 
 class FPTestEx1All(FPTestEx1):
     def setUp(self, fpalgo, one_ary=None):