[MRG] Add iterable values support for dictvectorizer

doc/whats_new/v0.24.rst

@@ -52,6 +52,13 @@ Changelog
   which allows monitoring of each stage.
   :pr:`16985` by :user:`Hao Chun Chang <haochunchang>`.
 
+:mod:`sklearn.feature_extraction`
+.................................
+
+- |Enhancement| :class:`feature_extraction.DictVectorizer` accepts multiple
+  values for one categorical feature. :pr:`17367` by :user:`Peng Yu <yupbank>`
+  and :user:`Chiara Marmo <cmarmo>`
+
 :mod:`sklearn.feature_selection`
 ................................
 

sklearn/feature_extraction/_dict_vectorizer.py

@@ -3,8 +3,9 @@
 # License: BSD 3 clause
 
 from array import array
-from collections.abc import Mapping
+from collections.abc import Mapping, Iterable
 from operator import itemgetter
+from numbers import Number
 
 import numpy as np
 import scipy.sparse as sp
@@ -35,6 +36,10 @@ class DictVectorizer(TransformerMixin, BaseEstimator):
     a feature "f" that can take on the values "ham" and "spam" will become two
     features in the output, one signifying "f=ham", the other "f=spam".
 
+    When feature values are iterables but not mapping, this transformer will
+    iterate over the values to perform the same transformation to every
+    element.
+
     However, note that this transformer will only do a binary one-hot encoding
     when feature values are of type string. If categorical features are
     represented as numeric values such as int, the DictVectorizer can be
@@ -78,12 +83,30 @@ class DictVectorizer(TransformerMixin, BaseEstimator):
     >>> X
     array([[2., 0., 1.],
            [0., 1., 3.]])
-    >>> v.inverse_transform(X) == \
-        [{'bar': 2.0, 'foo': 1.0}, {'baz': 1.0, 'foo': 3.0}]
+    >>> v.inverse_transform(X) == [{'bar': 2.0, 'foo': 1.0},
+    ...                            {'baz': 1.0, 'foo': 3.0}]
     True
     >>> v.transform({'foo': 4, 'unseen_feature': 3})
     array([[0., 0., 4.]])
 
+
+    Example with multiple values for one categorical values:
+
+
+    >>> D2 = [{'foo': '1', 'bar': '2'}, {'foo': '3', 'baz': '1'},
+    ...       {'foo': ['1', '3']}]
+    >>> X = v.fit_transform(D2)
+    >>> X
+    array([[1., 0., 1., 0.],
+           [0., 1., 0., 1.],
+           [0., 0., 1., 1.]])
+    >>> v.inverse_transform(X) == [{'foo=1': 1.0, 'bar=2': 1.0},
+    ...                            {'foo=3': 1.0, 'baz=1': 1.0},
+    ...                            {'foo=3': 1.0, 'foo=1': 1.0}]
+    True
+    >>> v.transform({'foo': '1', 'unseen_feature': [3]})
+    array([[0., 0., 1., 0.]])
+
     See also
     --------
     FeatureHasher : performs vectorization using only a hash function.
@@ -98,6 +121,32 @@ def __init__(self, *, dtype=np.float64, separator="=", sparse=True,
         self.sparse = sparse
         self.sort = sort
 
+    def _add_element(self, f, v, feature_names, vocab, fitting=True,
+                     transforming=False, indices=None, values=None):
+        if isinstance(v, str):
+            feature_name = "%s%s%s" % (f, self.separator, v)
+            v = 1
+        elif isinstance(v, Number) or (v is None):
+            feature_name = f
+        elif isinstance(v, Iterable) and not isinstance(v, str):
+            for vv in v:
+                self._add_element(f, vv, feature_names, vocab,
+                                  fitting, transforming, indices, values)
+            return
+        else:
+            raise ValueError(
+                'Unsupported Value Type %s for {%s: %s}' % (type(v), f, v))
+
+        if fitting:
+            if feature_name not in vocab:
+                vocab[feature_name] = len(feature_names)
+                feature_names.append(feature_name)
+
+        if transforming:
+            if feature_name in vocab:
+                indices.append(vocab[feature_name])
+                values.append(self.dtype(v))
+
     def fit(self, X, y=None):
         """Learn a list of feature name -> indices mappings.
 
@@ -117,11 +166,7 @@ def fit(self, X, y=None):
 
         for x in X:
             for f, v in x.items():
-                if isinstance(v, str):
-                    f = "%s%s%s" % (f, self.separator, v)
-                if f not in vocab:
-                    feature_names.append(f)
-                    vocab[f] = len(vocab)
+                self._add_element(f, v, feature_names, vocab)
 
         if self.sort:
             feature_names.sort()
@@ -150,6 +195,8 @@ def _transform(self, X, fitting):
             feature_names = self.feature_names_
             vocab = self.vocabulary_
 
+        transforming = True
+
         # Process everything as sparse regardless of setting
         X = [X] if isinstance(X, Mapping) else X
 
@@ -163,18 +210,8 @@ def _transform(self, X, fitting):
         # same time
         for x in X:
             for f, v in x.items():
-                if isinstance(v, str):
-                    f = "%s%s%s" % (f, self.separator, v)
-                    v = 1
-                if f in vocab:
-                    indices.append(vocab[f])
-                    values.append(dtype(v))
-                else:
-                    if fitting:
-                        feature_names.append(f)
-                        vocab[f] = len(vocab)
-                        indices.append(vocab[f])
-                        values.append(dtype(v))
+                self._add_element(f, v, feature_names, vocab,
+                                  fitting, transforming, indices, values)
 
             indptr.append(len(indices))
 

sklearn/feature_extraction/tests/test_dict_vectorizer.py

@@ -76,6 +76,23 @@ def test_one_of_k():
     assert "version" not in names
 
 
+def test_iterable_value():
+    D_in = [{"version": ["1", "2"], "ham": 2},
+            {"version": "2", "spam": .3},
+            {"version=3": True, "spam": -1}]
+    v = DictVectorizer()
+    X = v.fit_transform(D_in)
+    assert X.shape == (3, 5)
+
+    D_out = v.inverse_transform(X)
+    assert D_out[0] == {"version=1": 1, "version=2": 1, "ham": 2}
+
+    names = v.get_feature_names()
+    assert "version=2" in names
+    assert "version=1" in names
+    assert "version" not in names
+
+
 def test_unseen_or_no_features():
     D = [{"camelot": 0, "spamalot": 1}]
     for sparse in [True, False]: