-
-
Notifications
You must be signed in to change notification settings - Fork 4.4k
/
text2bow.py
122 lines (96 loc) · 3.99 KB
/
text2bow.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011 Radim Rehurek <radimrehurek@seznam.cz>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
"""Scikit learn interface for :class:`~gensim.corpora.dictionary.Dictionary`.
Follows scikit-learn API conventions to facilitate using gensim along with scikit-learn.
Examples
--------
.. sourcecode:: pycon
>>> from gensim.sklearn_api import Text2BowTransformer
>>>
>>> # Get a corpus as an iterable of unicode strings.
>>> texts = [u'complier system computer', u'loading computer system']
>>>
>>> # Create a transformer..
>>> model = Text2BowTransformer()
>>>
>>> # Use sklearn-style `fit_transform` to get the BOW representation of each document.
>>> model.fit_transform(texts)
[[(0, 1), (1, 1), (2, 1)], [(1, 1), (2, 1), (3, 1)]]
"""
from six import string_types
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.exceptions import NotFittedError
from gensim.corpora import Dictionary
from gensim.utils import tokenize
class Text2BowTransformer(TransformerMixin, BaseEstimator):
"""Base Text2Bow module , wraps :class:`~gensim.corpora.dictionary.Dictionary`.
For more information please have a look to `Bag-of-words model <https://en.wikipedia.org/wiki/Bag-of-words_model>`_.
"""
def __init__(self, prune_at=2000000, tokenizer=tokenize):
"""
Parameters
----------
prune_at : int, optional
Total number of unique words. Dictionary will keep not more than `prune_at` words.
tokenizer : callable (str -> list of str), optional
A callable to split a document into a list of each terms, default is :func:`gensim.utils.tokenize`.
"""
self.gensim_model = None
self.prune_at = prune_at
self.tokenizer = tokenizer
def fit(self, X, y=None):
"""Fit the model according to the given training data.
Parameters
----------
X : iterable of str
A collection of documents used for training the model.
Returns
-------
:class:`~gensim.sklearn_api.text2bow.Text2BowTransformer`
The trained model.
"""
tokenized_docs = [list(self.tokenizer(x)) for x in X]
self.gensim_model = Dictionary(documents=tokenized_docs, prune_at=self.prune_at)
return self
def transform(self, docs):
"""Get the BOW format for the `docs`.
Parameters
----------
docs : {iterable of str, str}
A collection of documents to be transformed.
Returns
-------
iterable of list (int, int) 2-tuples.
The BOW representation of each document.
"""
if self.gensim_model is None:
raise NotFittedError(
"This model has not been fitted yet. Call 'fit' with appropriate arguments before using this method."
)
# input as python lists
if isinstance(docs, string_types):
docs = [docs]
tokenized_docs = (list(self.tokenizer(doc)) for doc in docs)
return [self.gensim_model.doc2bow(doc) for doc in tokenized_docs]
def partial_fit(self, X):
"""Train model over a potentially incomplete set of documents.
This method can be used in two ways:
1. On an unfitted model in which case the dictionary is initialized and trained on `X`.
2. On an already fitted model in which case the dictionary is **expanded** by `X`.
Parameters
----------
X : iterable of str
A collection of documents used to train the model.
Returns
-------
:class:`~gensim.sklearn_api.text2bow.Text2BowTransformer`
The trained model.
"""
if self.gensim_model is None:
self.gensim_model = Dictionary(prune_at=self.prune_at)
tokenized_docs = [list(self.tokenizer(x)) for x in X]
self.gensim_model.add_documents(tokenized_docs)
return self