Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

WIP fix Naive Bayes coefficient stuff #2250

Open
wants to merge 3 commits into from

3 participants

Lars Andreas Mueller Olivier Grisel
Lars
Owner

This fixes #2237 and #2240.

Not yet MRG and 0.14-rc because I'm not sure if there's a good way to preserve backward compat and some more refactoring needs to be done.

sklearn/naive_bayes.py
@@ -48,7 +49,7 @@ def _joint_log_likelihood(self, X):
predict_proba and predict_log_proba.
"""
- def predict(self, X):
+ def _predict(self, X):
Olivier Grisel Owner
ogrisel added a note

This method is no longer used at all?

Lars Owner

I thought I'd removed that. It moved to GaussianNB, while the other two use LinearClassifierMixin.predict which is faster in the binary case.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Olivier Grisel ogrisel commented on the diff
sklearn/naive_bayes.py
@@ -340,21 +365,8 @@ def fit(self, X, y, sample_weight=None, class_prior=None):
self._update_class_log_prior(class_prior=class_prior)
Olivier Grisel Owner
ogrisel added a note

I think those two functions should not be collapsed into one private method, eg. _update_smoothed_params or similar.

Lars Owner

Indeed. The calling order matters now, so they should be a single _update_parameters.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Andreas Mueller
Owner

What is the status of this? It looks kinda old...

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jul 29, 2013
  1. Lars

    ENH multinomialnb linear model

    larsmans authored larsmans committed
  2. Lars

    WIP

    larsmans authored larsmans committed
  3. Lars

    WIP

    larsmans authored
This page is out of date. Refresh to see the latest.
Showing with 73 additions and 55 deletions.
  1. +72 −53 sklearn/naive_bayes.py
  2. +1 −2  sklearn/tests/test_naive_bayes.py
125 sklearn/naive_bayes.py
View
@@ -22,6 +22,7 @@
import warnings
from .base import BaseEstimator, ClassifierMixin
+from .linear_model.base import LinearClassifierMixin
from .preprocessing import binarize
from .preprocessing import LabelBinarizer
from .preprocessing import label_binarize
@@ -47,22 +48,6 @@ def _joint_log_likelihood(self, X):
predict_proba and predict_log_proba.
"""
- def predict(self, X):
- """
- Perform classification on an array of test vectors X.
-
- Parameters
- ----------
- X : array-like, shape = [n_samples, n_features]
-
- Returns
- -------
- C : array, shape = [n_samples]
- Predicted target values for X
- """
- jll = self._joint_log_likelihood(X)
- return self.classes_[np.argmax(jll, axis=1)]
-
def predict_log_proba(self, X):
"""
Return log-probability estimates for the test vector X.
@@ -172,6 +157,22 @@ def fit(self, X, y):
self.class_prior_[i] = np.float(np.sum(y == y_i)) / n_samples
return self
+ def predict(self, X):
+ """
+ Perform classification on an array of test vectors X.
+
+ Parameters
+ ----------
+ X : array-like, shape = [n_samples, n_features]
+
+ Returns
+ -------
+ C : array, shape = [n_samples]
+ Predicted target values for X
+ """
+ jll = self._joint_log_likelihood(X)
+ return self.classes_[np.argmax(jll, axis=1)]
+
def _joint_log_likelihood(self, X):
X = array2d(X)
joint_log_likelihood = []
@@ -201,13 +202,21 @@ def _update_class_log_prior(self, class_prior=None):
if len(class_prior) != n_classes:
raise ValueError("Number of priors must match number of"
" classes.")
- self.class_log_prior_ = np.log(class_prior)
+ class_log_prior = np.log(class_prior)
elif self.fit_prior:
# empirical prior, with sample_weight taken into account
- self.class_log_prior_ = (np.log(self.class_count_)
- - np.log(self.class_count_.sum()))
+ class_log_prior = (np.log(self.class_count_)
+ - np.log(self.class_count_.sum()))
+ else:
+ class_log_prior = np.zeros(n_classes) - np.log(n_classes)
+
+ self.class_log_prior_ = class_log_prior
+ if n_classes == 2:
+ # Set intercept_ to the log-odds ratio to make decision_function
+ # work out.
+ self.intercept_ = class_log_prior[1:] - class_log_prior[0]
else:
- self.class_log_prior_ = np.zeros(n_classes) - np.log(n_classes)
+ self.intercept_ = class_log_prior
def partial_fit(self, X, y, classes=None, sample_weight=None):
"""Incremental fit on a batch of samples.
@@ -342,21 +351,8 @@ def fit(self, X, y, sample_weight=None, class_prior=None):
self._update_class_log_prior(class_prior=class_prior)
Olivier Grisel Owner
ogrisel added a note

I think those two functions should not be collapsed into one private method, eg. _update_smoothed_params or similar.

Lars Owner

Indeed. The calling order matters now, so they should be a single _update_parameters.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
return self
- # XXX The following is a stopgap measure; we need to set the dimensions
- # of class_log_prior_ and feature_log_prob_ correctly.
- def _get_coef(self):
- return (self.feature_log_prob_[1:]
- if len(self.classes_) == 2 else self.feature_log_prob_)
- def _get_intercept(self):
- return (self.class_log_prior_[1:]
- if len(self.classes_) == 2 else self.class_log_prior_)
-
- coef_ = property(_get_coef)
- intercept_ = property(_get_intercept)
-
-
-class MultinomialNB(BaseDiscreteNB):
+class MultinomialNB(BaseDiscreteNB, LinearClassifierMixin):
"""
Naive Bayes classifier for multinomial models
@@ -381,20 +377,24 @@ class MultinomialNB(BaseDiscreteNB):
Attributes
----------
- `class_log_prior_` : array, shape (n_classes, )
- Smoothed empirical log probability for each class.
+ `class_log_prior_` : array, shape = (n_classes,)
+ Smoothed empirical log probability for each class. DEPRECATED; will be
+ removed in 0.16. Use intercept_ instead.
- `intercept_` : property
- Mirrors ``class_log_prior_`` for interpreting MultinomialNB
- as a linear model.
+ `intercept_` : array, shape = (n_classes,) or (1,)
+ Intercept for naive Bayes considered as a linear model. This is the
+ log of the class prior P(y) in the multiclass case; in the binary case,
+ this is the log-odds ratio of the positive and negative priors.
- `feature_log_prob_`: array, shape (n_classes, n_features)
- Empirical log probability of features
- given a class, ``P(x_i|y)``.
+ `feature_log_prob_`, `coef_` : array, shape = [n_classes, n_features]
+ Empirical log probability of features given a class, P(x_i|y).
+ DEPRECATED; will be removed in 0.16. Use coef_ instead.
- `coef_` : property
- Mirrors ``feature_log_prob_`` for interpreting MultinomialNB
- as a linear model.
+ `coef_` : array, shape = (n_classes, n_features) or (1, n_features)
+ Coefficients of naive Bayes as a linear model. This is a matrix of
+ feature log probabilities log P(x_i|y) in the multiclass case; in the
+ binary case, this is the log-odds ratio of the positive and negative
+ feature probabilities.
`class_count_` : array, shape (n_classes,)
Number of samples encountered for each class during fitting. This
@@ -419,9 +419,9 @@ class MultinomialNB(BaseDiscreteNB):
Notes
-----
- For the rationale behind the names `coef_` and `intercept_`, i.e.
- naive Bayes as a linear classifier, see J. Rennie et al. (2003),
- Tackling the poor assumptions of naive Bayes text classifiers, ICML.
+ For the rationale behind `coef_` and `intercept_`, i.e. naive Bayes
+ as a linear classifier, see J. Rennie et al. (2003), Tackling the poor
+ assumptions of naive Bayes text classifiers, ICML.
References
----------
@@ -448,8 +448,15 @@ def _update_feature_log_prob(self):
smoothed_fc = self.feature_count_ + self.alpha
smoothed_cc = smoothed_fc.sum(axis=1)
- self.feature_log_prob_ = (np.log(smoothed_fc)
- - np.log(smoothed_cc.reshape(-1, 1)))
+ feature_log_prob = (np.log(smoothed_fc)
+ - np.log(smoothed_cc.reshape(-1, 1)))
+
+ self.feature_log_prob_ = feature_log_prob
+ n_classes = len(self.classes_)
+ if n_classes == 2:
+ self.coef_ = feature_log_prob[1:] - feature_log_prob[0]
+ else:
+ self.coef_ = feature_log_prob
def _joint_log_likelihood(self, X):
"""Calculate the posterior log probability of the samples X"""
@@ -458,7 +465,7 @@ def _joint_log_likelihood(self, X):
+ self.class_log_prior_)
-class BernoulliNB(BaseDiscreteNB):
+class BernoulliNB(BaseDiscreteNB, LinearClassifierMixin):
"""Naive Bayes classifier for multivariate Bernoulli models.
Like MultinomialNB, this classifier is suitable for discrete data. The
@@ -547,8 +554,17 @@ def _update_feature_log_prob(self):
smoothed_fc = self.feature_count_ + self.alpha
smoothed_cc = self.class_count_ + self.alpha * n_classes
- self.feature_log_prob_ = (np.log(smoothed_fc)
- - np.log(smoothed_cc.reshape(-1, 1)))
+ feature_log_prob = (np.log(smoothed_fc)
+ - np.log(smoothed_cc.reshape(-1, 1)))
+
+ neg_log_prob = np.log(1 - np.exp(feature_log_prob))
+ feature_log_prob -= neg_log_prob
+
+ self.feature_log_prob_ = feature_log_prob
+ if n_classes == 2:
+ self.coef_ = feature_log_prob[1:]
+ else:
+ self.coef_ = feature_log_prob
def _joint_log_likelihood(self, X):
"""Calculate the posterior log probability of the samples X"""
@@ -571,3 +587,6 @@ def _joint_log_likelihood(self, X):
jll += self.class_log_prior_ + neg_prob.sum(axis=1)
return jll
+
+ def decision_function(self, X):
+ return self._joint_log_likelihood(X)
3  sklearn/tests/test_naive_bayes.py
View
@@ -308,8 +308,7 @@ def test_sample_weight_mnb():
[0, 0, 1],
sample_weight=[1, 1, 4])
assert_array_equal(clf.predict([1, 0]), [1])
- positive_prior = np.exp(clf.intercept_[0])
- assert_array_almost_equal([1 - positive_prior, positive_prior],
+ assert_array_almost_equal(np.exp(clf.class_log_prior_),
[1 / 3., 2 / 3.])
Something went wrong with that request. Please try again.