ENH add Naive Bayes Metaestimator ColumnwiseNB (aka "GeneralNB")

doc/modules/classes.rst

@@ -1316,6 +1316,7 @@ Visualization
    naive_bayes.ComplementNB
    naive_bayes.GaussianNB
    naive_bayes.MultinomialNB
+   naive_bayes.ColumnwiseNB
 
 
 .. _neighbors_ref:

doc/modules/naive_bayes.rst

@@ -281,3 +281,38 @@ For an overview of available strategies in scikit-learn, see also the
    The ``partial_fit`` method call of naive Bayes models introduces some
    computational overhead. It is recommended to use data chunk sizes that are as
    large as possible, that is as the available RAM allows.
+
+.. _columnwise_naive_bayes:
+
+Mix and match naive Bayes models
+--------------------------------
+
+A naive Bayes model that assumes different distribution families for different
+features (or subsets of features) can be constructed using :class:`ColumnwiseNB`.
+It is a meta-estimator, whose operation relies on naive Bayes
+sub-estimators, which can be chosen in any number or combination from
+:class:`GaussianNB`, :class:`MultinomialNB`, :class:`ComplementNB`,
+:class:`BernoulliNB`, :class:`CategoricalNB`, and user-defined models
+(provided they implement necessary methods).
+
+When creating a :class:`ColumnwiseNB` estimator, one specifies sub-estimators
+and their respective column subsets as a list of tuples.
+Each sub-estimator is fitted and evaluated independently of the
+others and "sees" only the features assigned to it. The predictions of sub-estimators are
+combined via
+
+.. math::
+
+   \log P(x,y)=\log P(x_{1},y) + \dots + \log P(x_{M},y) - (M - 1)\log P(y),
+
+where :math:`\log P(x,y)` is the joint log-probability predicted by the meta-estimator,
+:math:`\log P(x_{m},y)` is that by the :math:`m` th sub-estimator,
+:math:`\log P(y)` is the class prior used by the meta-estimator, and
+:math:`M\geq1` is the total number of sub-estimators.
+
+See :ref:`sphx_glr_auto_examples_miscellaneous_plot_combining_naive_bayes.py`
+for an example of a mixed naive Bayes model implementation.
+
+See also :ref:`voting_classifier` for a way of combining general classifiers.
+An introduction to processing datasets with heterogeneous features is available at
+:ref:`column_transformer`.

doc/whats_new/v1.3.rst

@@ -582,6 +582,10 @@ Changelog
 
 :mod:`sklearn.naive_bayes`
 ..........................
+- |Feature| Add :class:`naive_bayes.ColumnwiseNB`, a meta-estimator that allows
+  existing naive Bayes classifiers to be combined and applied to different columns
+  of `X`. :pr:`22574` by :user:`Andrey Melnik <avm19>`.
+
 
 - |Fix| :class:`naive_bayes.GaussianNB` does not raise anymore a `ZeroDivisionError`
   when the provided `sample_weight` reduces the problem to a single class in `fit`.

examples/miscellaneous/plot_combining_naive_bayes.py

@@ -0,0 +1,134 @@
+"""
+===================================================
+Combining Naive Bayes Estimators using ColumnwiseNB
+===================================================
+
+.. currentmodule:: sklearn
+
+This example shows how to use :class:`~naive_bayes.ColumnwiseNB`
+meta-estimator to construct a naive Bayes model from base naive Bayes
+estimators. The resulting model is applied to a dataset with a mixture of
+discrete and continuous features.
+
+We consider the titanic dataset, in which:
+
+- numerical (continous) features "age" and "fare" are handled by
+  :class:`~naive_bayes.GaussianNB`;
+- categorical (discrete) features "embarked", "sex", and "pclass" are handled
+  by :class:`~naive_bayes.CategoricalNB`.
+"""
+
+# Author: Andrey V. Melnik <andrey.melnik.maths@gmail.com>
+#         Pedro Morales <part.morales@gmail.com>
+#
+# License: BSD 3 clause
+
+# %%
+import pandas as pd
+from sklearn import set_config
+from sklearn.datasets import fetch_openml
+
+set_config(transform_output="pandas")
+
+X, y = fetch_openml(
+    "titanic", version=1, as_frame=True, return_X_y=True, n_retries=10, parser="auto"
+)
+X["pclass"] = X["pclass"].astype("category")
+# Add a category for NaNs to the "embarked" feature:
+X["embarked"] = X["embarked"].cat.add_categories("N/A").fillna("N/A")
+
+# %%
+# Build and use a pipeline around ``ColumnwiseNB``
+# ------------------------------------------------
+
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import OrdinalEncoder
+from sklearn.naive_bayes import GaussianNB, CategoricalNB, ColumnwiseNB
+from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.metrics import accuracy_score
+
+numeric_features = ["age", "fare"]
+numeric_transformer = SimpleImputer(strategy="median")
+
+categorical_features = ["embarked", "sex", "pclass"]
+categories = [X[c].unique().to_list() for c in X[categorical_features]]
+categorical_transformer = OrdinalEncoder(categories=categories)
+
+preprocessor = ColumnTransformer(
+    transformers=[
+        ("num", numeric_transformer, numeric_features),
+        ("cat", categorical_transformer, categorical_features),
+    ],
+    verbose_feature_names_out=False,
+)
+
+classifier = ColumnwiseNB(
+    nb_estimators=[
+        ("gnb", GaussianNB(), numeric_features),
+        ("cnb", CategoricalNB(), categorical_features),
+    ]
+)
+
+pipe = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", classifier)])
+pipe
+# %%
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
+
+pipe.fit(X_train, y_train)
+y_pred = pipe.predict(X_test)
+print(f"Test accuracy: {accuracy_score(y_test, y_pred)}")
+
+# %%
+# Compare choices of columns using ``GridSearchCV``
+# --------------------------------------------------
+#
+# The allocation of columns to constituent subestimators can be regarded as a
+# hyperparameter. We can explore the combinations of columns' choices and values
+# of other hyperparameters with the help of :class:`~.model_selection.GridSearchCV`.
+
+param_grid = {
+    "classifier__nb_estimators": [
+        [
+            ("gnb", GaussianNB(), ["age", "fare"]),
+            ("cnb", CategoricalNB(), categorical_features),
+        ],
+        [("gnb", GaussianNB(), []), ("cnb", CategoricalNB(), ["pclass"])],
+        [("gnb", GaussianNB(), ["embarked"]), ("cnb", CategoricalNB(), [])],
+    ],
+    "preprocessor__num__strategy": ["mean", "most_frequent"],
+}
+
+grid_search = GridSearchCV(pipe, param_grid, cv=10)
+grid_search
+
+# %%
+# Calling `fit` triggers the cross-validated search for the best
+# hyperparameters combination:
+
+grid_search.fit(X_train, y_train)
+
+print("Best params:")
+print(grid_search.best_params_)
+
+# %%
+# As it turns out, the best results are achieved by the naive Bayes model when "sex"
+# is the only feature used:
+
+cv_results = pd.DataFrame(grid_search.cv_results_)
+cv_results = cv_results.sort_values("mean_test_score", ascending=False)
+cv_results["Columns dictionary"] = cv_results["param_classifier__nb_estimators"].map(
+    lambda l: {e[0]: e[-1] for e in l}
+)
+cv_results["'gnb' columns"] = cv_results["Columns dictionary"].map(lambda d: d["gnb"])
+cv_results["'cnb' columns"] = cv_results["Columns dictionary"].map(lambda d: d["cnb"])
+cv_results[
+    [
+        "mean_test_score",
+        "std_test_score",
+        "param_preprocessor__num__strategy",
+        "'gnb' columns",
+        "'cnb' columns",
+    ]
+]