ENH add xavier init to MLP

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.0.0] - unreleased
+
+### Changed
+
+- Moved the MLPClassifier and MLPRegressor to using
+  [Xavier initialization](https://www.tensorflow.org/api_docs/python/tf/contrib/layers/xavier_initializer) (#68). 
+
 ## [1.2.0] - 2017-09-21
 
 ### Added

muffnn/mlp/base.py

@@ -144,7 +144,7 @@ def partial_fit(self, X, y, monitor=None, **kwargs):
                     self._random_state.randint(0, 10000000))
 
                 tf.get_variable_scope().set_initializer(
-                    tf.uniform_unit_scaling_initializer(self.init_scale))
+                    tf.contrib.layers.xavier_initializer())
 
                 self._build_tf_graph()
 
@@ -208,7 +208,6 @@ def __getstate__(self):
                           batch_size=self.batch_size,
                           keep_prob=self.keep_prob,
                           hidden_units=self.hidden_units,
-                          init_scale=self.init_scale,
                           random_state=self.random_state,
                           n_epochs=self.n_epochs,
                           solver=self.solver,

muffnn/mlp/mlp_classifier.py

@@ -44,11 +44,6 @@ class MLPClassifier(MLPBaseEstimator, ClassifierMixin):
         <https://www.tensorflow.org/versions/r0.11/api_docs/python/nn.html#dropout>`
     activation : callable, optional
         The activation function.  See tensorflow.python.ops.nn.
-    init_scale : float, optional
-        The scale for the initialization function.  The TF default initializer,
-        `uniform_unit_scaling_initializer
-        <https://www.tensorflow.org/versions/r0.8/api_docs/python/state_ops.html#uniform_unit_scaling_initializer>`,
-        is used.
     random_state : int, RandomState instance or None, optional
         If int, the random number generator seed. If RandomState instance,
         the random number generator itself. If None, then `np.random` will be
@@ -86,15 +81,14 @@ class MLPClassifier(MLPBaseEstimator, ClassifierMixin):
     """
 
     def __init__(self, hidden_units=(256,), batch_size=64, n_epochs=5,
-                 keep_prob=1.0, activation=nn.relu, init_scale=0.1,
+                 keep_prob=1.0, activation=nn.relu,
                  random_state=None, solver=tf.train.AdamOptimizer,
                  solver_kwargs=None, transform_layer_index=None):
         self.hidden_units = hidden_units
         self.batch_size = batch_size
         self.n_epochs = n_epochs
         self.keep_prob = keep_prob
         self.activation = activation
-        self.init_scale = init_scale
         self.random_state = random_state
         self.solver = solver
         self.solver_kwargs = solver_kwargs

muffnn/mlp/mlp_regressor.py

@@ -43,11 +43,6 @@ class MLPRegressor(MLPBaseEstimator, RegressorMixin):
         <https://www.tensorflow.org/versions/r0.11/api_docs/python/nn.html#dropout>`
     activation : callable, optional
         The activation function.  See tensorflow.python.ops.nn.
-    init_scale : float, optional
-        The scale for the initialization function.  The TF default initializer,
-        `uniform_unit_scaling_initializer
-        <https://www.tensorflow.org/versions/r0.8/api_docs/python/state_ops.html#uniform_unit_scaling_initializer>`,
-        is used.
     random_state : int, RandomState instance or None, optional
         If int, the random number generator seed. If RandomState instance,
         the random number generator itself. If None, then `np.random` will be
@@ -87,15 +82,14 @@ class MLPRegressor(MLPBaseEstimator, RegressorMixin):
     """
 
     def __init__(self, hidden_units=(256,), batch_size=64, n_epochs=5,
-                 keep_prob=1.0, activation=nn.relu, init_scale=0.1,
+                 keep_prob=1.0, activation=nn.relu,
                  random_state=None, solver=tf.train.AdamOptimizer,
                  solver_kwargs=None, transform_layer_index=None):
         self.hidden_units = hidden_units
         self.batch_size = batch_size
         self.n_epochs = n_epochs
         self.keep_prob = keep_prob
         self.activation = activation
-        self.init_scale = init_scale
         self.random_state = random_state
         self.solver = solver
         self.solver_kwargs = solver_kwargs

muffnn/mlp/tests/test_base.py

@@ -28,7 +28,7 @@ class SimpleTestEstimator(base.MLPBaseEstimator):
     _keep_prob = 1.0
 
     def __init__(self, hidden_units=(256,), batch_size=64, n_epochs=5,
-                 keep_prob=1.0, activation=nn.relu, init_scale=0.1,
+                 keep_prob=1.0, activation=nn.relu,
                  random_state=None, monitor=None,
                  solver=tf.train.AdamOptimizer, solver_kwargs=None,
                  transform_layer_index=None):
@@ -37,7 +37,6 @@ def __init__(self, hidden_units=(256,), batch_size=64, n_epochs=5,
         self.n_epochs = n_epochs
         self.keep_prob = keep_prob
         self.activation = activation
-        self.init_scale = init_scale
         self.random_state = random_state
         self.monitor = monitor
         self.solver = solver

muffnn/mlp/tests/test_mlp_classifier.py

@@ -57,11 +57,11 @@
 class MLPClassifierManyEpochs(MLPClassifier):
 
     def __init__(self, hidden_units=(256,), batch_size=64,
-                 keep_prob=1.0, activation=nn.relu, init_scale=0.1):
+                 keep_prob=1.0, activation=nn.relu):
         super(MLPClassifierManyEpochs, self).__init__(
             hidden_units=hidden_units, batch_size=batch_size,
             n_epochs=100, keep_prob=keep_prob,
-            activation=activation, init_scale=init_scale,
+            activation=activation,
             random_state=42)
 
     def predict_proba(self, *args, **kwargs):

muffnn/mlp/tests/test_mlp_regressor.py

@@ -45,11 +45,11 @@ def check_predictions(est, X, y):
 # the toy example tests that have only a handful of examples.
 class MLPRegressorManyEpochs(MLPRegressor):
     def __init__(self, hidden_units=(256,), batch_size=64,
-                 keep_prob=1.0, activation=nn.relu, init_scale=0.1):
+                 keep_prob=1.0, activation=nn.relu):
         super(MLPRegressorManyEpochs, self).__init__(
             hidden_units=hidden_units, batch_size=batch_size,
             n_epochs=100, keep_prob=keep_prob,
-            activation=activation, init_scale=init_scale,
+            activation=activation,
             random_state=42)