Merge pull request #16 from aigamedev/serialization

Support for cross-platform serialization of MLP.

sknn/mlp.py

@@ -147,7 +147,10 @@ def __init__(
         self.verbose = verbose
 
         self.unit_counts = None
+        self.input_space = None
         self.mlp = None
+        self.weights = None
+        self.vs = None
         self.ds = None
         self.trainer = None
         self.f = None
@@ -275,7 +278,7 @@ def _create_output_layer(self, name, args):
         raise NotImplementedError(
             "Output layer type `%s` is not implemented." % activation_type)
 
-    def _create_mlp(self, X, y, nvis=None, input_space=None):
+    def _create_mlp(self):
         # Create the layers one by one, connecting to previous.
         mlp_layers = []
         for i, layer in enumerate(self.layers[:-1]):
@@ -286,7 +289,7 @@ def _create_mlp(self, X, y, nvis=None, input_space=None):
             if layer[0] == "Tanh":
                lim *= 1.1 * lim
             elif layer[0] in ("Rectifier", "Maxout", "Convolution"):
-                #  He, Rang, Zhen and Sun, converted to uniform.
+                # He, Rang, Zhen and Sun, converted to uniform.
                lim *= numpy.sqrt(2)
             elif layer[0] == "Sigmoid":
                 lim *= 4
@@ -303,11 +306,18 @@ def _create_mlp(self, X, y, nvis=None, input_space=None):
         output_layer = self._create_output_layer(output_layer_name, output_layer_info)
         mlp_layers.append(output_layer)
 
-        return mlp.MLP(
+        self.mlp = mlp.MLP(
             mlp_layers,
-            nvis=nvis,
+            nvis=None if self.is_convolution else self.unit_counts[0],
             seed=self.random_state,
-            input_space=input_space)
+            input_space=self.input_space)
+
+        if self.weights is not None:
+            self._array_to_mlp(self.weights, self.mlp)
+            self.weights = None
+
+        inputs = self.mlp.get_input_space().make_theano_batch()
+        self.f = theano.function([inputs], self.mlp.fprop(inputs))
 
     def _create_matrix_input(self, X, y):
         if self.is_convolution:
@@ -348,27 +358,24 @@ def _initialize(self, X, y):
         self.train_set = X, y
 
         # Convolution networks need a custom input space.
-        self.ds, input_space = self._create_matrix_input(X, y)
+        self.ds, self.input_space = self._create_matrix_input(X, y)
         if self.valid_set:
             X_v, y_v = self.valid_set
             self.vs, _ = self._create_matrix_input(X_v, y_v)
         else:
             self.vs = None
 
-        if self.mlp is None:
-            nvis = None if self.is_convolution else self.unit_counts[0]
-            self.mlp = self._create_mlp(X, y, input_space=input_space, nvis=nvis)
+        self._create_mlp()
 
         self.trainer = self._create_trainer(self.vs)
         self.trainer.setup(self.mlp, self.ds)
-        inputs = self.mlp.get_input_space().make_theano_batch()
-        self.f = theano.function([inputs], self.mlp.fprop(inputs))
+
 
     @property
     def is_initialized(self):
         """Check if the neural network was setup already.
         """
-        return not (self.ds is None or self.trainer is None or self.f is None)
+        return not (self.mlp is None or self.f is None)
 
     @property
     def is_convolution(self):
@@ -381,16 +388,29 @@ def __getstate__(self):
             "The neural network has not been initialized."
 
         d = self.__dict__.copy()
-        for k in ['ds', 'f', 'trainer']:
+        d['weights'] = self._mlp_to_array()
+
+        for k in ['ds', 'vs', 'f', 'trainer', 'mlp']:
             if k in d:
                 del d[k]
         return d
 
+    def _mlp_to_array(self):
+        return [(l.get_weights(), l.get_biases()) for l in self.mlp.layers]
+
     def __setstate__(self, d):
         self.__dict__.update(d)
-
-        for k in ['ds', 'f', 'trainer']:
+        for k in ['ds', 'vs', 'f', 'trainer', 'mlp']:
             setattr(self, k, None)
+        self._create_mlp()
+
+    def _array_to_mlp(self, array, nn):
+        for layer, (weights, biases) in zip(nn.layers, array):
+            assert layer.get_weights().shape == weights.shape
+            layer.set_weights(weights)
+
+            assert layer.get_biases().shape == biases.shape
+            layer.set_biases(biases)
 
     def _fit(self, X, y, test=None):
         assert X.shape[0] == y.shape[0],\
@@ -405,14 +425,13 @@ def _fit(self, X, y, test=None):
             y = y.toarray()
 
         if not self.is_initialized:            
-            self._initialize(X, y)            
+            self._initialize(X, y)
             X, y = self.train_set
         else:
             self.train_set = X, y
 
         if self.is_convolution:
             X = self.ds.view_converter.topo_view_to_design_mat(X)
-
         self.ds.X, self.ds.y = X, y
 
         # Bug in PyLearn2 that has some unicode channels, can't sort.
@@ -475,7 +494,6 @@ def _predict(self, X):
         if not isinstance(X, numpy.ndarray):
             X = X.toarray()
 
-
         return self.f(X)
 
 

sknn/tests/test_deep.py

@@ -1,9 +1,10 @@
 import unittest
-from nose.tools import (assert_is_not_none, assert_raises, assert_equal)
+from nose.tools import (assert_false, assert_raises, assert_true, assert_equal)
 
 import io
 import pickle
 import numpy
+from sklearn.base import clone
 
 from sknn.mlp import MultiLayerPerceptronRegressor as MLPR
 from . import test_linear
@@ -32,3 +33,47 @@ def test_UnknownHiddenActivation(self):
         assert_raises(NotImplementedError, nn.fit, a_in, a_in)
 
     # This class also runs all the tests from the linear network too.
+
+
+class TestDeepDeterminism(unittest.TestCase):
+
+    def setUp(self):
+        self.a_in = numpy.random.uniform(0.0, 1.0, (8,16))
+        self.a_out = numpy.zeros((8,1))
+
+    def run_EqualityTest(self, copier, asserter):
+        for activation in ["Rectifier", "Sigmoid", "Maxout", "Tanh"]:
+            nn1 = MLPR(layers=[(activation, 16, 2), ("Linear", 8)], random_state=1234)
+            nn1._initialize(self.a_in, self.a_out)
+
+            nn2 = copier(nn1, activation)
+            asserter(numpy.all(nn1.predict(self.a_in) == nn2.predict(self.a_in)))
+
+    def test_DifferentSeedPredictNotEquals(self):
+        def ctor(_, activation):
+            nn = MLPR(layers=[(activation, 16, 2), ("Linear", 8)], random_state=2345)
+            nn._initialize(self.a_in, self.a_out)
+            return nn
+        self.run_EqualityTest(ctor, assert_false)
+
+    def test_SameSeedPredictEquals(self):
+        def ctor(_, activation):
+            nn = MLPR(layers=[(activation, 16, 2), ("Linear", 8)], random_state=1234)
+            nn._initialize(self.a_in, self.a_out)
+            return nn
+        self.run_EqualityTest(ctor, assert_true)
+
+    def test_ClonePredictEquals(self):
+        def cloner(nn, _):
+            cc = clone(nn)
+            cc._initialize(self.a_in, self.a_out)
+            return cc
+        self.run_EqualityTest(cloner, assert_true)
+
+    def test_SerializedPredictEquals(self):
+        def serialize(nn, _):
+            buf = io.BytesIO()
+            pickle.dump(nn, buf)
+            buf.seek(0)
+            return pickle.load(buf)
+        self.run_EqualityTest(serialize, assert_true)

sknn/tests/test_linear.py

@@ -22,7 +22,7 @@ def test_PredictUninitialized(self):
         assert_raises(ValueError, self.nn.predict, a_in)
 
     def test_FitAutoInitialize(self):
-        a_in, a_out = numpy.zeros((8,16)), numpy.zeros((8,1))
+        a_in, a_out = numpy.zeros((8,16)), numpy.zeros((8,4))
         self.nn.fit(a_in, a_out)
         assert_true(self.nn.is_initialized)
 
@@ -64,25 +64,33 @@ def test_SerializeCorrect(self):
         assert_equal(nn.layers, self.nn.layers)
 
 
-"""
 class TestSerializedNetwork(TestLinearNetwork):
 
     def setUp(self):
         self.original = MLPR(layers=[("Linear",)])
         a_in, a_out = numpy.zeros((8,16)), numpy.zeros((8,4))
-        self.original.initialize(a_in, a_out)
+        self.original._initialize(a_in, a_out)
 
         buf = io.BytesIO()
         pickle.dump(self.original, buf)
         buf.seek(0)
         self.nn = pickle.load(buf)
 
+    def test_TypeOfWeightsArray(self):
+        for w, b in self.nn._mlp_to_array():
+            assert_equal(type(w), numpy.ndarray)
+            assert_equal(type(b), numpy.ndarray)
+
+    def test_FitAutoInitialize(self):
+        # Override base class test, you currently can't re-train a network that
+        # was serialized and deserialized.
+        pass
+
     def test_PredictUninitialized(self):
         # Override base class test, this is not initialized but it
         # should be able to predict without throwing assert.
-        assert_false(self.nn.is_initialized)
+        assert_true(self.nn.is_initialized)
 
     def test_PredictAlreadyInitialized(self):
         a_in = numpy.zeros((8,16))
         self.nn.predict(a_in)
-"""

sknn/tests/test_output.py

@@ -15,4 +15,4 @@ def setUp(self):
 class TestSoftmaxOutput(test_linear.TestLinearNetwork):
 
     def setUp(self):
-        self.nn = MLPC(layers=[("Softmax",)], n_iter=1)
+        self.nn = MLPR(layers=[("Softmax",)], n_iter=1)

sknn/tests/test_pipeline.py

@@ -32,7 +32,6 @@ def test_ScalerThenNeuralNetwork(self):
         self._run(pipeline)
 
 
-"""
 class TestSerializedPipeline(TestPipeline):
 
     def _run(self, pipeline):
@@ -47,4 +46,3 @@ def _run(self, pipeline):
         p = pickle.load(buf)
 
         assert_true((a_test == p.predict(a_in)).all())
-"""