test

.gitignore

@@ -61,4 +61,6 @@ examples/metastore_db
 
 examples/*.csv
 
-.idea/
+.idea/
+
+.pytest_cache

elephas/hyperparam.py

@@ -11,9 +11,11 @@
 
 
 class HyperParamModel(object):
-    '''
-    HyperParamModel
-    '''
+    """HyperParamModel
+
+    Computes distributed hyper-parameter optimization using Hyperas and
+    Spark.
+    """
     def __init__(self, sc, num_workers=4):
         self.spark_context = sc
         self.num_workers = num_workers
@@ -26,7 +28,7 @@ def compute_trials(self, model, data, max_evals):
         hyperas_worker = HyperasWorker(bc_model, bc_max_evals)
         dummy_rdd = self.spark_context.parallelize([i for i in range(1, 1000)])
         dummy_rdd = dummy_rdd.repartition(self.num_workers)
-        trials_list = dummy_rdd.mapPartitions(hyperas_worker.minimize).collect()
+        trials_list = dummy_rdd.mapPartitions(hyperas_worker._minimize).collect()
 
         return trials_list
 
@@ -72,11 +74,15 @@ def best_models(self, nb_models, model, data, max_evals):
 
 
 class HyperasWorker(object):
+    """ HyperasWorker
+
+    Executes hyper-parameter search on each worker and returns results.
+    """
     def __init__(self, bc_model, bc_max_evals):
         self.model_string = bc_model.value
         self.max_evals = bc_max_evals.value
 
-    def minimize(self, dummy_iterator):
+    def _minimize(self, dummy_iterator):
         trials = Trials()
         algo = rand.suggest
 

examples/hyperparam_optimization.py

@@ -1,22 +1,21 @@
 from __future__ import print_function
 from __future__ import absolute_import
-from hyperopt import Trials, STATUS_OK, tpe
 
-from hyperas import optim
+from pyspark import SparkContext, SparkConf
+
+from hyperopt import STATUS_OK
 from hyperas.distributions import choice, uniform
 
 from elephas.hyperparam import HyperParamModel
 
-from pyspark import SparkContext, SparkConf
 
 def data():
-    '''
-    Data providing function:
+    """Data providing function:
 
     Make sure to have every relevant import statement included here and return data as
     used in model function below. This function is separated from model() so that hyperopt
     won't reload data for each evaluation run.
-    '''
+    """
     from keras.datasets import mnist
     from keras.utils import np_utils
     (X_train, y_train), (X_test, y_test) = mnist.load_data()
@@ -33,16 +32,15 @@ def data():
 
 
 def model(X_train, Y_train, X_test, Y_test):
-    '''
-    Model providing function:
+    """Model providing function:
 
     Create Keras model with double curly brackets dropped-in as needed.
     Return value has to be a valid python dictionary with two customary keys:
         - loss: Specify a numeric evaluation metric to be minimized
         - status: Just use STATUS_OK and see hyperopt documentation if not feasible
     The last one is optional, though recommended, namely:
         - model: specify the model just created so that we can later use it again.
-    '''
+    """
     from keras.models import Sequential
     from keras.layers.core import Dense, Dropout, Activation
     from keras.optimizers import RMSprop