Merge branch 'custom-grid'

README.md

@@ -101,7 +101,7 @@ Contributions and especially bug reports are more than welcome.
 ## Citation
 
 ```bibtex
-@article{al2017learning,
+@article{alshedivat2017srk,
   title={Learning scalable deep kernels with recurrent structure},
   author={Al-Shedivat, Maruan and Wilson, Andrew Gordon and Saatchi, Yunus and Hu, Zhiting and Xing, Eric P},
   journal={Journal of Machine Learning Research},

examples/msgp_lstm_actuator.py

@@ -1,8 +1,11 @@
-"""
-MSGP-LSTM regression on Actuator data.
-"""
+"""MSGP-LSTM regression on Actuator data."""
+
+from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
+from six.moves import xrange
+
 import numpy as np
 np.random.seed(42)
 

examples/msgp_lstm_actuator_manual_grid.py

@@ -0,0 +1,119 @@
+"""MSGP-LSTM regression on Actuator data with manually specified grid."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from six.moves import xrange
+
+import numpy as np
+np.random.seed(42)
+
+# Keras
+from keras.optimizers import Adagrad, Adam, SGD, RMSprop
+from keras.callbacks import EarlyStopping
+
+# Dataset interfaces
+from kgp.datasets.sysid import load_data
+from kgp.datasets.data_utils import data_to_seq, standardize_data
+
+# Model assembling and executing
+from kgp.utils.assemble import load_NN_configs, load_GP_configs, assemble
+from kgp.utils.experiment import train
+
+# Metrics & losses
+from kgp.losses import gen_gp_loss
+from kgp.metrics import root_mean_squared_error as RMSE
+
+
+def main():
+    # Load data
+    X, y = load_data('actuator', use_targets=False)
+    X_seq, y_seq = data_to_seq(X, y,
+        t_lag=32, t_future_shift=1, t_future_steps=1, t_sw_step=1)
+
+    # Split
+    train_end = int((45. / 100.) * len(X_seq))
+    test_end = int((90. / 100.) * len(X_seq))
+    X_train, y_train = X_seq[:train_end], y_seq[:train_end]
+    X_test, y_test = X_seq[train_end:test_end], y_seq[train_end:test_end]
+    X_valid, y_valid = X_seq[test_end:], y_seq[test_end:]
+
+    data = {
+        'train': [X_train, y_train],
+        'valid': [X_valid, y_valid],
+        'test': [X_test, y_test],
+    }
+
+    # Re-format targets
+    for set_name in data:
+        y = data[set_name][1]
+        y = y.reshape((-1, 1, np.prod(y.shape[1:])))
+        data[set_name][1] = [y[:,:,i] for i in xrange(y.shape[2])]
+
+    # Model & training parameters
+    nb_train_samples = data['train'][0].shape[0]
+    input_shape = data['train'][0].shape[1:]
+    nb_outputs = len(data['train'][1])
+    gp_input_shape = (1,)
+    batch_size = 128
+    epochs = 5
+
+    nn_params = {
+        'H_dim': 16,
+        'H_activation': 'tanh',
+        'dropout': 0.1,
+    }
+    gp_params = {
+        'cov': 'SEiso',
+        'hyp_lik': -2.0,
+        'hyp_cov': [[-0.7], [0.0]],
+        'opt': {'cg_maxit': 500, 'cg_tol': 1e-4},
+        'grid_kwargs': {'eq': 1, 'k': 1e2},
+        'update_grid': False,  # when using manual grid, turn off grid updates
+    }
+
+    # Retrieve model config
+    nn_configs = load_NN_configs(filename='lstm.yaml',
+                                 input_shape=input_shape,
+                                 output_shape=gp_input_shape,
+                                 params=nn_params)
+    gp_configs = load_GP_configs(filename='gp.yaml',
+                                 nb_outputs=nb_outputs,
+                                 batch_size=batch_size,
+                                 nb_train_samples=nb_train_samples,
+                                 params=gp_params)
+
+    # Specify manual grid for MSGP (100 equidistant points per input dimension).
+    # Note: each np.ndarray in the xg must be a column vector.
+    gp_configs['MSGP']['config']['grid_kwargs']['xg'] = (
+            gp_input_shape[0] * [np.linspace(-1.0, 1.0, 100)[:, None]])
+
+    # Construct & compile the model
+    model = assemble('GP-LSTM', [nn_configs['1H'], gp_configs['MSGP']])
+    loss = [gen_gp_loss(gp) for gp in model.output_layers]
+    model.compile(optimizer=Adam(1e-2), loss=loss)
+
+    # Callbacks
+    callbacks = [EarlyStopping(monitor='val_mse', patience=10)]
+
+    # Train the model
+    history = train(model, data, callbacks=callbacks, gp_n_iter=5,
+                    checkpoint='lstm', checkpoint_monitor='val_mse',
+                    epochs=epochs, batch_size=batch_size, verbose=2)
+
+    # Finetune the model
+    model.finetune(*data['train'],
+                   batch_size=batch_size,
+                   gp_n_iter=100,
+                   verbose=0)
+
+    # Test the model
+    X_test, y_test = data['test']
+    y_preds = model.predict(X_test)
+    rmse_predict = RMSE(y_test, y_preds)
+    print('Test predict RMSE:', rmse_predict)
+
+
+if __name__ == '__main__':
+    main()

kgp/backend/gpml.py

@@ -1,7 +1,8 @@
-"""
-GPML backend for Gaussian processes.
-"""
+"""GPML backend for Gaussian processes."""
+
 from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
 
 import os
 import numpy as np
@@ -29,6 +30,7 @@
 xg = covGrid('create', {X}, eq, k);
 """
 
+
 class GPML(object):
     """Class that implements backend functionality for Gaussian processes.
 
@@ -60,8 +62,8 @@ def __init__(self, engine=None, engine_kwargs=None, gpml_path=None):
                 if not os.path.isfile(os.path.join(gpml_path, 'startup.m')):
                     raise ValueError(
                         "Neither GPML_PATH is provided nor GPML library is "
-                        "available directly from KGP. "
-                        "Please make sure you cloned KGP *recursively*.")
+                        "available directly from keras-gp. "
+                        "Please make sure you cloned keras-gp *recursively*.")
 
         engine_kwargs = engine_kwargs or {}
         self.eng = Engine(**engine_kwargs)
@@ -94,33 +96,39 @@ def configure(self, input_dim, hyp, opt, inf, mean, cov, lik, dlik,
             dlik : str
                 Name of the function that computes dlik/dx.
             grid_kwargs : dict
-                'eq' : uint (default: 1)
+                'eq' : uint
                     Whether to enforce an equispaced grid.
                 'k' : uint or float in (0, 1]
                     Number of inducing points per dimension.
+                'xg' : list
+                    Manually specified grid. Must be represented in the form of
+                    a list of np.ndarrays that specify the grid points for each
+                    dimension.
         """
         self.config = {}
         self.config['lik']  = "{@%s}" % lik
 
         if mean_args is None:
             self.config['mean'] = "{@%s}" % mean
         else:
-            self.config['mean'] = '{@%s, %s}' % (mean, ', '.join(str(e) for e in mean_args))
+            self.config['mean'] = '{@%s, %s}' % (
+                mean, ', '.join(str(e) for e in mean_args))
 
         self.config['inf']  = "{@(varargin) %s(varargin{:}, opt)}" % inf
         self.config['dlik'] = "@(varargin) %s(varargin{:}, opt)" % dlik
 
         if inf == 'infGrid':
-            assert grid_kwargs is not None, \
-                "GPML: No arguments provided for grid generation for infGrid."
+            assert grid_kwargs is not None, (
+                "GPML: No arguments provided for grid generation for infGrid.")
             self.eng.push('k', float(grid_kwargs['k']))
             self.eng.push('eq', float(grid_kwargs['eq']))
+            if 'xg' in grid_kwargs:
+                self.eng.push('xg', grid_kwargs['xg'])
             if cov_args is None:
                 cov = ','.join(input_dim * ['{@%s}' % cov])
             else:
-                cov = ','.join(input_dim * ['{@%s, ' % cov + 
-                                            ', '.join(str(e) for e in cov_args) + 
-                                            '}'])
+                cov = ','.join(input_dim * [
+                    '{@%s, %s}' % (cov, ', '.join(str(e) for e in cov_args))])
             if input_dim > 1:
                 cov = '{' + cov + '}'
             hyp['cov'] = np.tile(hyp['cov'], (1, input_dim))

kgp/callbacks.py

@@ -1,8 +1,8 @@
-"""
-Gaussian Process callbacks for Keras.
-"""
+"""Gaussian Process callbacks for Keras."""
+
 from __future__ import absolute_import
 from __future__ import division
+from __future__ import print_function
 
 import warnings
 

kgp/layers.py

@@ -1,7 +1,7 @@
-"""
-Gaussian Process layers for Keras.
-"""
+"""Gaussian Process layers for Keras."""
+
 from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
 import numpy as np

kgp/models.py

@@ -1,24 +1,15 @@
-"""
-Gaussian Process models for Keras 2.
-"""
+"""Gaussian Process models for Keras 2."""
+
 from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
 
-import os
-import sys
-import numpy as np
-
-from keras import optimizers
 from keras.models import Model as KerasModel
 from keras.engine.topology import _to_list
 from keras.engine.training import _standardize_input_data
 
-from .callbacks import Timer
 from .callbacks import UpdateGP
 
-# Backend for neural networks
-from keras import backend as K
-
 
 class Model(KerasModel):
     """Model that supports arbitrary structure with GP output layers.

kgp/tweaks.py

@@ -2,7 +2,10 @@
 Tweaks for KGP and original Keras classes or methods.
 To keep the primary KGP code clean, all tweaks are kept in a separate file.
 """
+
 from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
 
 import numpy as np