Skip to content

Commit

Permalink
ESN reset
Browse files Browse the repository at this point in the history
  • Loading branch information
@nschaetti
nschaetti committed Jun 17, 2018
1 parent 0ce1841 commit 8a34de4
Show file tree
Hide file tree
Showing 3 changed files with 209 additions and 56 deletions.
108 changes: 63 additions & 45 deletions echotorch/nn/GatedESN.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@
import torch
import torch.nn as nn
import torch.nn.functional as F
from .BDESNCell import BDESNCell
from .LiESNCell import LiESNCell
from sklearn.decomposition import IncrementalPCA
from .PCACell import PCACell
import matplotlib.pyplot as plt
from torch.autograd import Variable

Expand All @@ -43,8 +44,8 @@ class GatedESN(nn.Module):
# Constructor
def __init__(self, input_dim, reservoir_dim, pca_dim, hidden_dim, leaky_rate=1.0, spectral_radius=0.9,
bias_scaling=0, input_scaling=1.0, w=None, w_in=None, w_bias=None, sparsity=None,
input_set=[1.0, -1.0], w_sparsity=None, nonlin_func=torch.tanh, learning_algo='inv', ridge_param=0.0,
create_cell=True, pca_batch_size=10):
input_set=[1.0, -1.0], w_sparsity=None, nonlin_func=torch.tanh,
create_cell=True):
"""
Constructor
:param input_dim: Inputs dimension.
Expand All @@ -68,38 +69,30 @@ def __init__(self, input_dim, reservoir_dim, pca_dim, hidden_dim, leaky_rate=1.0
self.reservoir_dim = reservoir_dim
self.pca_dim = pca_dim
self.hidden_dim = hidden_dim
self.finalized = False

# Recurrent layer
if create_cell:
self.esn_cell = BDESNCell(
input_dim=input_dim, hidden_dim=reservoir_dim, spectral_radius=spectral_radius, bias_scaling=bias_scaling,
self.esn_cell = LiESNCell(
input_dim=input_dim, output_dim=reservoir_dim, spectral_radius=spectral_radius, bias_scaling=bias_scaling,
input_scaling=input_scaling, w=w, w_in=w_in, w_bias=w_bias, sparsity=sparsity, input_set=input_set,
w_sparsity=w_sparsity, nonlin_func=nonlin_func, leaky_rate=leaky_rate, create_cell=create_cell
w_sparsity=w_sparsity, nonlin_func=nonlin_func, leaky_rate=leaky_rate
)
# end if

# PCA
if self.pca_dim > 0:
self.ipca = IncrementalPCA(n_components=pca_dim, batch_size=pca_batch_size)
self.pca_cell = PCACell(input_dim=reservoir_dim, output_dim=pca_dim)
# end if

# Init hidden vector
self.register_buffer('hidden', self.init_hidden())

# Init update vector
self.register_buffer('update', self.init_update())

# Initialize input update weights
self.register_buffer('wzp', self._init_wzp())
self.register_parameter('wzp', nn.Parameter(self.init_wzp()))

# Initialize hidden update weights
self.register_buffer('wzh', self._init_wzh())

# Initialize input update weights
self.register_buffer('wzp', self._init_wzp())
self.register_parameter('wzh', nn.Parameter(self.init_wzh()))

# Initialize update bias
self.register_buffer('bz', self._init_bz())
self.register_parameter('bz', nn.Parameter(self.init_bz()))
# end __init__

###############################################
Expand Down Expand Up @@ -164,7 +157,7 @@ def init_wzp(self):
Init update-reduced matrix
:return: Initiated update-reduced matrix
"""
return Variable(torch.rand(self.pca_dim, self.hidden_dim))
return torch.rand(self.pca_dim, self.hidden_dim)
# end init_hidden

# Init update-hidden matrix
Expand All @@ -173,20 +166,26 @@ def init_wzh(self):
Init update-hidden matrix
:return: Initiated update-hidden matrix
"""
return Variable(torch.rand(self.pca_dim, self.hidden_dim))
return torch.rand(self.pca_dim, self.hidden_dim)
# end init_hidden

# Init update bias
def init_bz(self):
"""
Init update bias
:return:
"""
return torch.rand(self.hidden_dim)
# end init_bz

# Reset learning
def reset(self):
"""
Reset learning
:return:
"""
# Reset PCA layer
self.output.reset()

# Reset hidden vector
self.reset_hidden()
self.pca_cell.reset()

# Reset reservoir
self.reset_reservoir()
Expand All @@ -210,39 +209,58 @@ def forward(self, u, y=None):

# Compute reservoir states
reservoir_states = self.esn_cell(u)
reservoir_states.required_grad = False

# Resulting reduced stated
pca_states = torch.zeros(1, reservoir_states.size(1), self.pca_dim)

# For each batch
pca_states[0] = torch.from_numpy(self.ipca.fit_transform(reservoir_states.data[0].numpy()).copy())
pca_states = Variable(pca_states)

# Hidden states
hidden_states = Variable(torch.zeros(n_batches, time_length, self.hidden_dim))
hidden_states = hidden_states.cuda() if reservoir_states.is_cuda else hidden_states
# Reduce
if self.pca_dim > 0:
# Reduce states
pca_states = self.pca_cell(reservoir_states)
pca_states.required_grad = False

# Stop here if we learn PCA
if self.finalized:
return
# end if

# Hidden states
hidden_states = Variable(torch.zeros(n_batches, time_length, self.hidden_dim))
hidden_states = hidden_states.cuda() if pca_states.is_cuda else hidden_states
else:
# Hidden states
hidden_states = Variable(torch.zeros(n_batches, time_length, self.hidden_dim))
hidden_states = hidden_states.cuda() if reservoir_states.is_cuda else hidden_states
# end if

# For each batch
for b in range(n_batches):
# Reset hidden layer
self.reset_hidden()
hidden = self.init_hidden()

# TO CUDA
if u.is_cuda:
hidden = hidden.cuda()
# end if

# For each steps
for t in range(time_length):
# Current reduced state
pt = pca_states[b, t]
if self.pca_dim > 0:
pt = pca_states[b, t]
else:
pt = reservoir_states[b, t]
# end if

# Compute update vectpr
zt = F.sigmoid(self.wzp.mv(pt) + self.wzh.mv(self.hidden) + self.bz)
# Compute update vector
zt = F.sigmoid(self.wzp.mv(pt) + self.wzh.mv(hidden) + self.bz)

# Compute hidden state
ht = (1.0 - zt) * self.hidden + zt * pt
ht = (1.0 - zt) * hidden + zt * pt

# Add to outputs
self.hidden.data = ht.view(self.output_dim).data
hidden = ht.view(self.hidden_dim)

# New last state
hidden_states[b, t] = self.hidden
hidden_states[b, t] = hidden
# end for
# end for

Expand All @@ -256,10 +274,10 @@ def finalize(self):
Finalize training with LU factorization
"""
# Finalize output training
self.output.finalize()
self.pca_cell.finalize()

# Not in training mode anymore
self.train(False)
# Finalized
self.finalized = True
# end finalize

# Reset reservoir layer
Expand Down
23 changes: 12 additions & 11 deletions echotorch/nn/StackedESN.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from . import LiESNCell
from RRCell import RRCell
from ESNCell import ESNCell
import numpy as np


# Stacked Echo State Network module
Expand Down Expand Up @@ -78,13 +79,13 @@ def __init__(self, input_dim, hidden_dim, output_dim, leaky_rate=1.0, spectral_r
self.n_features += hidden_dim[n]

# Parameters
layer_leaky_rate = leaky_rate[n] if type(leaky_rate) is list else leaky_rate
layer_spectral_radius = spectral_radius[n] if type(spectral_radius) is list else spectral_radius
layer_bias_scaling = bias_scaling[n] if type(bias_scaling) is list else bias_scaling
layer_input_scaling = input_scaling[n] if type(input_scaling) is list else input_scaling
layer_leaky_rate = leaky_rate[n] if type(leaky_rate) is list or type(leaky_rate) is np.ndarray else leaky_rate
layer_spectral_radius = spectral_radius[n] if type(spectral_radius) is list or type(spectral_radius) is np.ndarray else spectral_radius
layer_bias_scaling = bias_scaling[n] if type(bias_scaling) is list or type(bias_scaling) is np.ndarray else bias_scaling
layer_input_scaling = input_scaling[n] if type(input_scaling) is list or type(input_scaling) is np.ndarray else input_scaling

# W
if type(w) is torch.Tensor and w.ndim == 3:
if type(w) is torch.Tensor and w.dim() == 3:
layer_w = w[n]
elif type(w) is torch.Tensor:
layer_w = w
Expand All @@ -93,7 +94,7 @@ def __init__(self, input_dim, hidden_dim, output_dim, leaky_rate=1.0, spectral_r
# end if

# W in
if type(w_in) is torch.Tensor and w_in.ndim == 3:
if type(w_in) is torch.Tensor and w_in.dim() == 3:
layer_w_in = w_in[n]
elif type(w_in) is torch.Tensor:
layer_w_in = w_in
Expand All @@ -102,7 +103,7 @@ def __init__(self, input_dim, hidden_dim, output_dim, leaky_rate=1.0, spectral_r
# end if

# W bias
if type(w_bias) is torch.Tensor and w_bias.ndim == 2:
if type(w_bias) is torch.Tensor and w_bias.dim() == 2:
layer_w_bias = w_bias[n]
elif type(w_bias) is torch.Tensor:
layer_w_bias = w_bias
Expand All @@ -111,10 +112,10 @@ def __init__(self, input_dim, hidden_dim, output_dim, leaky_rate=1.0, spectral_r
# end if

# Parameters
layer_sparsity = sparsity[n] if type(sparsity) is list else sparsity
layer_input_set = input_set[n] if type(input_set) is list else input_set
layer_w_sparsity = w_sparsity[n] if type(w_sparsity) is list else w_sparsity
layer_nonlin_func = nonlin_func[n] if type(nonlin_func) is list else nonlin_func
layer_sparsity = sparsity[n] if type(sparsity) is list or type(sparsity) is np.ndarray else sparsity
layer_input_set = input_set[n] if type(input_set) is list or type(input_set) is np.ndarray else input_set
layer_w_sparsity = w_sparsity[n] if type(w_sparsity) is list or type(w_sparsity) is np.ndarray else w_sparsity
layer_nonlin_func = nonlin_func[n] if type(nonlin_func) is list or type(nonlin_func) is np.ndarray else nonlin_func

self.esn_layers.append(LiESNCell(
layer_leaky_rate, False, layer_input_dim, hidden_dim[n], layer_spectral_radius, layer_bias_scaling,
Expand Down
134 changes: 134 additions & 0 deletions examples/timeserie_prediction/narma10_gated_esn.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
#
# File : examples/timeserie_prediction/switch_attractor_esn
# Description : NARMA 30 prediction with ESN.
# Date : 26th of January, 2018
#
# This file is part of EchoTorch. EchoTorch is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Nils Schaetti <nils.schaetti@unine.ch>


# Imports
import torch
import torch.optim as optim
from echotorch.datasets.NARMADataset import NARMADataset
import echotorch.nn as etnn
import echotorch.utils
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data.dataloader import DataLoader
import numpy as np
import mdp
import matplotlib.pyplot as plt

# Parameters
spectral_radius = 0.9
leaky_rate = 1.0
learning_rate = 0.04
reservoir_dim = 100
hidden_dim = 20
input_dim = 1
n_hidden = 100
n_iterations = 2000
train_sample_length = 5000
test_sample_length = 1000
n_train_samples = 1
n_test_samples = 1
batch_size = 1
momentum = 0.95
weight_decay = 0

# Use CUDA?
use_cuda = True
use_cuda = torch.cuda.is_available() if use_cuda else False

# Manual seed
mdp.numx.random.seed(1)
np.random.seed(2)
torch.manual_seed(1)

# NARMA30 dataset
narma10_train_dataset = NARMADataset(train_sample_length, n_train_samples, system_order=10, seed=1)
narma10_test_dataset = NARMADataset(test_sample_length, n_test_samples, system_order=10, seed=10)

# Data loader
trainloader = DataLoader(narma10_train_dataset, batch_size=batch_size, shuffle=False, num_workers=2)
testloader = DataLoader(narma10_test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

# Linear output
linear = nn.Linear(in_features=hidden_dim, out_features=1)

# ESN cell
gated_esn = etnn.GatedESN(
input_dim=input_dim,
reservoir_dim=input_dim,
pca_dim=hidden_dim,
hidden_dim=hidden_dim,
leaky_rate=leaky_rate,
spectral_radius=spectral_radius
)
if use_cuda:
gated_esn.cuda()
linear.cuda()
# end if

# Objective function
criterion = nn.MSELoss()

# Stochastic Gradient Descent
optimizer = optim.SGD(gated_esn.parameters(), lr=learning_rate, momentum=momentum)

# For each iteration
for epoch in range(n_iterations):
# Iterate over batches
for data in trainloader:
# Inputs and outputs
inputs, targets = data
inputs, targets = Variable(inputs), Variable(targets)
if use_cuda: inputs, targets = inputs.cuda(), targets.cuda()
print(inputs)
print(targets)
# Gradients to zero
optimizer.zero_grad()

# Forward
out = linear(gated_esn(inputs))
print(out)
exit()
loss = criterion(out, targets)

# Backward pass
loss.backward()

# Optimize
optimizer.step()

# Print error measures
print(u"Train MSE: {}".format(float(loss.data)))
print(u"Train NRMSE: {}".format(echotorch.utils.nrmse(out.data, targets.data)))
# end for

# Test reservoir
dataiter = iter(testloader)
test_u, test_y = dataiter.next()
test_u, test_y = Variable(test_u), Variable(test_y)
if use_cuda: test_u, test_y = test_u.cuda(), test_y.cuda()
y_predicted = esn(test_u)

# Print error measures
print(u"Test MSE: {}".format(echotorch.utils.mse(y_predicted.data, test_y.data)))
print(u"Test NRMSE: {}".format(echotorch.utils.nrmse(y_predicted.data, test_y.data)))
print(u"")
# end for

0 comments on commit 8a34de4

Please sign in to comment.