Added Autoencoder, VAE, CNN, and GAN

code/autoencoder.py

@@ -0,0 +1,72 @@
+
+import torch.nn as nn
+from torch.autograd import Variable
+
+
+class Autoencoder(nn.Module):
+    def __init__(self, in_shape):
+        super().__init__()
+        c,h,w = in_shape
+        self.encoder = nn.Sequential(
+            nn.Linear(c*h*w, 128),
+            nn.ReLU(),
+            nn.Linear(128, 64),
+            nn.ReLU(),
+            nn.Linear(64, 12),
+            nn.ReLU()
+        )
+        self.decoder = nn.Sequential(
+            nn.Linear(12, 64),
+            nn.ReLU(),
+            nn.Linear(64, 128),
+            nn.ReLU(),
+            nn.Linear(128, c*h*w),
+            nn.Sigmoid()
+        )
+
+    def forward(self, x):
+        bs,c,h,w = x.size()
+        x = x.view(bs, -1)
+        x = self.encoder(x)
+        x = self.decoder(x)
+        x = x.view(bs, c, h, w)
+        return x
+
+
+class ConvAutoencoder(nn.Module):
+    def __init__(self, in_shape):
+        super().__init__()
+        c,h,w = in_shape
+        self.encoder = nn.Sequential(
+            nn.Conv2d(c, 16, kernel_size=3, stride=1, padding=1),  # b, 16, 32, 32
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=2, stride=2),  # b, 16, 16, 16
+            nn.Conv2d(16, 8, kernel_size=3, stride=1, padding=1),  # b, 8, 16, 16
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=2, stride=2)  # b, 8, 8, 8
+        )
+        self.decoder = nn.Sequential(
+            nn.ConvTranspose2d(8, 16, kernel_size=3, stride=2, padding=0),  # 16, 17, 17
+            nn.ReLU(),
+            nn.ConvTranspose2d(16, c, kernel_size=3, stride=2, padding=1),  # 3, 33, 33
+            CenterCrop(h, w), # 3, 32, 32
+            nn.Sigmoid()
+        )
+
+    def forward(self, x):
+        x = self.encoder(x)
+        x = self.decoder(x)
+        return x
+
+
+def train(net, loader, loss_func, optimizer):
+    net.train()
+    for inputs, _ in loader:
+        inputs = Variable(inputs)
+
+        output = net(inputs)
+        loss = loss_func(output, inputs)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()

code/cnn.py

@@ -0,0 +1,62 @@
+import torch.nn as nn
+from torch.autograd import Variable
+
+
+class CNN(nn.Module):
+    def __init__(self, in_shape, n_classes):
+        super().__init__()
+        c, w, h = in_shape
+        pool_layers = 2
+        fc_h = int(h / 2**pool_layers)
+        fc_w = int(w / 2**pool_layers)
+        self.features = nn.Sequential(
+            *conv_bn_relu(c, 16, kernel_size=1, stride=1, padding=0),
+            *conv_bn_relu(16, 32, kernel_size=3, stride=1, padding=1),
+            nn.MaxPool2d(kernel_size=2, stride=2), #size/2
+            *conv_bn_relu(32, 64, kernel_size=3, stride=1, padding=1),
+            nn.MaxPool2d(kernel_size=2, stride=2), #size/2
+        )
+        self.classifier = nn.Sequential(
+            *linear_bn_relu_drop(64 * fc_h * fc_w, 128, dropout=0.5),
+            nn.Linear(128, n_classes),
+            nn.Softmax(dim=1)
+        )
+
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(x.size(0), -1)
+        x = self.classifier(x)
+        return x
+
+def conv_bn_relu(in_chans, out_chans, kernel_size=3, stride=1,
+                 padding=1, bias=False):
+    return [
+        nn.Conv2d(in_chans, out_chans, kernel_size=kernel_size,
+                  stride=stride, padding=padding, bias=bias),
+        nn.BatchNorm2d(out_chans),
+        nn.ReLU(inplace=True),
+    ]
+
+def linear_bn_relu_drop(in_chans, out_chans, dropout=0.5, bias=False):
+    layers = [
+        nn.Linear(in_chans, out_chans, bias=bias),
+        nn.BatchNorm1d(out_chans),
+        nn.ReLU(inplace=True)
+    ]
+    if dropout > 0:
+        layers.append(nn.Dropout(dropout))
+    return layers
+
+def train(net, loader, loss_func, optimizer):
+    net.train()
+    n_batches = len(loader)
+    for inputs, targets in loader:
+        inputs = Variable(inputs)
+        targets = Variable(targets)
+
+        output = net(inputs)
+        loss = loss_func(output, targets)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()

code/vae.py

@@ -0,0 +1,75 @@
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+
+
+class VAE(nn.Module):
+    def __init__(self, in_shape, n_latent):
+        super().__init__()
+        self.in_shape = in_shape
+        self.n_latent = n_latent
+        c,h,w = in_shape
+        self.z_dim = h//2**2 # receptive field downsampled 2 times
+        self.encoder = nn.Sequential(
+            nn.BatchNorm2d(c),
+            nn.Conv2d(c, 32, kernel_size=4, stride=2, padding=1),  # 32, 16, 16
+            nn.BatchNorm2d(32),
+            nn.LeakyReLU(),
+            nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=1),  # 32, 8, 8
+            nn.BatchNorm2d(64),
+            nn.LeakyReLU(),
+        )
+        self.z_mean = nn.Linear(64 * self.z_dim**2, n_latent)
+        self.z_var = nn.Linear(64 * self.z_dim**2, n_latent)
+        self.z_develop = nn.Linear(n_latent, 64 * self.z_dim**2)
+        self.decoder = nn.Sequential(
+            nn.ConvTranspose2d(64, 32, kernel_size=3, stride=2, padding=0),
+            nn.BatchNorm2d(32),
+            nn.ReLU(),
+            nn.ConvTranspose2d(32, 1, kernel_size=3, stride=2, padding=1),
+            CenterCrop(h,w),
+            nn.Sigmoid()
+        )
+
+    def sample_z(self, mean, logvar):
+        stddev = torch.exp(0.5 * logvar)
+        noise = Variable(torch.randn(stddev.size()))
+        return (noise * stddev) + mean
+
+    def encode(self, x):
+        x = self.encoder(x)
+        x = x.view(x.size(0), -1)
+        mean = self.z_mean(x)
+        var = self.z_var(x)
+        return mean, var
+
+    def decode(self, z):
+        out = self.z_develop(z)
+        out = out.view(z.size(0), 64, self.z_dim, self.z_dim)
+        out = self.decoder(out)
+        return out
+
+    def forward(self, x):
+        mean, logvar = self.encode(x)
+        z = self.sample_z(mean, logvar)
+        out = self.decode(z)
+        return out, mean, logvar
+
+
+def vae_loss(output, input, mean, logvar, loss_func):
+    recon_loss = loss_func(output, input)
+    kl_loss = torch.mean(0.5 * torch.sum(
+        torch.exp(logvar) + mean**2 - 1. - logvar, 1))
+    return recon_loss + kl_loss
+
+def train(model, loader, loss_func, optimizer):
+    model.train()
+    for inputs, _ in loader:
+        inputs = Variable(inputs)
+
+        output, mean, logvar = model(inputs)
+        loss = vae_loss(output, inputs, mean, logvar, loss_func)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()

docs/architectures.rst

@@ -0,0 +1,156 @@
+.. _architectures:
+
+=============
+Architectures
+=============
+
+.. contents:: :local:
+
+Autoencoder
+===========
+
+TODO: Description of Autoencoder use case and basic architecture. Figure from [1].
+
+.. image:: images/autoencoder.png
+      :align: center
+
+.. rubric:: Model
+
+An example implementation in PyTorch.
+
+.. literalinclude:: ../code/autoencoder.py
+      :pyobject: Autoencoder
+
+.. rubric:: Training
+
+.. literalinclude:: ../code/autoencoder.py
+      :pyobject: train
+
+.. rubric:: Further reading
+
+- `Convolutional Autoencoders <https://pgaleone.eu/neural-networks/2016/11/24/convolutional-autoencoders/>`_
+- `Deep Learning Book <http://www.deeplearningbook.org/contents/autoencoders.html>`_
+
+
+CNN
+===
+
+TODO: Description of CNN use case and basic architecture. Figure from [2].
+
+.. image:: images/cnn.jpeg
+      :align: center
+
+.. rubric:: Model
+
+An example implementation in PyTorch.
+
+.. literalinclude:: ../code/cnn.py
+      :pyobject: CNN
+
+.. rubric:: Training
+
+.. literalinclude:: ../code/cnn.py
+      :pyobject: train
+
+.. rubric:: Further reading
+
+- `CS231 Convolutional Networks <http://cs231n.github.io/convolutional-networks>`_
+- `Deep Learning Book <http://www.deeplearningbook.org/contents/convnets.html>`_
+
+
+GAN
+===
+
+TODO: Description of GAN use case and basic architecture. Figure from [3].
+
+.. image:: images/gan.png
+      :align: center
+
+.. rubric:: Model
+
+TODO: An example implementation in PyTorch.
+
+.. rubric:: Training
+
+TODO
+
+.. rubric:: Further reading
+
+- `Generative Adversarial Networks <http://guertl.me/post/162759264070/generative-adversarial-networks>`_
+- `Deep Learning Book <http://www.deeplearningbook.org/contents/generative_models.html>`_
+
+
+RNN
+===
+
+Description of RNN use case and basic architecture.
+
+.. image:: images/rnn.png
+      :align: center
+
+.. rubric:: Model
+
+.. literalinclude:: ../code/rnn.py
+      :pyobject: RNN
+
+.. rubric:: Training
+
+In this example, our input is a list of last names, where each name is
+a variable length array of one-hot encoded characters. Our target is is a list of
+indices representing the class (language) of the name.
+
+1. For each input name..
+2. Initialize the hidden vector
+3. Loop through the characters and predict the class
+4. Pass the final character's prediction to the loss function
+5. Backprop and update the weights
+
+.. literalinclude:: ../code/rnn.py
+      :pyobject: train
+
+.. rubric:: Further reading
+
+- `Jupyter notebook <https://github.com/bfortuner/ml-cheatsheet/blob/master/notebooks/rnn.ipynb>`_
+- `Deep Learning Book <http://www.deeplearningbook.org/contents/rnn.html>`_
+
+
+VAE
+===
+
+Variational Autoencoder. Use case and basic architecture. Figure from [4].
+
+.. image:: images/vae.png
+      :align: center
+
+.. rubric:: Loss Function
+
+The VAE loss function combines reconstruction loss (e.g. Cross Entropy, MSE) with KL divergence.
+
+.. literalinclude:: ../code/vae.py
+      :pyobject: vae_loss
+
+.. rubric:: Model
+
+An example implementation in PyTorch of a Convolutional Variational Autoencoder.
+
+.. literalinclude:: ../code/vae.py
+      :pyobject: VAE
+
+.. rubric:: Training
+
+.. literalinclude:: ../code/vae.py
+      :pyobject: train
+
+.. rubric:: Further reading
+
+- `Original Paper <https://arxiv.org/abs/1312.6114>`_
+- `VAE Explained <http://kvfrans.com/variational-autoencoders-explained>`_
+- `Deep Learning Book <http://www.deeplearningbook.org/contents/autoencoders.html>`_
+
+
+.. rubric:: References
+
+.. [1] https://hackernoon.com/autoencoders-deep-learning-bits-1-11731e200694
+.. [2] http://cs231n.github.io/convolutional-networks
+.. [3] http://guertl.me/post/162759264070/generative-adversarial-networks
+.. [4] http://kvfrans.com/variational-autoencoders-explained