Dropout_Simple_4_Gaussian_RandomSample.py

import tensorflow as tf
import mnist_data

import tensorflow.contrib.slim as slim
import time
import seaborn as sns
from Assign_Dataset import *
from tensorflow.examples.tutorials.mnist import input_data
from keras.datasets import mnist
from Support import *
from Mnist_DataHandle import *
from HSICSupport import *
from scipy.misc import imsave as ims
from utils import *

import keras
from keras.datasets import reuters
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.preprocessing.text import Tokenizer

distributions = tf.distributions
from Mixture_Models import *


def merge(images, size):
    h, w = images.shape[1], images.shape[2]
    img = np.zeros((h * size[0], w * size[1]))

    for idx, image in enumerate(images):
        i = idx % size[1]
        j = idx / size[1]
        j = int(j)

        img[int(j * h):int(j * h + h), int(i * w):int(i * w + w)] = image

    return img


def custom_layer(input_matrix, mix, resue=False):
    # with tf.variable_scope("custom_layer",reuse=resue):
    # w_init = tf.contrib.layers.variance_scaling_initializer()
    # b_init = tf.constant_initializer(0.)

    # weights = tf.get_variable(name="mix_weights", initializer=[0.25,0.25,0.25,0.25],trainable=True)
    weights = mix
    a1 = input_matrix[:, 0, :]
    a2 = input_matrix[:, 1, :]
    a3 = input_matrix[:, 2, :]
    a4 = input_matrix[:, 3, :]
    a5 = input_matrix[:, 4, :]
    a6 = input_matrix[:, 5, :]
    a7 = input_matrix[:, 6, :]
    a8 = input_matrix[:, 7, :]
    a9 = input_matrix[:, 8, :]
    a10 = input_matrix[:, 9, :]

    w1 = mix[:, 0:1]
    w2 = mix[:, 1:2]
    w3 = mix[:, 2:3]
    w4 = mix[:, 3:4]
    w5 = mix[:, 4:5]
    w6 = mix[:, 5:6]
    w7 = mix[:, 6:7]
    w8 = mix[:, 7:8]
    w9 = mix[:, 8:9]
    w10 = mix[:, 9:10]

    outputs = w1 * a1 + w2 * a2 + w3 * a3 + w4 * a4 + w5 * a5 + w6 * a6 + w7 * a7 + w8 * a8 + w9 * a9 + w10 * a10
    return outputs


def KL_Dropout2(log_alpha):
    ab = tf.cast(log_alpha, tf.float32)
    k1, k2, k3 = 0.63576, 1.8732, 1.48695;
    C = -k1
    mdkl = k1 * tf.nn.sigmoid(k2 + k3 * ab) - 0.5 * tf.log1p(tf.exp(-ab)) + C
    return -tf.reduce_sum(mdkl)


# Gateway
def autoencoder(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term, Component_Count):
    # encoding
    mu1, sigma1, mix1 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder1")
    mu2, sigma2, mix2 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder2")
    mu3, sigma3, mix3 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder3")
    mu4, sigma4, mix4 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder4")

    z1 = distributions.Normal(loc=mu1, scale=sigma1)
    z2 = distributions.Normal(loc=mu2, scale=sigma2)
    z3 = distributions.Normal(loc=mu3, scale=sigma3)
    z4 = distributions.Normal(loc=mu4, scale=sigma4)

    p = 0.5
    # a = p / (1.0-p)
    ard_init = -10.
    dropout_a = tf.get_variable("dropout", shape=[1], initializer=tf.constant_initializer(ard_init))

    # Dropout of components
    m1 = np.ones(batch_size)
    s1 = np.zeros(batch_size)
    dropout_a = tf.cast(dropout_a, tf.float64)

    dropout_dis = distributions.Normal(loc=m1, scale=dropout_a)
    dropout_samples = dropout_dis.sample(sample_shape=(4))
    dropout_samples = tf.transpose(dropout_samples)
    dropout_samples = tf.cast(dropout_samples, tf.float32)
    dropout_samples = tf.clip_by_value(dropout_samples, 1e-8, 1 - 1e-8)

    mix1 = dropout_samples[:, 0:1]
    mix2 = dropout_samples[:, 1:2]
    mix3 = dropout_samples[:, 2:3]
    mix4 = dropout_samples[:, 3:4]

    sum1 = mix1 + mix2 + mix3 + mix4
    mix1 = mix1 / sum1
    mix2 = mix2 / sum1
    mix3 = mix3 / sum1
    mix4 = mix4 / sum1

    mix = tf.concat([mix1, mix2, mix3, mix4], 1)
    mix_parameters = mix
    dist = tf.distributions.Dirichlet(mix)
    mix_samples = dist.sample()
    mix = mix_samples

    # sampling by re-parameterization technique
    # z = mu + sigma * tf.random_normal(tf.shape(mu), 0, 1, dtype=tf.float32)

    z1_samples = z1.sample()
    z2_samples = z2.sample()
    z3_samples = z3.sample()
    z4_samples = z4.sample()

    ttf = []
    ttf.append(z1_samples)
    ttf.append(z2_samples)
    ttf.append(z3_samples)
    ttf.append(z4_samples)

    dHSIC_Value = dHSIC(ttf)

    # decoding
    y1 = Create_SubDecoder(z1_samples, n_hidden, dim_img, keep_prob, "decoder1")
    # y1 = tf.clip_by_value(y1, 1e-8, 1 - 1e-8)

    y2 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder2")
    # y2 = tf.clip_by_value(y2, 1e-8, 1 - 1e-8)

    y3 = Create_SubDecoder(z3_samples, n_hidden, dim_img, keep_prob, "decoder3")
    # y3 = tf.clip_by_value(y3, 1e-8, 1 - 1e-8)

    y4 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder4")
    # y4 = tf.clip_by_value(y4, 1e-8, 1 - 1e-8)

    # dropout out
    y1 = y1 * mix_samples[:, 0:1]
    y2 = y2 * mix_samples[:, 1:2]
    y3 = y3 * mix_samples[:, 2:3]
    y4 = y4 * mix_samples[:, 3:4]

    y = y1 + y2 + y3 + y4
    output = Create_FinalDecoder(y, n_hidden, dim_img, keep_prob, "final")
    y = output

    m1 = np.zeros(dim_z, dtype=np.float32)
    m1[:] = 0
    v1 = np.zeros(dim_z, dtype=np.float32)
    v1[:] = 1

    # p_z1 = distributions.Normal(loc=np.zeros(dim_z, dtype=np.float32),
    #                           scale=np.ones(dim_z, dtype=np.float32))
    p_z1 = distributions.Normal(loc=m1,
                                scale=v1)

    m2 = np.zeros(dim_z, dtype=np.float32)
    m2[:] = 0
    v2 = np.zeros(dim_z, dtype=np.float32)
    v2[:] = 1
    p_z2 = distributions.Normal(loc=m2,
                                scale=v2)

    m3 = np.zeros(dim_z, dtype=np.float32)
    m3[:] = 0
    v3 = np.zeros(dim_z, dtype=np.float32)
    v3[:] = 1
    p_z3 = distributions.Normal(loc=m3,
                                scale=v3)

    m4 = np.zeros(dim_z, dtype=np.float32)
    m4[:] = 0
    v4 = np.zeros(dim_z, dtype=np.float32)
    v4[:] = 1
    p_z4 = distributions.Normal(loc=m4,
                                scale=v4)

    kl1 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z1, p_z1), 1))
    kl2 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z2, p_z2), 1))
    kl3 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z3, p_z3), 1))
    kl4 = tf.reduce_mean(tf.reduce_sum(distributions.kl_divergence(z4, p_z4), 1))

    kl1 = kl1
    kl2 = kl2
    kl3 = kl3
    kl4 = kl4

    KL_divergence = (kl1 + kl2 + kl3 + kl4) / 4.0

    # loss
    marginal_likelihood = tf.reduce_sum(x * tf.log(y) + (1 - x) * tf.log(1 - y), 1)

    marginal_likelihood = tf.reduce_mean(marginal_likelihood)

    # KL divergence between two Dirichlet distributions
    a1 = tf.clip_by_value(mix_parameters, 0.1, 0.8)
    a2 = tf.constant((0.25, 0.25, 0.25, 0.25), shape=(batch_size, 4))

    r = tf.reduce_sum((a1 - a2) * (tf.polygamma(0.0, a1) - tf.polygamma(0.0, 1)), axis=1)
    a = tf.lgamma(tf.reduce_sum(a1, axis=1)) - tf.lgamma(tf.reduce_sum(a2, axis=1)) + tf.reduce_sum(tf.lgamma(a2),
                                                                                                    axis=-1) - tf.reduce_sum(
        tf.lgamma(a1), axis=1) + r
    kl = a
    kl = tf.reduce_mean(kl)

    p1 = 1
    p2 = 1
    p4 = 1
    ELBO = marginal_likelihood - KL_divergence * p2

    loss = -ELBO + kl * p1 + p4 * dHSIC_Value + KL_Dropout2(dropout_a)

    z = z1_samples
    return y, z, loss, -marginal_likelihood, KL_divergence


def decoder(z, dim_img, n_hidden):
    y = bernoulli_MLP_decoder(z, n_hidden, dim_img, 1.0, reuse=True)
    return y


def HiddenOuputs(x_hat, x, dim_img, dim_z, n_hidden, keep_prob, last_term):
    # encoding
    mu1, sigma1, mix1 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder1", True)
    z1 = distributions.Normal(loc=mu1, scale=sigma1)

    mu2, sigma2, mix2 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder2", True)
    z2 = distributions.Normal(loc=mu2, scale=sigma2)

    mu3, sigma3, mix3 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder3", True)
    z3 = distributions.Normal(loc=mu3, scale=sigma3)

    mu4, sigma4, mix4 = Create_Encoder_MNIST(x_hat, n_hidden, dim_z, keep_prob, "encoder4", True)
    z4 = distributions.Normal(loc=mu4, scale=sigma4)

    z1 = distributions.Normal(loc=mu1, scale=sigma1)
    z2 = distributions.Normal(loc=mu2, scale=sigma2)
    z3 = distributions.Normal(loc=mu3, scale=sigma3)
    z4 = distributions.Normal(loc=mu4, scale=sigma4)

    z1_samples = z1.sample()
    z2_samples = z2.sample()
    z3_samples = z3.sample()
    z4_samples = z4.sample()

    sum1 = mix1 + mix2 + mix3 + mix4
    mix1 = mix1 / sum1
    mix2 = mix2 / sum1
    mix3 = mix3 / sum1
    mix4 = mix4 / sum1
    mix = tf.concat([mix1, mix2, mix3, mix4], 1)
    mix_parameters = mix
    dist = tf.distributions.Dirichlet(mix)
    mix_samples = dist.sample()
    mix = mix_samples

    return z1_samples, z2_samples, z3_samples, z4_samples, mix


def Final_Output(z1_samples, z2_samples, z3_samples, z4_samples, mix_samples, dim_img, dim_z, n_hidden, keep_prob):
    ard_init = -10.
    with tf.variable_scope("", reuse=True):  # root variable scope
        dropout_a = tf.get_variable("dropout")

    # Dropout of components
    m1 = np.ones(batch_size)
    s1 = np.zeros(batch_size)
    dropout_a = tf.cast(dropout_a, tf.float64)

    dropout_dis = distributions.Normal(loc=m1, scale=dropout_a)
    dropout_samples = dropout_dis.sample(sample_shape=(10))
    dropout_samples = tf.transpose(dropout_samples)
    dropout_samples = tf.cast(dropout_samples, tf.float32)
    dropout_samples = tf.clip_by_value(dropout_samples, 1e-8, 1 - 1e-8)

    # decoding
    y1 = Create_SubDecoder(z1_samples, n_hidden, dim_img, keep_prob, "decoder1", True)
    # y1 = tf.clip_by_value(y1, 1e-8, 1 - 1e-8)

    y2 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder2", True)
    # y2 = tf.clip_by_value(y2, 1e-8, 1 - 1e-8)

    y3 = Create_SubDecoder(z3_samples, n_hidden, dim_img, keep_prob, "decoder3", True)
    # y3 = tf.clip_by_value(y3, 1e-8, 1 - 1e-8)

    y4 = Create_SubDecoder(z4_samples, n_hidden, dim_img, keep_prob, "decoder4", True)
    # y4 = tf.clip_by_value(y4, 1e-8, 1 - 1e-8)

    # dropout out
    y1 = y1 * mix_samples[:, 0:1]
    y2 = y2 * mix_samples[:, 1:2]
    y3 = y3 * mix_samples[:, 2:3]
    y4 = y4 * mix_samples[:, 3:4]

    y = y1 + y2 + y3 + y4
    output = Create_FinalDecoder(y, n_hidden, dim_img, keep_prob, "final", True)
    output = tf.clip_by_value(output, 1e-8, 1 - 1e-8)
    y = output
    return y


def Final_Output_Confusion(z1_samples, z2_samples, z3_samples, z4_samples,mix_samples, dim_img, dim_z, n_hidden, keep_prob):
    ard_init = -10.
    with tf.variable_scope("", reuse=True):  # root variable scope
        dropout_a = tf.get_variable("dropout")

    # Dropout of components
    m1 = np.ones(batch_size)
    s1 = np.zeros(batch_size)
    dropout_a = tf.cast(dropout_a, tf.float64)

    dropout_dis = distributions.Normal(loc=m1, scale=dropout_a)
    dropout_samples = dropout_dis.sample(sample_shape=(10))
    dropout_samples = tf.transpose(dropout_samples)
    dropout_samples = tf.cast(dropout_samples, tf.float32)
    dropout_samples = tf.clip_by_value(dropout_samples, 1e-8, 1 - 1e-8)

    # decoding
    y1 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder1", True)
    # y1 = tf.clip_by_value(y1, 1e-8, 1 - 1e-8)

    y2 = Create_SubDecoder(z1_samples, n_hidden, dim_img, keep_prob, "decoder2", True)
    # y2 = tf.clip_by_value(y2, 1e-8, 1 - 1e-8)

    y3 = Create_SubDecoder(z2_samples, n_hidden, dim_img, keep_prob, "decoder3", True)
    # y3 = tf.clip_by_value(y3, 1e-8, 1 - 1e-8)

    y4 = Create_SubDecoder(z3_samples, n_hidden, dim_img, keep_prob, "decoder4", True)
    # y4 = tf.clip_by_value(y4, 1e-8, 1 - 1e-8)

    # dropout out
    y1 = y1 * mix_samples[:, 0:1]
    y2 = y2 * mix_samples[:, 1:2]
    y3 = y3 * mix_samples[:, 2:3]
    y4 = y4 * mix_samples[:, 3:4]

    y = y1 + y2 + y3 + y4
    output = Create_FinalDecoder(y, n_hidden, dim_img, keep_prob, "final", True)
    output = tf.clip_by_value(output, 1e-8, 1 - 1e-8)
    y = output
    return y

def Samples_of_Gaussian():
    m3 = np.zeros(dim_z, dtype=np.float32)
    m3[:] = 0
    v3 = np.zeros(dim_z, dtype=np.float32)
    v3[:] = 1
    p_z3 = distributions.Normal(loc=m3,
                                scale=v3)

    samples = p_z3.sample()

    return samples

n_hidden = 500
IMAGE_SIZE_MNIST = 28
dim_img = IMAGE_SIZE_MNIST ** 2  # number of pixels for a MNIST image

dim_z = 50

# train
n_epochs = 100
batch_size = 128
learn_rate = 0.001

train_total_data, train_size, _, _, test_data, test_labels = mnist_data.prepare_MNIST_data()
n_samples = train_size
# input placeholders

# In denoising-autoencoder, x_hat == x + noise, otherwise x_hat == x
x_hat = tf.placeholder(tf.float32, shape=[None, dim_img], name='input_img')
x = tf.placeholder(tf.float32, shape=[None, dim_img], name='target_img')

# dropout
keep_prob = tf.placeholder(tf.float32, name='keep_prob')

# input for PMLR
z_in = tf.placeholder(tf.float32, shape=[None, dim_z], name='latent_variable')
last_term = tf.placeholder(tf.float32, shape=[10])
Component_Count = tf.placeholder(tf.float32)

# network architecture
y, z, loss, neg_marginal_likelihood, KL_divergence = autoencoder(x_hat, x, dim_img, dim_z, n_hidden, keep_prob,
                                                                 last_term, Component_Count)
z1_samples, z2_samples, z3_samples, z4_samples, mix = HiddenOuputs(x_hat, x, dim_img, dim_z, n_hidden, keep_prob,
                                                                   last_term)
# y1, y2, y3, y4 = Outpiuts_Component(x_hat, x, dim_img, dim_z, n_hidden, keep_prob)

# optimization
t_vars = tf.trainable_variables()
train_op = tf.train.AdamOptimizer(learn_rate).minimize(loss, var_list=t_vars)

# train

total_batch = int(n_samples / batch_size)

min_tot_loss = 1e99
ADD_NOISE = False

train_data2_ = train_total_data[:, :-mnist_data.NUM_LABELS]
train_y = train_total_data[:, 784:784 + mnist_data.NUM_LABELS]

# MNIST dataset
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

x_fixed = train_data2_[0:128]
saver = tf.train.Saver()

isWeight = True

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer(), feed_dict={keep_prob: 0.9})
    lastvalue = np.ones((4))

    if isWeight:
        saver.restore(sess, 'models/Dropout_Sinmple_4_Gaussian')

        bc = 0
        sample_arr = []
        gaussian_sample = Samples_of_Gaussian()
        for k1 in range(batch_size):
            gaussian_sample1 = sess.run(gaussian_sample)
            sample_arr.append(gaussian_sample1)
        sample_arr = np.array(sample_arr)

        tIndex = 5
        x_fixed = x_train[batch_size * tIndex:batch_size * tIndex + batch_size]
        x_fixed = x_test[batch_size * tIndex:batch_size * tIndex + batch_size]
        x_fixed = np.reshape(x_fixed, (-1, 28 * 28))

        reco = sess.run(y, feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})
        reco = np.reshape(reco, (-1, 28, 28))
        # x_fixed = np.reshape(x_fixed,(-1,28,28))
        # ims("results/" + "MNIST__Real" + str(0) + ".png", merge(x_fixed[:64], [8, 8]))
        # ims("results/" + "MNIST__Reco" + str(0) + ".png", merge(reco[:64], [8, 8]))

        # yy = sess.run(y,feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})
        # yy = np.reshape(yy,(-1,28,28))
        # ims("results/" + "VAE" + str(0) + ".jpg", merge(yy[:64], [8, 8]))

        z_in1 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in2 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in3 = tf.placeholder(tf.float32, shape=[None, dim_z])
        z_in4 = tf.placeholder(tf.float32, shape=[None, dim_z])
        mix_in = tf.placeholder(tf.float32, shape=[None, 4])

        yy = Final_Output_Confusion(z_in1, z_in2, z_in3, z_in4, mix_in, dim_img, dim_z, n_hidden, keep_prob)

        z1_samples_, z2_samples_, z3_samples_, z4_samples_, mix_ = sess.run(
            (z1_samples, z2_samples, z3_samples, z4_samples, mix),
            feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})

        #z1_samples_ = sample_arr
        #z2_samples_ = sample_arr
        #z3_samples_ = sample_arr
        #z4_samples_ = sample_arr

        mix_[:, :] = 0
        mix_[:, 0:1] = 1
        y1 = sess.run(yy, feed_dict={z_in1: z1_samples_, z_in2: z2_samples_, z_in3: z3_samples_, z_in4: z4_samples_,
                                     keep_prob: 1.0, mix_in: mix_})

        mix_[:, :] = 0
        mix_[:, 1:2] = 1
        y2 = sess.run(yy, feed_dict={z_in1: z1_samples_, z_in2: z2_samples_, z_in3: z3_samples_, z_in4: z4_samples_,
                                     keep_prob: 1.0, mix_in: mix_})

        mix_[:, :] = 0
        mix_[:, 2:3] = 1
        y3 = sess.run(yy, feed_dict={z_in1: z1_samples_, z_in2: z2_samples_, z_in3: z3_samples_, z_in4: z4_samples_,
                                     keep_prob: 1.0, mix_in: mix_})

        mix_[:, :] = 0
        mix_[:, 3:4] = 1
        y4 = sess.run(yy, feed_dict={z_in1: z1_samples_, z_in2: z2_samples_, z_in3: z3_samples_, z_in4: z4_samples_,
                                     keep_prob: 1.0, mix_in: mix_})

        y5 = sess.run(y, feed_dict={x_hat: x_fixed, x: x_fixed, keep_prob: 1.0})

        yyArr = np.zeros((batch_size, 28 * 28))
        yyArr[0:20, :] = x_fixed[0:20]
        yyArr[20:40, :] = y1[0:20]
        yyArr[40:60, :] = y2[0:20]
        yyArr[60:80, :] = y3[0:20]
        yyArr[80:100, :] = y4[0:20]
        yyArr[100:120, :] = y5[0:20]

        yyArr = np.reshape(yyArr, (-1, 28, 28))

        ims("results/" + "b" + str(0) + ".png", merge(yyArr[:100], [5, 20]))

        ims("results/" + "MNIST_Fusion" + str(0) + ".png", merge(yyArr[0:20], [1, 20]))
        ims("results/" + "MNIST_Fusion" + str(1) + ".png", merge(yyArr[20:40], [1, 20]))
        ims("results/" + "MNIST_Fusion" + str(2) + ".png", merge(yyArr[40:60], [1, 20]))
        ims("results/" + "MNIST_Fusion" + str(3) + ".png", merge(yyArr[60:80], [1, 20]))
        ims("results/" + "MNIST_Fusion" + str(4) + ".png", merge(yyArr[80:100], [1, 20]))

        bc = 0

    # training

    lastvalue = np.ones((10))
    valid_reco = Give_Reconstruction(x_hat, x)
    x_valid = x_test[0:batch_size]
    x_valid = np.reshape(x_valid, (-1, 28 * 28))
    bestScore = 100

    n_epochs = 100
    for epoch in range(n_epochs):
        # Random shuffling
        np.random.shuffle(train_total_data)
        train_data_ = train_total_data[:, :-mnist_data.NUM_LABELS]

        # Loop over all batches
        for i in range(total_batch):
            # Compute the offset of the current minibatch in the data.
            offset = (i * batch_size) % (n_samples)
            batch_xs_input = train_data_[offset:(offset + batch_size), :]
            batch_xs_target = batch_xs_input

            if ADD_NOISE:
                batch_xs_input = batch_xs_input * np.random.randint(2, size=batch_xs_input.shape)
                batch_xs_input += np.random.randint(2, size=batch_xs_input.shape)

            _, tot_loss, loss_likelihood, loss_divergence = sess.run(
                (train_op, loss, neg_marginal_likelihood, KL_divergence),
                feed_dict={x_hat: batch_xs_input, x: batch_xs_target, keep_prob: 1.0})

        y_PRR = sess.run(y, feed_dict={x_hat: x_fixed, keep_prob: 1})
        y_RPR = np.reshape(y_PRR, (-1, 28, 28))
        ims("results/" + "VAE" + str(epoch) + ".jpg", merge(y_RPR[:64], [8, 8]))

        print("epoch %f: L_tot %03.2f L_likelihood %03.2f L_divergence %03.2f" % (
            epoch, tot_loss, loss_likelihood, loss_divergence))

        if epoch > 0:
            x_fixed_image = np.reshape(x_fixed, (-1, 28, 28))
            ims("results/" + "Real" + str(epoch) + ".jpg", merge(x_fixed_image[:64], [8, 8]))

        y_PRR = sess.run(y, feed_dict={x_hat: x_valid, keep_prob: 1})
        recoValue = sess.run(valid_reco, feed_dict={x_hat: x_valid, x: y_PRR})
        if bestScore < recoValue:
            bestScore = recoValue
            saver.save(sess, 'models/Dropout_Sinmple_4_Gaussian')
    saver.save(sess, 'models/Dropout_Sinmple_4_Gaussian')