diff --git a/LSTM-2ConvNet.py b/LSTM-2ConvNet.py new file mode 100644 index 0000000..647c2e9 --- /dev/null +++ b/LSTM-2ConvNet.py @@ -0,0 +1,114 @@ +import tensorflow as tf +import time + +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') + +def rnn_layer(x, timesteps, num_hidden, weights): + x = tf.unstack(x, timesteps, 1) + lstm_cell_a = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + lstm_cell_b = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm_cell_a, lstm_cell_b, x, dtype=tf.float32) + return tf.matmul(outputs[-1], weights) + +# Placeholders +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +# ConvLayer 1 with max-pooling +W_conv1 = weight_variable([5, 5, 1, 32]) +b_conv1 = bias_variable([32]) +x_image = tf.reshape(x, [-1, 28, 28, 1]) +h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) +h_pool1 = max_pool_2x2(h_conv1) + +# ConvLayer 2 with max-pooling +W_conv2 = weight_variable([5, 5, 32, 64]) +b_conv2 = bias_variable([64]) +h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) +h_pool2 = max_pool_2x2(h_conv2) + +# LSTM Branch +h_pool1_reshape = tf.transpose(h_pool1, [0, 3, 1, 2]) +h_pool1_reshape = tf.reshape(h_pool1_reshape, [-1, 32, 144]) +W_lstm = weight_variable([256, 1536]) +h_lstm = rnn_layer(h_pool1_reshape, 32, 128, W_lstm) + +# Dense Layer 1 +W_dense1 = weight_variable([4 * 4 * 64, 1536]) +b_dense1 = bias_variable([1536]) +h_pool2 = tf.reshape(h_pool2, [-1, 4 * 4 * 64]) +h_dense1 = tf.nn.relu(tf.matmul(h_pool2, W_dense1) + h_lstm + b_dense1) + +# Dense Layer 2 +W_dense2 = weight_variable([1536, 128]) +b_dense2 = bias_variable([128]) +h_dense2 = tf.nn.relu(tf.matmul(h_dense1, W_dense2) + b_dense2) + +# Dropout +keep_prob = tf.placeholder(tf.float32) +h_dense2_drop = tf.nn.dropout(h_dense2, keep_prob) + +# Dense Layer 3 with Softmax Output +W_dense3 = weight_variable([128, 10]) +b_dense3 = bias_variable([10]) +y_conv = tf.matmul(h_dense2_drop, W_dense3) + b_dense3 + +# Training Parameters +training_rate = tf.placeholder(tf.float32) +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(training_rate).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +saver = tf.train.Saver(tf.global_variables()) + +with tf.Session() as sess: + # Training + sess.run(tf.global_variables_initializer()) + data_location = './MNIST-LSTM-2ConvNet-DATA/MNIST_LSTM_ConvNet' + saver.restore(sess, data_location) + last_time = time.time() + rate = 0.0001 + for i in range(50000): + batch = mnist.train.next_batch(50) + sess.run(train_step, feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 0.5, training_rate: rate}) + if i % 10 == 0: + loss, acc = sess.run([cross_entropy, accuracy], feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0, training_rate: rate}) + print('Step: %d, Accuracy: %.2f, Loss: %.5f, Speed: %.1f sec/10 steps' % + (i, acc, loss, time.time() - last_time)) + last_time = time.time() + if i % 250 == 0: + current_accuracy = accuracy.eval( + feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0, training_rate: rate}) + print('- Current Test Accuracy %.4f' % current_accuracy) + saver.save(sess, data_location) + print('- Model Saved in Step %d' % i) + if current_accuracy > 0.98: + rate = 0.00003 + if current_accuracy > 0.99: + rate = 0.00001 + if current_accuracy > 0.993: + rate = 0.000003 + if current_accuracy > 0.995: + print('- Accuracy Reached 99.5% in Step %d' % i) + break + last_time = time.time() \ No newline at end of file diff --git a/LSTM-3ConvNet.py b/LSTM-3ConvNet.py new file mode 100644 index 0000000..a73294e --- /dev/null +++ b/LSTM-3ConvNet.py @@ -0,0 +1,120 @@ +import tensorflow as tf +import time + +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') + +def rnn_layer(x, timesteps, num_hidden, weights): + x = tf.unstack(x, timesteps, 1) + lstm_cell_a = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + lstm_cell_b = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm_cell_a, lstm_cell_b, x, dtype=tf.float32) + return tf.matmul(outputs[-1], weights) + +# Placeholders +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +# ConvLayer 1 with max-pooling +W_conv1 = weight_variable([5, 5, 1, 48]) +b_conv1 = bias_variable([48]) +x_image = tf.reshape(x, [-1, 28, 28, 1]) +h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) +h_pool1 = max_pool_2x2(h_conv1) + +# ConvLayer 2 with max-pooling +W_conv2 = weight_variable([4, 4, 48, 96]) +b_conv2 = bias_variable([96]) +h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) +h_pool2 = max_pool_2x2(h_conv2) + +# ConvLayer 3 without pooling +W_conv3 = weight_variable([3, 3, 96, 192]) +b_conv3 = bias_variable([192]) +h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3) + +# LSTM Branch +h_conv2_reshape = tf.transpose(h_conv2, [0, 3, 1, 2]) +h_conv2_reshape = tf.reshape(h_conv2_reshape, [-1, 96, 81]) +W_lstm = weight_variable([512, 1024]) +h_lstm = rnn_layer(h_conv2_reshape, 96, 256, W_lstm) + +# Dense Layer 1 +W_dense1 = weight_variable([3 * 3 * 192, 1024]) +b_dense1 = bias_variable([1024]) +h_conv3 = tf.reshape(h_conv3, [-1, 3 * 3 * 192]) +h_dense1 = tf.nn.relu(tf.matmul(h_conv3, W_dense1) + h_lstm + b_dense1) + +# Dense Layer 2 +W_dense2 = weight_variable([1024, 256]) +b_dense2 = bias_variable([256]) +h_dense2 = tf.nn.relu(tf.matmul(h_dense1, W_dense2) + b_dense2) + +# Dropout +keep_prob = tf.placeholder(tf.float32) +h_dense2_drop = tf.nn.dropout(h_dense2, keep_prob) + +# Dense Layer 3 with Softmax Output +W_dense3 = weight_variable([256, 10]) +b_dense3 = bias_variable([10]) +y_conv = tf.matmul(h_dense2_drop, W_dense3) + b_dense3 + +# Training Parameters +training_rate = tf.placeholder(tf.float32) +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(training_rate).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +saver = tf.train.Saver(tf.global_variables()) + + +with tf.Session() as sess: + # Training + sess.run(tf.global_variables_initializer()) + data_location = './MNIST-LSTM-3ConvNet-DATA/MNIST_LSTM_ConvNet' + saver.restore(sess, data_location) + last_time = time.time() + rate = 0.0001 + for i in range(50000): + batch = mnist.train.next_batch(50) + sess.run(train_step, feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 0.5, training_rate: rate}) + if i % 10 == 0: + loss, acc = sess.run([cross_entropy, accuracy], feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0, training_rate: rate}) + print('Step: %d, Accuracy: %.2f, Loss: %.5f, Speed: %.1f sec/10 steps' % + (i, acc, loss, time.time() - last_time)) + last_time = time.time() + if i % 250 == 0: + current_accuracy = accuracy.eval( + feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0, training_rate: rate}) + print('- Current Test Accuracy %.4f' % current_accuracy) + saver.save(sess, data_location) + print('- Model Saved in Step %d' % i) + if current_accuracy > 0.98: + rate = 0.00003 + if current_accuracy > 0.99: + rate = 0.00001 + if current_accuracy > 0.993: + rate = 0.000003 + if current_accuracy > 0.995: + print('- Accuracy Reached 99.5% in Step %d' % i) + break + last_time = time.time() \ No newline at end of file diff --git a/LSTM-ConvNet.py b/LSTM-ConvNet.py new file mode 100644 index 0000000..08569e3 --- /dev/null +++ b/LSTM-ConvNet.py @@ -0,0 +1,104 @@ +import tensorflow as tf +import time + +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') + +def rnn_layer(x, timesteps, num_hidden, weights): + x = tf.unstack(x, timesteps, 1) + lstm_cell_a = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + lstm_cell_b = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm_cell_a, lstm_cell_b, x, dtype=tf.float32) + return tf.matmul(outputs[-1], weights) + +# Placeholders +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +# ConvLayer 1 with max-pooling +W_conv1 = weight_variable([5, 5, 1, 32]) +b_conv1 = bias_variable([32]) +x_image = tf.reshape(x, [-1, 28, 28, 1]) +h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) +h_pool1 = max_pool_2x2(h_conv1) + +# ConvLayer 2 with max-pooling +W_conv2 = weight_variable([5, 5, 32, 64]) +b_conv2 = bias_variable([64]) +h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) + +# LSTM Layer +h_conv2_reshape = tf.transpose(h_conv2, [0, 3, 1, 2]) +h_conv2_reshape = tf.reshape(h_conv2_reshape, [-1, 64, 64]) +W_dense1 = weight_variable([1024, 1024]) +b_dense1 = bias_variable([1024]) +h_lstm = rnn_layer(h_conv2_reshape, 64, 512, W_dense1) +h_dense1 = tf.nn.relu(h_lstm + b_dense1) + +# Dropout +keep_prob = tf.placeholder(tf.float32) +h_dense1_drop = tf.nn.dropout(h_dense1, keep_prob) + +# Dense Layer 2 with Softmax Output +W_dense2 = weight_variable([1024, 10]) +b_dense2 = bias_variable([10]) +y_conv = tf.matmul(h_dense1_drop, W_dense2) + b_dense2 + +# Training Parameters +training_rate = tf.placeholder(tf.float32) +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(training_rate).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +saver = tf.train.Saver(tf.global_variables()) + +with tf.Session() as sess: + # Training + sess.run(tf.global_variables_initializer()) + data_location = './MNIST-LSTM-ConvNet-DATA/MNIST_LSTM_ConvNet' + #saver.restore(sess, data_location) + last_time = time.time() + rate = 0.0001 + for i in range(100000): + batch = mnist.train.next_batch(50) + sess.run(train_step, feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 0.5, training_rate: rate}) + if i % 10 == 0: + loss, acc = sess.run([cross_entropy, accuracy], feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0, training_rate: rate}) + print('Step: %d, Accuracy: %.2f, Loss: %.5f, Speed: %.1f sec/10 steps' % + (i, acc, loss, time.time() - last_time)) + last_time = time.time() + if i % 250 == 0: + current_accuracy = accuracy.eval( + feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0, training_rate: rate}) + print('- Current Test Accuracy %.4f' % current_accuracy) + saver.save(sess, data_location) + print('- Model Saved in Step %d' % i) + if current_accuracy > 0.98: + rate = 0.00003 + if current_accuracy > 0.99: + rate = 0.000008 + if current_accuracy > 0.992: + rate = 0.000003 + if current_accuracy > 0.995: + print('- Accuracy Reached 99.5% in Step %d' % i) + break + last_time = time.time() \ No newline at end of file diff --git a/LSTM-DenseNet.py b/LSTM-DenseNet.py new file mode 100644 index 0000000..328e0c3 --- /dev/null +++ b/LSTM-DenseNet.py @@ -0,0 +1,175 @@ +import tensorflow as tf +import time + +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') + +def rnn_layer(x, timesteps, num_hidden, weights, bias): + x = tf.unstack(x, timesteps, 1) + lstm_cell_a = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + lstm_cell_b = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm_cell_a, lstm_cell_b, x, dtype=tf.float32) + return tf.nn.relu(tf.matmul(outputs[-1], weights) + bias) + +# Placeholders +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +# Main ConvLayer with max-pooling +with tf.variable_scope('ConvLayer'): + W_conv = weight_variable([5, 5, 1, 64]) + b_conv = bias_variable([64]) + x_image = tf.reshape(x, [-1, 28, 28, 1]) + h_conv = tf.nn.relu(conv2d(x_image, W_conv) + b_conv) + h_pool = max_pool_2x2(h_conv) + +with tf.variable_scope('ConvLayer1'): + # ConvLayer 1 + W_conv1 = weight_variable([3, 3, 64, 64]) + b_conv1 = bias_variable([64]) + h_conv1 = tf.nn.relu(conv2d(h_pool, W_conv1) + b_conv1) + +with tf.variable_scope('ConvLayer2'): + # ConvLayer 2 + W_conv2 = weight_variable([3, 3, 64, 64]) + b_conv2 = bias_variable([64]) + h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) + +with tf.variable_scope('ConvLayer3'): + # ConvLayer 3 + W_conv3_1 = weight_variable([3, 3, 64, 64]) + b_conv3_1 = bias_variable([64]) + h_conv3_1 = tf.nn.relu(conv2d(h_conv1, W_conv3_1) + b_conv3_1) + + W_conv3 = weight_variable([3, 3, 64, 64]) + b_conv3 = bias_variable([64]) + h_conv3 = tf.nn.relu(conv2d(h_conv2 + h_conv3_1, W_conv3) + b_conv3) + +with tf.variable_scope('ConvLayer4'): + # ConvLayer 4 + W_conv4_1 = weight_variable([3, 3, 64, 64]) + b_conv4_1 = bias_variable([64]) + h_conv4_1 = tf.nn.relu(conv2d(h_conv1, W_conv4_1) + b_conv4_1) + + W_conv4_2 = weight_variable([3, 3, 64, 64]) + b_conv4_2 = bias_variable([64]) + h_conv4_2 = tf.nn.relu(conv2d(h_conv2, W_conv4_2) + b_conv4_2) + + W_conv4 = weight_variable([3, 3, 64, 64]) + b_conv4 = bias_variable([64]) + h_conv4 = tf.nn.relu(conv2d(h_conv3 + h_conv4_1 + h_conv4_2, W_conv4) + b_conv4) + +with tf.variable_scope('ConvLayer5'): + # ConvLayer 5 + W_conv5_1 = weight_variable([3, 3, 64, 64]) + b_conv5_1 = bias_variable([64]) + h_conv5_1 = tf.nn.relu(conv2d(h_conv1, W_conv5_1) + b_conv5_1) + + W_conv5_2 = weight_variable([3, 3, 64, 64]) + b_conv5_2 = bias_variable([64]) + h_conv5_2 = tf.nn.relu(conv2d(h_conv2, W_conv5_2) + b_conv5_2) + + W_conv5_3 = weight_variable([3, 3, 64, 64]) + b_conv5_3 = bias_variable([64]) + h_conv5_3 = tf.nn.relu(conv2d(h_conv3, W_conv5_3) + b_conv5_3) + + W_conv5 = weight_variable([3, 3, 64, 64]) + b_conv5 = bias_variable([64]) + h_conv5 = tf.nn.relu(conv2d(h_conv4 + h_conv5_1 + h_conv5_2 + h_conv5_3, W_conv5) + b_conv5) + +with tf.variable_scope('ConvLayer6'): + # ConvLayer 6 + W_conv6_1 = weight_variable([3, 3, 64, 64]) + b_conv6_1 = bias_variable([64]) + h_conv6_1 = tf.nn.relu(conv2d(h_conv1, W_conv6_1) + b_conv6_1) + + W_conv6_2 = weight_variable([3, 3, 64, 64]) + b_conv6_2 = bias_variable([64]) + h_conv6_2 = tf.nn.relu(conv2d(h_conv2, W_conv6_2) + b_conv6_2) + + W_conv6_3 = weight_variable([3, 3, 64, 64]) + b_conv6_3 = bias_variable([64]) + h_conv6_3 = tf.nn.relu(conv2d(h_conv3, W_conv6_3) + b_conv6_3) + + W_conv6_4 = weight_variable([3, 3, 64, 64]) + b_conv6_4 = bias_variable([64]) + h_conv6_4 = tf.nn.relu(conv2d(h_conv4, W_conv6_4) + b_conv6_4) + + W_conv6 = weight_variable([3, 3, 64, 64]) + b_conv6 = bias_variable([64]) + h_conv6 = tf.nn.relu(conv2d(h_conv5 + h_conv6_1 + h_conv6_2 + h_conv6_3 + h_conv6_4, W_conv6) + b_conv6) + +with tf.variable_scope('LSTM'): + # LSTM Layer + h_conv6_reshape = tf.reshape(tf.transpose(h_conv6, [0, 3, 1, 2]), [-1, 64, 14 * 14]) + W_lstm = weight_variable([1024 * 2, 2048]) + b_lstm = bias_variable([2048]) + h_lstm = rnn_layer(h_conv6_reshape, 64, 1024, W_lstm, b_lstm) + +# Dropout +keep_prob = tf.placeholder(tf.float32) +h_dense1_drop = tf.nn.dropout(h_lstm, keep_prob) + +# Dense Layer 2 with Softmax Output +W_dense2 = weight_variable([2048, 10]) +b_dense2 = bias_variable([10]) +y_conv = tf.matmul(h_dense1_drop, W_dense2) + b_dense2 + +# Training Parameters +training_rate = tf.placeholder(tf.float32) +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(training_rate).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +saver = tf.train.Saver(tf.global_variables()) + +with tf.Session() as sess: + # Training + sess.run(tf.global_variables_initializer()) + data_location = './Mixed-LSTM_ConvNet-DATA/MNIST' + #saver.restore(sess, data_location) + last_time = time.time() + rate = 0.0001 + for i in range(100000): + batch = mnist.train.next_batch(50) + sess.run(train_step, feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 0.5, training_rate: rate}) + if i % 10 == 0: + loss, acc = sess.run([cross_entropy, accuracy], feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0, training_rate: rate}) + print('Step: %d, Accuracy: %.2f, Loss: %.5f, Speed: %.1f sec/10 steps' % + (i, acc, loss, time.time() - last_time)) + last_time = time.time() + if i % 1000 == 0 and i > 0: + current_accuracy = accuracy.eval( + feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0, training_rate: rate}) + print('- Current Test Accuracy %.4f' % current_accuracy) + saver.save(sess, data_location) + print('- Model Saved in Step %d' % i) + if current_accuracy > 0.98: + rate = 0.00003 + if current_accuracy > 0.99: + rate = 0.00001 + if current_accuracy > 0.993: + rate = 0.000003 + if current_accuracy > 0.995: + print('- Accuracy Reached 99.5% in Step %d' % i) + break + last_time = time.time() \ No newline at end of file diff --git a/MNIST_LeNet.py b/MNIST_LeNet.py new file mode 100644 index 0000000..d7899d9 --- /dev/null +++ b/MNIST_LeNet.py @@ -0,0 +1,65 @@ +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], + strides=[1, 2, 2, 1], padding='SAME') + +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +W_conv1 = weight_variable([5, 5, 1, 32]) +b_conv1 = bias_variable([32]) +x_image = tf.reshape(x, [-1, 28, 28, 1]) +h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) +h_pool1 = max_pool_2x2(h_conv1) + +W_conv2 = weight_variable([5, 5, 32, 64]) +b_conv2 = bias_variable([64]) +h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) +h_pool2 = max_pool_2x2(h_conv2) + +W_fc1 = weight_variable([7 * 7 * 64, 1024]) +b_fc1 = bias_variable([1024]) + +h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64]) +h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) + +keep_prob = tf.placeholder(tf.float32) +h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) + +W_fc2 = weight_variable([1024, 10]) +b_fc2 = bias_variable([10]) +y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 + +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + +with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for i in range(5000): + batch = mnist.train.next_batch(50) + if i % 10 == 0: + train_accuracy = accuracy.eval(feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0}) + print('step %d, training accuracy %g' % (i, train_accuracy)) + train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) + + print('test accuracy %g' % accuracy.eval(feed_dict={ + x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})) \ No newline at end of file diff --git a/MNIST_Softmax.py b/MNIST_Softmax.py new file mode 100644 index 0000000..64afb4e --- /dev/null +++ b/MNIST_Softmax.py @@ -0,0 +1,35 @@ +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("MNIST_data/", one_hot = True) + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +x = tf.placeholder(tf.float32, [None, 784]) + +W1 = weight_variable([784, 10]) +b1 = bias_variable([10]) + +y = tf.matmul(x,W1) + b1 +y_ = tf.placeholder(tf.float32, [None, 10]) + +cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) +train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) + +correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) + +sess = tf.InteractiveSession() +tf.global_variables_initializer().run() +for i in range(30000): + batch_xs, batch_ys = mnist.train.next_batch(100) + if i % 10 == 0: + accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + train_accuracy = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}) + print('Step: %d Training Accuracy: %g' %(i, train_accuracy)) + sess.run(train_step, feed_dict = {x: batch_xs, y_: batch_ys}) + diff --git a/Mix-2LSTM-3ConvNet.py b/Mix-2LSTM-3ConvNet.py new file mode 100644 index 0000000..c425406 --- /dev/null +++ b/Mix-2LSTM-3ConvNet.py @@ -0,0 +1,133 @@ +import tensorflow as tf +import time + +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') + +# Placeholders +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +# ConvLayer 1 with max-pooling +W_conv1 = weight_variable([5, 5, 1, 64]) +b_conv1 = bias_variable([64]) +x_image = tf.reshape(x, [-1, 28, 28, 1]) +h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) +h_pool1 = max_pool_2x2(h_conv1) + +# ConvLayer 2 with max-pooling +W_conv2 = weight_variable([4, 4, 64, 128]) +b_conv2 = bias_variable([128]) +h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) +h_pool2 = max_pool_2x2(h_conv2) + +# ConvLayer 3 without pooling +W_conv3 = weight_variable([3, 3, 128, 192]) +b_conv3 = bias_variable([192]) +h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3) + +# LSTM Branch from ConvLayer 2 +with tf.variable_scope('LSTM2'): + lstm2_num_hidden = 256 + lstm2_timesteps = 128 + h_conv2_reshape = tf.transpose(h_conv2, [0, 3, 1, 2]) + h_conv2_reshape = tf.reshape(h_conv2_reshape, [-1, lstm2_timesteps, 81]) + W_lstm2 = weight_variable([2 * lstm2_num_hidden, 1536]) + h_conv2_reshape = tf.unstack(h_conv2_reshape, lstm2_timesteps, 1) + lstm2_cell_fw = tf.contrib.rnn.BasicLSTMCell(lstm2_num_hidden, forget_bias=1.0) + lstm2_cell_bw = tf.contrib.rnn.BasicLSTMCell(lstm2_num_hidden, forget_bias=1.0) + lstm2_outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm2_cell_fw, lstm2_cell_bw, h_conv2_reshape, dtype=tf.float32) +h_lstm2 = tf.matmul(lstm2_outputs[-1], W_lstm2) + +# LSTM Branch from ConvLayer 1 +with tf.variable_scope('LSTM1'): + lstm1_num_hidden = 128 + lstm1_timesteps = 64 + h_conv1_reshape = tf.transpose(h_conv1, [0, 3, 1, 2]) + h_conv1_reshape = tf.reshape(h_conv1_reshape, [-1, lstm1_timesteps, 24 * 24]) + W_lstm1 = weight_variable([2 * lstm1_num_hidden, 1536]) + h_conv1_reshape = tf.unstack(h_conv1_reshape, lstm1_timesteps, 1) + lstm1_cell_fw = tf.contrib.rnn.BasicLSTMCell(lstm1_num_hidden, forget_bias=1.0) + lstm1_cell_bw = tf.contrib.rnn.BasicLSTMCell(lstm1_num_hidden, forget_bias=1.0) + lstm1_outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm1_cell_fw, lstm1_cell_bw, h_conv1_reshape, dtype=tf.float32) +h_lstm1 = tf.matmul(lstm1_outputs[-1], W_lstm1) + +# Dense Layer 1 +W_dense1 = weight_variable([3 * 3 * 192, 1536]) +b_dense1 = bias_variable([1536]) +h_conv3 = tf.reshape(h_conv3, [-1, 3 * 3 * 192]) +h_dense1 = tf.nn.relu(tf.matmul(h_conv3, W_dense1) + h_lstm1 + h_lstm2 + b_dense1) + +# Dense Layer 2 +W_dense2 = weight_variable([1536, 256]) +b_dense2 = bias_variable([256]) +h_dense2 = tf.nn.relu(tf.matmul(h_dense1, W_dense2) + b_dense2) + +# Dropout +keep_prob = tf.placeholder(tf.float32) +h_dense2_drop = tf.nn.dropout(h_dense2, keep_prob) + +# Dense Layer 3 with Softmax Output +W_dense3 = weight_variable([256, 10]) +b_dense3 = bias_variable([10]) +y_conv = tf.matmul(h_dense2_drop, W_dense3) + b_dense3 + +# Training Parameters +training_rate = tf.placeholder(tf.float32) +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(training_rate).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +saver = tf.train.Saver(tf.global_variables()) + +with tf.Session() as sess: + # Training + sess.run(tf.global_variables_initializer()) + data_location = './MNIST_LSTM_ConvNet' + #saver.restore(sess, data_location) + last_time = time.time() + rate = 0.0001 + for i in range(100000): + batch = mnist.train.next_batch(50) + sess.run(train_step, feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 0.5, training_rate: rate}) + if i % 10 == 0: + loss, acc = sess.run([cross_entropy, accuracy], feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0, training_rate: rate}) + print('Step: %d, Accuracy: %.2f, Loss: %.5f, Speed: %.1f sec/10 steps' % + (i, acc, loss, time.time() - last_time)) + last_time = time.time() + if i % 1000 == 0 and i > 0: + current_accuracy = accuracy.eval( + feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0, training_rate: rate}) + print('- Current Test Accuracy %.4f' % current_accuracy) + saver.save(sess, data_location) + print('- Model Saved in Step %d' % i) + if current_accuracy > 0.98: + rate = 0.00003 + if current_accuracy > 0.99: + rate = 0.00001 + if current_accuracy > 0.993: + rate = 0.000003 + if current_accuracy > 0.995: + print('- Accuracy Reached 99.5% in Step %d' % i) + break + last_time = time.time() \ No newline at end of file diff --git a/Mix-5LSTM_5ConvNet.py b/Mix-5LSTM_5ConvNet.py new file mode 100644 index 0000000..2c650e8 --- /dev/null +++ b/Mix-5LSTM_5ConvNet.py @@ -0,0 +1,187 @@ +import tensorflow as tf +import time + +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets('MNIST_data', one_hot=True) +sess = tf.InteractiveSession() + +def weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') + +def max_pool_2x2(x): + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') + +def rnn_layer(x, timesteps, num_hidden, weights, bias): + x = tf.unstack(x, timesteps, 1) + lstm_cell_a = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + lstm_cell_b = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn( + lstm_cell_a, lstm_cell_b, x, dtype=tf.float32) + return tf.nn.relu(tf.matmul(outputs[-1], weights) + bias) + +def rnn_mix(x, timesteps, num_hidden, weights): + x = tf.unstack(x, timesteps, 1) + lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) + outputs, _ = tf.contrib.rnn.static_rnn( + lstm_cell, x, dtype=tf.float32) + return tf.matmul(outputs[-1], weights) + +# Placeholders +x = tf.placeholder(tf.float32, shape=[None, 784]) +y_ = tf.placeholder(tf.float32, shape=[None, 10]) + +# Main ConvLayer with max-pooling +with tf.variable_scope('ConvLayer'): + W_conv = weight_variable([5, 5, 1, 64]) + b_conv = bias_variable([64]) + x_image = tf.reshape(x, [-1, 28, 28, 1]) + h_conv = tf.nn.relu(conv2d(x_image, W_conv) + b_conv) + h_pool = max_pool_2x2(h_conv) + +with tf.variable_scope('ConvLayers'): + # ConvLayer 1 + W_conv1 = weight_variable([3, 3, 64, 64]) + b_conv1 = bias_variable([64]) + h_conv1 = tf.nn.relu(conv2d(h_pool, W_conv1) + b_conv1) + + # ConvLayer 2 + W_conv2 = weight_variable([3, 3, 64, 64]) + b_conv2 = bias_variable([64]) + h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) + + # ConvLayer 3 + W_conv3 = weight_variable([3, 3, 64, 64]) + b_conv3 = bias_variable([64]) + h_conv3 = tf.nn.relu(conv2d(h_conv2, W_conv3) + b_conv3) + + # ConvLayer 4 + W_conv4 = weight_variable([3, 3, 64, 64]) + b_conv4 = bias_variable([64]) + h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4) + + # ConvLayer 5 + W_conv5 = weight_variable([3, 3, 64, 64]) + b_conv5 = bias_variable([64]) + h_conv5 = tf.nn.relu(conv2d(h_conv4, W_conv5) + b_conv5) + +# LSTM Branches Params +lstm_num_hidden = 64 +lstm_timesteps = 64 +lstm_num_output = 512 + +# LSTM Branch from ConvLayer 1 +with tf.variable_scope('LSTM1'): + h_conv1_reshape = tf.reshape(tf.transpose(h_conv1, [0, 3, 1, 2]), [-1, lstm_timesteps, 14 * 14]) + W_lstm1 = weight_variable([lstm_num_hidden * 2, lstm_num_output]) + b_lstm1 = bias_variable([lstm_num_output]) + h_lstm1 = rnn_layer(h_conv1_reshape, lstm_timesteps, lstm_num_hidden, W_lstm1, b_lstm1) + +# LSTM Branch from ConvLayer 2 +with tf.variable_scope('LSTM2'): + h_conv2_reshape = tf.reshape(tf.transpose(h_conv1, [0, 3, 1, 2]), [-1, lstm_timesteps, 14 * 14]) + W_lstm2 = weight_variable([lstm_num_hidden * 2, lstm_num_output]) + b_lstm2 = bias_variable([lstm_num_output]) + h_lstm2 = rnn_layer(h_conv2_reshape, lstm_timesteps, lstm_num_hidden, W_lstm2, b_lstm2) + +# LSTM Branch from ConvLayer 3 +with tf.variable_scope('LSTM3'): + h_conv3_reshape = tf.reshape(tf.transpose(h_conv3, [0, 3, 1, 2]), [-1, lstm_timesteps, 14 * 14]) + W_lstm3 = weight_variable([lstm_num_hidden * 2, lstm_num_output]) + b_lstm3 = bias_variable([lstm_num_output]) + h_lstm3 = rnn_layer(h_conv3_reshape, lstm_timesteps, lstm_num_hidden, W_lstm3, b_lstm3) + +# LSTM Branch from ConvLayer 4 +with tf.variable_scope('LSTM4'): + h_conv4_reshape = tf.reshape(tf.transpose(h_conv4, [0, 3, 1, 2]), [-1, lstm_timesteps, 14 * 14]) + W_lstm4 = weight_variable([lstm_num_hidden * 2, lstm_num_output]) + b_lstm4 = bias_variable([lstm_num_output]) + h_lstm4 = rnn_layer(h_conv4_reshape, lstm_timesteps, lstm_num_hidden, W_lstm4, b_lstm4) + +# LSTM Branch from ConvLayer 5 +with tf.variable_scope('LSTM5'): + h_conv5_reshape = tf.reshape(tf.transpose(h_conv5, [0, 3, 1, 2]), [-1, lstm_timesteps, 14 * 14]) + W_lstm5 = weight_variable([lstm_num_hidden * 2, lstm_num_output]) + b_lstm5 = bias_variable([lstm_num_output]) + h_lstm5 = rnn_layer(h_conv5_reshape, lstm_timesteps, lstm_num_hidden, W_lstm5, b_lstm5) + +# LSTM Mixed +with tf.variable_scope('LSTM'): + lstm_mix_num_hidden = 64 + b_mix = bias_variable([lstm_num_output]) + h_mix = tf.stack([h_lstm1, h_lstm2, h_lstm3, h_lstm4, h_lstm5]) + print(h_mix) + h_mix_reshape = tf.transpose(h_mix, [1, 0, 2]) + print(h_mix_reshape) + W_mix = weight_variable([lstm_mix_num_hidden, 4096]) + h_mix_out = rnn_mix(h_mix_reshape, 5, lstm_mix_num_hidden, W_mix) + +# Dense Layer 1 +W_dense1 = weight_variable([14 * 14 * 64, 4096]) +b_dense1 = bias_variable([4096]) +h_conv5 = tf.reshape(h_conv5, [-1, 14 * 14 * 64]) +h_dense1 = tf.nn.relu(tf.matmul(h_conv5, W_dense1) + h_mix_out + b_dense1) + +# Dense Layer 2 +W_dense2 = weight_variable([4096, 1024]) +b_dense2 = bias_variable([1024]) +h_dense2 = tf.nn.relu(tf.matmul(h_dense1, W_dense2) + b_dense2) + +# Dropout +keep_prob = tf.placeholder(tf.float32) +h_dense2_drop = tf.nn.dropout(h_dense2, keep_prob) + +# Dense Layer 3 with Softmax Output +W_dense3 = weight_variable([1024, 10]) +b_dense3 = bias_variable([10]) +y_conv = tf.matmul(h_dense2_drop, W_dense3) + b_dense3 + +# Training Parameters +training_rate = tf.placeholder(tf.float32) +cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) +train_step = tf.train.AdamOptimizer(training_rate).minimize(cross_entropy) +correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) +accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +saver = tf.train.Saver(tf.global_variables()) + +with tf.Session() as sess: + # Training + sess.run(tf.global_variables_initializer()) + data_location = './Mixed-LSTM_ConvNet-DATA/MNIST' + #saver.restore(sess, data_location) + last_time = time.time() + rate = 0.0001 + for i in range(100000): + batch = mnist.train.next_batch(50) + sess.run(train_step, feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 0.5, training_rate: rate}) + if i % 10 == 0: + loss, acc = sess.run([cross_entropy, accuracy], feed_dict={ + x: batch[0], y_: batch[1], keep_prob: 1.0, training_rate: rate}) + print('Step: %d, Accuracy: %.2f, Loss: %.5f, Speed: %.1f sec/10 steps' % + (i, acc, loss, time.time() - last_time)) + last_time = time.time() + if i % 1000 == 0 and i > 0: + current_accuracy = accuracy.eval( + feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0, training_rate: rate}) + print('- Current Test Accuracy %.4f' % current_accuracy) + saver.save(sess, data_location) + print('- Model Saved in Step %d' % i) + if current_accuracy > 0.98: + rate = 0.00003 + if current_accuracy > 0.99: + rate = 0.00001 + if current_accuracy > 0.993: + rate = 0.000003 + if current_accuracy > 0.995: + print('- Accuracy Reached 99.5% in Step %d' % i) + break + last_time = time.time() \ No newline at end of file