Finished

Author: Aftaab99

Checkpoints/weights-epoch-6loss-0.005/checkpoint

@@ -1,13 +1,13 @@
 ## Denoising Autoencoder
 Implementation of a denoising autoencoder trained on the RENOIR dataset(MI 3 images). 
+[Model Architecture](https://github.com/titu1994/Image-Super-Resolution/blob/master/architectures/Denoise.png)
 
 ## Setting up locally
 
 	pip install -r requirements.txt
 
-
 ## Dataset
-50x50px patches were taken from the reference and noisy images in the dataset. I've serialised these into TFRecords, which can be downloaded using,
+33x33px patches were taken from the reference and noisy images in the dataset. I've serialised these into TFRecords, which can be downloaded using, 
 
 	python download_data.py
 
@@ -16,30 +16,27 @@ This will download the train and validation records required for training.
 ## Training and inference
 1. For training you can run,
 
-		python train.py -e <num_of_epochs> -c <checkpoint_after> -v <validation_enabled, 1 or 0>
-	Example:
-
-		python train.py -e 50 -c 5 -v 1
-Default values are training for 10 epochs, checkpointing every 1 epoch with validation enabled
+		python train.py -e <num_of_epochs>
 
 2. For inference,
 
 		python predict.py -i <input_file> -o <output_file>
 
+The model doesn't have a fixed input shape so for smaller images(<400x400px), the entire image vector is feed into the model. For larger images, I've used a window of size 33x33px for generating the output image.
 
 ## Results
-I've trained the model for only 6 epochs(which is a very very small fraction of what a lot of papers recommend), so the results aren't particularly good. 
+The model was trained for 25 epochs on Google colab's GPU(NVIDIA Tesla k8).
 
 1. Reference:
 ![Reference Image](https://github.com/Aftaab99/DenoisingAutoencoder/blob/master/images/reference.bmp  "Reference Image")
 
 2. Noisy
-![Noisy Image](https://github.com/Aftaab99/DenoisingAutoencoder/blob/master/images/noisy.png  "Noisy Image")
+![Noisy Image](https://github.com/Aftaab99/DenoisingAutoencoder/blob/master/images/noisy.bmp "Noisy Image")
 
 3. Denoised
 
-![Denoised Image](https://github.com/Aftaab99/DenoisingAutoencoder/blob/master/images/denoised.png  "Denoised Image")
+![Denoised Image](https://github.com/Aftaab99/DenoisingAutoencoder/blob/master/images/denoised.bmp  "Denoised Image")
 
 ### References
 1. J. Anaya, A. Barbu. RENOIR - A Dataset for Real Low-Light Image Noise Reduction.([arxiv](https://arxiv.org/abs/1409.8230))
-
+2. Image Restoration Using ConvolutionalAuto-encoders with Symmetric Skip Connections-Xiao-Jiao Mao, Chunhua Shen, Yu-Bin Yang([code](https://github.com/titu1994/Image-Super-Resolution/))

Checkpoints/weights-epoch-6loss-0.005/weights-epoch-6loss-0.005.ckpt.data-00000-of-00001

@@ -13,11 +13,11 @@ def _bytes_feature(value):
 
 def extract_patches(ref, noisy):
 	patch_list = []
-	for x in range(50, 3000, 50):
-		for y in range(50, 3000, 50):
-			patch_ref = ref[x - 50:x, y - 50:y, :]
-			patch_noisy = noisy[x - 50:x, y - 50:y, :]
-			if patch_ref.shape[0] != 50 or patch_ref.shape[1] != 50:
+	for x in range(33, 3000, 33):
+		for y in range(33, 3000, 33):
+			patch_ref = ref[x - 33:x, y - 33:y, :]
+			patch_noisy = noisy[x - 33:x, y - 33:y, :]
+			if patch_ref.shape[0] != 33 or patch_ref.shape[1] != 33:
 				continue
 			patch_list.append({'ref': patch_ref, 'noisy': patch_noisy})
 	return patch_list

Checkpoints/weights-epoch-6loss-0.005/weights-epoch-6loss-0.005.ckpt.index

@@ -47,4 +47,4 @@ def save_response_content(response, destination):
 		download_file_from_google_drive(train_file_id, train_file_dest)
 
 		print('Downloading validation records...')
-		download_file_from_google_drive(val_file_id, val_file_dest)
+		download_file_from_google_drive(val_file_id, val_file_dest)

Checkpoints/weights-epoch-6loss-0.005/weights-epoch-6loss-0.005.ckpt.meta

@@ -0,0 +1,105 @@
+import tensorflow as tf
+from tensorflow.python.keras.layers import Convolution2D, Convolution2DTranspose, merge, Input
+from tensorflow.python.keras.models import Model
+from PIL import Image
+import numpy as np
+
+
+class DAE:
+
+	def __init__(self):
+		input_1 = Input(shape=(None, None, 3))
+		conv_1 = Convolution2D(64, kernel_size=(3, 3), padding='same', activation='relu')(input_1)
+		conv_2 = Convolution2D(64, kernel_size=(5, 5), padding='same', activation='relu')(conv_1)
+		dconv_1 = Convolution2DTranspose(64, kernel_size=(3, 3), padding='same', activation='relu')(conv_2)
+		merge_1 = merge.maximum([dconv_1, conv_2])
+		dconv_2 = Convolution2DTranspose(64, kernel_size=(3, 3), padding="same", activation='relu')(merge_1)
+		merge_2 = merge.maximum([dconv_2, conv_1])
+		conv3 = Convolution2D(3, (5, 5), padding="same", activation='relu')(merge_2)
+
+		self.model = Model(inputs=input_1, outputs=conv3)
+		self.model.compile(optimizer='adam', loss='mean_squared_error', metrics=['acc'])
+		self.model.summary()
+		self.batch_size = 128
+
+	def load_model_weights(self, save_path):
+		self.model.load_weights(save_path)
+
+	def save_model(self, save_path):
+		self.model.save(save_path)
+
+	def train(self, epochs):
+		n_records = 0
+		for _ in tf.python_io.tf_record_iterator('Data/train.tfrecords'):
+			n_records += 1
+		x, y = self.input_fn('Data/train.tfrecords')
+		self.model.fit(x, y, epochs=epochs, steps_per_epoch=n_records // self.batch_size)
+
+	def denoise_patch(self, image_patch):
+		image_patch = image_patch[np.newaxis, ...]
+		output_t = self.model.predict(image_patch)
+		output_t = np.array(output_t)
+		output_t = np.clip(output_t, 0, 255)
+		return output_t
+
+	def denoise(self, image_array):
+		dim = image_array.shape
+		img_h = dim[0]
+		img_w = dim[1]
+		d_image = image_array
+
+		if img_w * img_h < 400 * 400:
+			image_array = image_array[np.newaxis, ...]
+			a = np.clip(self.model.predict(image_array), 0, 255).astype('uint8')
+			a = a.squeeze(0)
+			img1 = Image.fromarray(a)
+			return img1
+
+		for y in range(0, img_w, 33):
+			for x in range(0, img_h, 33):
+				patch = image_array[x:x + 33, y:y + 33, :]
+				if patch.shape[0] == 33 and patch.shape[1] == 33:
+					patch = self.denoise_patch(patch)
+					d_image[x:x + 33, y:y + 33, :] = patch
+
+				elif patch.shape[0] < 33 and patch.shape[1] < 33:
+					patch = self.denoise_patch(patch)
+					d_image[x:, y:, :] = patch
+
+				elif patch.shape[1] < 33 and patch.shape[0] == 33:
+					l = patch.shape[1]
+					patch = self.denoise_patch(patch)
+					d_image[x:x + 33, y:y + l, :] = patch
+
+				elif patch.shape[0] < 33 and patch.shape[1] == 33:
+					l = patch.shape[0]
+					patch = self.denoise_patch(patch)
+					d_image[x:x + l, y:y + 33, :] = patch[0:l, :, :]
+
+		d_image = Image.fromarray(d_image.astype('uint8'))
+		return d_image
+
+	def parser(self, record):
+		keys_to_feature = {
+			"reference": tf.FixedLenFeature([], tf.string),
+			"noisy": tf.FixedLenFeature([], tf.string)
+		}
+		parsed = tf.parse_single_example(record, keys_to_feature)
+		target_image = tf.decode_raw(parsed['reference'], tf.uint8)
+		target_image = tf.cast(target_image, tf.float32)
+
+		target_image = tf.reshape(target_image, shape=[33, 33, 3])
+		noisy_image = tf.decode_raw(parsed['noisy'], tf.uint8)
+		noisy_image = tf.cast(noisy_image, tf.float32)
+		noisy_image = tf.reshape(noisy_image, shape=[33, 33, 3])
+		return noisy_image, target_image
+
+	def input_fn(self, filename):
+		dataset = tf.data.TFRecordDataset(filename)
+		dataset = dataset.map(self.parser)
+		dataset = dataset.repeat()
+
+		dataset = dataset.batch(self.batch_size)
+		iterator = dataset.make_one_shot_iterator()
+		noisy_batch, target_batch = iterator.get_next()
+		return noisy_batch, target_batch

README.md

@@ -1,9 +1,8 @@
-import tensorflow as tf
 import numpy as np
 from PIL import Image
 import sys
 import getopt
-from model import DenoisingAutoEncoder
+from model_keras import DAE
 
 
 def main(argv):
@@ -29,13 +28,12 @@ def main(argv):
 		elif opt in ('-o', '--ofile'):
 			output_file = arg
 
-	input_image = Image.open(input_file).convert('RGB').resize([3000, 3000])
-	input_image_array = np.array(input_image).reshape(3000, 3000, 3)
-	input_image_array = input_image_array / 255.0
-	d = DenoisingAutoEncoder((50, 50, 3), (None, 50, 50, 3), tf.train.AdamOptimizer(), False)
-
+	input_image = Image.open(input_file).convert('RGB')
+	input_image_array = np.array(input_image)
+	d = DAE()
+	d.load_model_weights('model_weights.hdf5')
 	output_image = d.denoise(input_image_array)
-	output_image.save(output_file, format='PNG')
+	output_image.save(output_file, format='BMP')
 
 
 if __name__ == '__main__':

create_datasets.py

@@ -3,3 +3,4 @@ sklearn
 tensorflow
 pillow
 requests
+keras