add Neural_style_transfer code

Neural_style_transfer/fix_produce_image.py

@@ -0,0 +1,69 @@
+import tensorflow as tf
+import matplotlib.pyplot as plt
+
+def load_raw_img(path_to_img):
+    img = tf.io.read_file(path_to_img)
+    img = tf.image.decode_image(img, channels=3)
+    img = tf.image.convert_image_dtype(img, tf.float32)
+    img = img[tf.newaxis, :]
+    return img
+
+def load_img_and_reshape(path_to_img):
+    max_dim = 512
+    img = tf.io.read_file(path_to_img)
+    img = tf.image.decode_image(img, channels=3)
+    img = tf.image.convert_image_dtype(img, tf.float32)
+
+    shape = tf.cast(tf.shape(img)[:-1], tf.float32)
+    tf.print(f"raw_img_shape {shape}")
+    long_dim = max(shape)
+    scale = max_dim / long_dim
+
+    new_shape = tf.cast(shape * scale, tf.int32)
+    tf.print(f"new_shape {new_shape}")
+    img = tf.image.resize(img, new_shape)
+    img = img[tf.newaxis, :]
+    return img
+
+
+def imshow(image, title=None):
+    if len(image.shape) > 3:
+        image = tf.squeeze(image, axis=0)
+
+    plt.imshow(image)
+    #if title:
+    #    plt.title(title)
+    plt.gca().xaxis.set_major_locator(plt.NullLocator())
+    plt.gca().yaxis.set_major_locator(plt.NullLocator())
+    plt.subplots_adjust(top=1, bottom=0, left=0, right=1, hspace=0, wspace=0)
+    plt.margins(0, 0)
+    plt.axis('off')
+    plt.savefig(f"{title}.png")
+    plt.close()
+
+
+def recovery_to_raw_image_shape(path_to_raw_img, path_to_produce_img):
+    raw_img = tf.io.read_file(path_to_raw_img)
+    raw_img = tf.image.decode_image(raw_img, channels=3)
+    tf.print(f"raw_img_shape {tf.shape(raw_img)[:-1]}")
+
+    produce_img = tf.io.read_file(path_to_produce_img)
+    produce_img = tf.image.decode_image(produce_img, channels=3)
+    produce_img = tf.image.convert_image_dtype(produce_img, tf.float32)
+    tf.print(f"produce_img_shape {tf.shape(produce_img)[:-1]}")
+    produce_img = tf.image.resize(produce_img, tf.shape(raw_img)[:-1])
+    tf.print(f"recovery produce_img_shape {tf.shape(produce_img)[:-1]}")
+    produce_img = produce_img[tf.newaxis, :]
+    return produce_img
+
+content_path = "/home/b418a/disk1/pycharm_room/yuanxiao/my_lenovo_P50s/Neural_style_transfer/datasets/content.jpg"
+path_to_produce_img = "/home/b418a/disk1/pycharm_room/yuanxiao/my_lenovo_P50s/Neural_style_transfer/Reshape Content Image.png"
+
+raw_content_image = load_raw_img(content_path)
+imshow(raw_content_image, 'Raw Content Image')
+
+reshape_content_image = load_img_and_reshape(content_path)
+imshow(reshape_content_image, 'Reshape Content Image')
+
+recovery_content_image = recovery_to_raw_image_shape(content_path, path_to_produce_img)
+imshow(recovery_content_image, 'recovery Content Image')

Neural_style_transfer/image_classifier_VGG19_model.py

@@ -0,0 +1,48 @@
+import tensorflow as tf
+import numpy as np
+
+
+class ImageClassifierBaseOnVGG19(object):
+    """https://keras.io/applications/#vgg19"""
+    def __init__(self):
+        self.VGG19 = tf.keras.applications.VGG19(include_top=True, weights='imagenet')
+        self.labels_path = tf.keras.utils.get_file(
+            'ImageNetLabels.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')
+        self.imagenet_labels = np.array(open(self.labels_path).read().splitlines())
+
+    def load_img(self, path_to_img):
+        """Define a function to load an image and limit its maximum dimension to 512 pixels."""
+        max_dim = 512
+        img = tf.io.read_file(path_to_img)
+        img = tf.image.decode_image(img, channels=3)
+        img = tf.image.convert_image_dtype(img, tf.float32)
+
+        shape = tf.cast(tf.shape(img)[:-1], tf.float32)
+        long_dim = max(shape)
+        scale = max_dim / long_dim
+
+        new_shape = tf.cast(shape * scale, tf.int32)
+
+        img = tf.image.resize(img, new_shape)
+        img = img[tf.newaxis, :]
+        return img
+
+    def classify(self, image_path, top_k=10):
+        image = self.load_img(image_path)
+        x = tf.keras.applications.vgg19.preprocess_input(image * 255)
+        x = tf.image.resize(x, (224, 224))  # The default input size for VGG19 model is 224x224.
+        results = self.VGG19(x)
+        decode_predictions = tf.keras.applications.vgg19.decode_predictions(results.numpy())
+        predict_img_label_list = self.imagenet_labels[np.argsort(results)[0, ::-1][:top_k] + 1]
+        return predict_img_label_list, decode_predictions
+
+
+if __name__ == "__main__":
+    img_classifier = ImageClassifierBaseOnVGG19()
+    image_path = tf.keras.utils.get_file(fname='samoyed_dog.jpg',
+                                         origin='https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1561387878331&di=033973e3a9e7fb2581914e5409055b8c&imgtype=0&src=http%3A%2F%2Fgss0.baidu.com%2F-vo3dSag_xI4khGko9WTAnF6hhy%2Fzhidao%2Fpic%2Fitem%2Fd043ad4bd11373f08779bd0ba60f4bfbfaed04db.jpg',
+                                         cache_dir='.')
+
+    predict_img_label_list, decode_predictions = img_classifier.classify(image_path, top_k=5)
+    print(f"predict_img_label_list {predict_img_label_list}")
+    print(f"decode_predictions {decode_predictions}")

Neural_style_transfer/image_style_transfer_model.py

@@ -0,0 +1,93 @@
+import tensorflow as tf
+
+class StyleContentModel(tf.keras.models.Model):
+    def __init__(self, style_layers=None, content_layers=None, show_all_optional_layer_name=False):
+        super(StyleContentModel, self).__init__()
+
+        if style_layers is None:
+            # Style layer of interest
+            style_layers = ['block1_conv1',
+                            'block2_conv1',
+                            'block3_conv1',
+                            'block4_conv1',
+                            'block5_conv1']
+        if content_layers is None:
+            # Content layer where will pull our feature maps
+            content_layers = ['block5_conv2']
+
+        self.vgg = self.vgg_layers(style_layers + content_layers, show_all_optional_layer_name)
+        self.style_layers = style_layers
+        self.content_layers = content_layers
+        self.num_style_layers = len(style_layers)
+        self.num_content_layers = len(content_layers)
+        self.vgg.trainable = False
+
+
+    def vgg_layers(self, layer_names, show_all_optional_layer_name):
+        """ Creates a vgg model that returns a list of intermediate output values."""
+        # Load our model. Load pretrained VGG, trained on imagenet data
+        vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
+        vgg.trainable = False
+        if show_all_optional_layer_name:
+            for layer in self.vgg.layers:
+                print(layer.name)
+
+        outputs = [vgg.get_layer(name).output for name in layer_names]
+
+        model = tf.keras.Model([vgg.input], outputs)
+        return model
+
+    def gram_matrix(self, input_tensor):
+        result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)
+        input_shape = tf.shape(input_tensor)
+        num_locations = tf.cast(input_shape[1] * input_shape[2], tf.float32)
+        return result / (num_locations)
+
+    def call(self, inputs):
+        "Expects float input in [0,1]"
+        inputs = inputs * 255.0
+        preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)
+        outputs = self.vgg(preprocessed_input)
+        style_outputs, content_outputs = (outputs[:self.num_style_layers],
+                                          outputs[self.num_style_layers:])
+
+        style_outputs = [self.gram_matrix(style_output)
+                         for style_output in style_outputs]
+
+        content_dict = {content_name: value
+                        for content_name, value
+                        in zip(self.content_layers, content_outputs)}
+
+        style_dict = {style_name: value
+                      for style_name, value
+                      in zip(self.style_layers, style_outputs)}
+
+        return {'content': content_dict, 'style': style_dict}
+
+
+
+if __name__=="__main__":
+    extractor = StyleContentModel()
+    import numpy as np
+    fack_image = tf.constant(np.random.random(size=(1, 244, 244, 3)), dtype=tf.float32)
+    #fack_image = tf.keras.applications.vgg19.preprocess_input(fack_image)
+    results = extractor(tf.constant(fack_image))
+
+    style_results = results['style']
+
+    print('Styles:')
+    for name, output in sorted(results['style'].items()):
+        print("  ", name)
+        print("    shape: ", output.numpy().shape)
+        # print("    min: ", output.numpy().min())
+        # print("    max: ", output.numpy().max())
+        # print("    mean: ", output.numpy().mean())
+        print()
+
+    print("Contents:")
+    for name, output in sorted(results['content'].items()):
+        print("  ", name)
+        print("    shape: ", output.numpy().shape)
+        # print("    min: ", output.numpy().min())
+        # print("    max: ", output.numpy().max())
+        # print("    mean: ", output.numpy().mean())

Neural_style_transfer/train_and_inference_by_image_transfer_model.py

@@ -0,0 +1,120 @@
+import tensorflow as tf
+import matplotlib.pyplot as plt
+import os
+import time
+
+from image_style_transfer_model import StyleContentModel
+
+
+def load_img(path_to_img):
+    """Define a function to load an image and limit its maximum dimension to 512 pixels."""
+    max_dim = 512
+    img = tf.io.read_file(path_to_img)
+    img = tf.image.decode_image(img, channels=3)
+    img = tf.image.convert_image_dtype(img, tf.float32)
+
+    shape = tf.cast(tf.shape(img)[:-1], tf.float32)
+    long_dim = max(shape)
+    scale = max_dim / long_dim
+
+    new_shape = tf.cast(shape * scale, tf.int32)
+
+    img = tf.image.resize(img, new_shape)
+    img = img[tf.newaxis, :]
+    return img
+
+
+class ImageTransferBaseOnVGG19(object):
+
+    def __init__(self, style_layers=None, content_layers=None, show_all_optional_layer_name=False, learning_rate=0.02,
+                 beta_1=0.99, epsilon=1e-1):
+        self.extractor = StyleContentModel(style_layers, content_layers, show_all_optional_layer_name)
+        self.opt = tf.optimizers.Adam(learning_rate, beta_1, epsilon)
+
+    def clip_0_1(self, image):
+        return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)
+
+    def VGG19_imshow(self, image, title=None, produce_image_file=None):
+        if len(image.shape) > 3:
+            image = tf.squeeze(image, axis=0)
+        if title:
+            plt.title(title)
+        plt.imshow(image)
+        plt.savefig(os.path.join(produce_image_file, title + ".png"))
+        plt.close()
+
+    @tf.function()
+    def train_step(self, produce_image, total_variation_weight=1e8):
+        with tf.GradientTape() as tape:
+            outputs = self.extractor(produce_image)
+            loss = self.style_content_loss(outputs)
+            loss += total_variation_weight * self.total_variation_loss(produce_image)
+
+        grad = tape.gradient(loss, produce_image)
+        self.opt.apply_gradients([(grad, produce_image)])
+        produce_image.assign(self.clip_0_1(produce_image))
+
+    def transfer(self, content_image, style_image, produce_image_file="produce_images", epochs=5, steps_per_epoch=100):
+        if not os.path.exists(produce_image_file):
+            os.mkdir(produce_image_file)
+        self.content_image = content_image
+        self.style_targets = self.extractor(style_image)['style']
+        self.content_targets = self.extractor(content_image)['content']
+
+        # Define a tf.Variable to contain the image to optimize.
+        # To make this quick, initialize it with the content image
+        # (the tf.Variable must be the same shape as the content image):
+        produce_image = tf.Variable(self.content_image)
+
+        start = time.time()
+        step = 0
+        for n in range(epochs):
+            for m in range(steps_per_epoch):
+                step += 1
+                self.train_step(produce_image)
+                tf.print(".", end='')
+            self.VGG19_imshow(produce_image.read_value(),
+                              title=f"{step}_steps", produce_image_file=produce_image_file)
+        end = time.time()
+        tf.print("Total time: {:.1f}".format(end - start))
+
+    def style_content_loss(self, outputs, style_weight=1e-2, content_weight=1e4):
+        style_outputs = outputs['style']
+        content_outputs = outputs['content']
+        style_loss = tf.add_n([tf.reduce_mean((style_outputs[name] - self.style_targets[name]) ** 2)
+                               for name in style_outputs.keys()])
+        style_loss *= style_weight / self.extractor.num_style_layers
+
+        content_loss = tf.add_n([tf.reduce_mean((content_outputs[name] - self.content_targets[name]) ** 2)
+                                 for name in content_outputs.keys()])
+        content_loss *= content_weight / self.extractor.num_content_layers
+        loss = style_loss + content_loss
+        return loss
+
+    def total_variation_loss(self, image):
+        def high_pass_x_y(image):
+            x_var = image[:, :, 1:, :] - image[:, :, :-1, :]
+            y_var = image[:, 1:, :, :] - image[:, :-1, :, :]
+            return x_var, y_var
+
+        x_deltas, y_deltas = high_pass_x_y(image)
+        return tf.reduce_mean(x_deltas ** 2) + tf.reduce_mean(y_deltas ** 2)
+
+
+if __name__ == "__main__":
+    content_image_url = "https://raw.githubusercontent.com/ckmarkoh/neuralart_tensorflow/master/images/Taipei101.jpg"
+    style_image_url = "https://raw.githubusercontent.com/ckmarkoh/neuralart_tensorflow/master/images/StarryNight.jpg"
+
+    produce_image_file = "produce_images"
+    epochs = 10
+    steps_per_epoch = 100
+
+    content_path = tf.keras.utils.get_file(fname='content.jpg', origin=content_image_url, cache_dir='.')
+    style_path = tf.keras.utils.get_file(fname='style.jpg', origin=style_image_url, cache_dir='.')
+
+    content_image = load_img(content_path)
+    style_image = load_img(style_path)
+
+    img_transfer = ImageTransferBaseOnVGG19()
+    img_transfer.transfer(content_image, style_image, produce_image_file,
+                          epochs, steps_per_epoch)

README.md

@@ -1,12 +1,13 @@
 # DeepNude-an-Image-to-Image-technology
-> This resource includes the TensorFlow2 implementation of models such as pix2pix, CycleGAN, DCGAN, and VAE. 本资源包含pix2pix、CycleGAN、DCGAN、VAE等模型的TensorFlow2实现。
 
 ![](paper_images/Image2Image_logo.png)
 
 This repository contains the pix2pixHD algorithms(proposed by NVIDIA) of DeepNude, and more importantly, the general image generation theory and practice behind DeepNude.
 
 这个仓库包含DeepNude的pix2pixHD(由英伟达提出)算法，更重要的是DeepNude背后通用的图像生成理论与实践研究。
 
+> This resource includes the TensorFlow2 implementation of image generation models such as pix2pix, CycleGAN, DCGAN, and VAE. 本资源包含pix2pix、CycleGAN、DCGAN、VAE等图像生成模型的 [TensorFlow2](https://www.tensorflow.org/) 实现。
+
 ## What is DeepNude? 什么是 DeepNude?
 DeepNude uses a slightly modified version of the [pix2pixHD](https://github.com/NVIDIA/pix2pixHD) GAN architecture, quoted from deepnude_official. pix2pixHD is a general-purpose Image2Image technology proposed by NVIDIA. Obviously, DeepNude is the wrong application of artificial intelligence technology, but it uses Image2Image technology for researchers and developers working in other fields such as fashion, film and visual effects.
 
@@ -157,7 +158,7 @@ In the image interface of [Image_Inpainting(NVIDIA_2018).mp4](https://github.com
 ---
 
 ## Image Generation Practice Research
-> More models and functions will be added in the future.
+> These models are based on the latest implementation of TensorFlow2.
 
 This section explains DeepNude-related AI/Deep Learning (especially computer vision) code practices, and if you like to experiment, enjoy them.
 
@@ -199,6 +200,10 @@ Use VGG19 to achieve image style migration effects, such as photo changes to oil
 
 [Click Start Experience 6](Neural_style_transfer)
 
+..........................................................................
+
+如果你是使用[PaddlePaddle](https://github.com/PaddlePaddle/Paddle)的用户，可以参考以上模型的Paddle版本 [图像生成模型库 PaddleGAN](https://github.com/PaddlePaddle/models/tree/develop/PaddleCV/PaddleGAN)。
+
 ---
 
 ## Future