DEV: realism score

.gitignore

@@ -1,5 +1,6 @@
 # Model files
 *.npz
+*.caffemodel
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

load_caffe_model.py

@@ -0,0 +1,51 @@
+from __future__ import print_function
+import argparse
+
+import numpy as np
+
+from chainer import Variable, link, serializers
+
+from chainer.links import caffe
+
+from model import RealismCNN
+
+def cnn2fcn(src, dst):
+    print('Copying layers %s -> %s:' % (src.__class__.__name__, dst.__class__.__name__))
+
+    for child in src.children():
+        dst_child = dst['fc8' if child.name.startswith('fc8') else child.name]
+
+        if isinstance(child, link.Link):
+            print('Copying {} ...'.format(child.name))
+
+            if child.name.startswith('fc'):
+                dst_child.__dict__['W'].data[...] = np.reshape(child.__dict__['W'].data, dst_child.__dict__['W'].data.shape)
+                dst_child.__dict__['b'].data[...] = np.reshape(child.__dict__['b'].data, dst_child.__dict__['b'].data.shape)
+            else:
+                dst_child.copyparams(child)
+
+            print('\tlayer: %s -> %s' % (child.name, dst_child.name))
+
+    return dst
+
+def main():
+    parser = argparse.ArgumentParser(description='Load caffe model for chainer')
+    parser.add_argument('--caffe_model_path', default='models/realismCNN_all_iter3.caffemodel', help='Path for caffe model')
+    parser.add_argument('--chainer_model_path', default='models/realismCNN_all_iter3.npz', help='Path for saving chainer model')
+    args = parser.parse_args()
+
+    print('Load caffe model from {} ...'.format(args.caffe_model_path))
+    caffe_model = caffe.CaffeFunction(args.caffe_model_path)
+    print('Load caffe model, DONE')
+
+    print('\nTurn CNN into FCN, start ...\n')
+    chainer_model = RealismCNN()
+    chainer_model(Variable(np.zeros((1, 3, 227, 227), dtype=np.float32), volatile='on'))
+    chainer_model = cnn2fcn(caffe_model, chainer_model)
+
+    print('\nTurn CNN into FCN, DONE. Save to {} ...'.format(args.chainer_model_path))
+    serializers.save_npz(args.chainer_model_path, chainer_model)
+
+
+if __name__ == '__main__':
+    main()

model.py

@@ -122,4 +122,60 @@ def __call__(self, x, test=False):
         h = self.encode(x, test=test)
         h = self.decode(h, test=test)
 
+        return h
+
+class RealismCNN(chainer.Chain):
+    def __init__(self, w_init=None):
+        super(RealismCNN, self).__init__(
+            conv1_1=L.Convolution2D(None, 64, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv1_2=L.Convolution2D(None, 64, ksize=3, stride=1, pad=1, initialW=w_init),
+
+            conv2_1=L.Convolution2D(None, 128, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv2_2=L.Convolution2D(None, 128, ksize=3, stride=1, pad=1, initialW=w_init),
+
+            conv3_1=L.Convolution2D(None, 256, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv3_2=L.Convolution2D(None, 256, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv3_3=L.Convolution2D(None, 256, ksize=3, stride=1, pad=1, initialW=w_init),
+
+            conv4_1=L.Convolution2D(None, 512, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv4_2=L.Convolution2D(None, 512, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv4_3=L.Convolution2D(None, 512, ksize=3, stride=1, pad=1, initialW=w_init),
+
+            conv5_1=L.Convolution2D(None, 512, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv5_2=L.Convolution2D(None, 512, ksize=3, stride=1, pad=1, initialW=w_init),
+            conv5_3=L.Convolution2D(None, 512, ksize=3, stride=1, pad=1, initialW=w_init),
+
+            fc6=L.Convolution2D(None, 4096, ksize=7, stride=1, pad=0, initialW=w_init),
+            fc7=L.Convolution2D(None, 4096, ksize=1, stride=1, pad=0, initialW=w_init),
+            fc8=L.Convolution2D(None, 2, ksize=1, stride=1, pad=0, initialW=w_init)
+        )
+
+    def __call__(self, x, dropout=True):
+        h = F.relu(self.conv1_1(x))
+        h = F.relu(self.conv1_2(h))
+        h = F.max_pooling_2d(h, ksize=2, stride=2)
+
+        h = F.relu(self.conv2_1(h))
+        h = F.relu(self.conv2_2(h))
+        h = F.max_pooling_2d(h, ksize=2, stride=2)
+
+        h = F.relu(self.conv3_1(h))
+        h = F.relu(self.conv3_2(h))
+        h = F.relu(self.conv3_3(h))
+        h = F.max_pooling_2d(h, ksize=2, stride=2)
+
+        h = F.relu(self.conv4_1(h))
+        h = F.relu(self.conv4_2(h))
+        h = F.relu(self.conv4_3(h))
+        h = F.max_pooling_2d(h, ksize=2, stride=2)
+
+        h = F.relu(self.conv5_1(h))
+        h = F.relu(self.conv5_2(h))
+        h = F.relu(self.conv5_3(h))
+        h = F.max_pooling_2d(h, ksize=2, stride=2)
+
+        h = F.dropout(F.relu(self.fc6(h)), train=dropout)
+        h = F.dropout(F.relu(self.fc7(h)), train=dropout)
+        h = self.fc8(h)
+
         return h

predict_realism.py

@@ -0,0 +1,50 @@
+from __future__ import print_function
+import argparse
+
+import chainer
+from chainer import Variable, serializers
+
+from model import RealismCNN
+
+from utils import im_preprocess_vgg
+
+import numpy as np
+
+def main():
+    parser = argparse.ArgumentParser(description='Predict a list of images wheather realistic or not')
+    parser.add_argument('--gpu', type=int, default=0, help='GPU ID (negative value indicates CPU)')
+    parser.add_argument('--model_path', default='models/realismCNN_all_iter3.npz', help='Path for pretrained model')
+    parser.add_argument('--list_path', help='Path for file storing image list')
+    parser.add_argument('--batch_size', type=int, default=10, help='Batchsize of 1 iteration')
+    parser.add_argument('--load_size', type=int, default=256, help='Scale image to load_size')
+    parser.add_argument('--result_path', default='result.txt', help='Path for file storing results')
+    args = parser.parse_args()
+
+    model = RealismCNN()
+    print('Load pretrained model from {} ...'.format(args.model_path))
+    serializers.load_npz(args.model_path, model)
+    if args.gpu >= 0:
+        chainer.cuda.get_device(args.gpu).use()  # Make a specified GPU current
+        model.to_gpu()                           # Copy the model to the GPU
+
+    print('Load images from {} ...'.format(args.list_path))
+    dataset = chainer.datasets.ImageDataset(paths=args.list_path, root='')
+    print('{} images in total loaded'.format(len(dataset)))
+    data_iterator = chainer.iterators.SerialIterator(dataset, args.batch_size, repeat=False, shuffle=False)
+
+    scores = np.zeros((0, 2))
+    for idx, batch in enumerate(data_iterator):
+        print('Processing batch {}->{}/{} ...'.format(idx*args.batch_size+1, min(len(dataset), (idx+1)*args.batch_size), len(dataset)))
+        batch = [im_preprocess_vgg(np.transpose(im, [1, 2, 0]), args.load_size) for im in batch]
+        batch = Variable(chainer.dataset.concat_examples(batch, args.gpu), volatile='on')
+        result = chainer.cuda.to_cpu(model(batch, dropout=False).data)
+        scores = np.vstack((scores, np.mean(result, axis=(2, 3))))
+
+    print('Processing DONE !')
+    print('Saving result to {} ...'.format(args.result_path))
+    with open(args.result_path, 'w') as f:
+        for score in scores:
+            f.write('{},{}\n'.format(score[0], score[1]))
+
+if __name__ == '__main__':
+    main()

utils.py

@@ -2,6 +2,22 @@
 
 import numpy as np
 
+from skimage.transform import resize
+
+mean = np.array([103.939, 116.779, 123.68])
+def im_preprocess_vgg(im, load_size=None, sub_mean=True, dtype=None, order=1, preserve_range=True):
+    if not dtype:
+        dtype = im.dtype
+    if load_size:
+        im = resize(im, (load_size, load_size), order=order, preserve_range=preserve_range, mode='constant')
+    if im.ndim == 2:
+        im = im[:, :, np.newaxis]
+    im = im[:, :, ::-1]
+    if sub_mean:
+        im = im - mean
+    im = np.asarray(np.transpose(im, [2, 0, 1]), dtype=dtype)
+    return im
+
 def make_grid(tensor, padding=2):
     """
     Given a 4D mini-batch Tensor of shape (B x C x H x W), makes a grid of images