Permalink
Browse files

added "neural style" loss. turn it on with `--nstyle-w=1` bumped ver…

…sion
1 parent 968d02b commit c064d5d871e046af54fc75662060a76ddf73ec91 @awentzonline committed Mar 17, 2016
Showing with 40 additions and 4 deletions.
  1. +2 −1 README.md
  2. +2 −2 image_analogy/argparser.py
  3. +24 −0 image_analogy/losses/neural_style.py
  4. +11 −0 image_analogy/models/nnf.py
  5. +1 −1 setup.py
View
@@ -113,7 +113,8 @@ Parameters
them into one image (slower/more memory but maybe more accurate)
* --model Select the patch matching model ('patchmatch' or 'brute') patchmatch is
the default and requires less GPU memory but is less accurate then brute.
-
+ * --nstyle-w Weight for neural style loss between A' and B'
+ * --nstyle-layers Comma-separated list of layer names to be used for the neural style
The analogy loss is the amount of influence of B -> A -> A' -> B'. It's a
structure-preserving mapping of Image B into A' via A.
@@ -71,8 +71,8 @@ def parse_args():
default=['conv3_1', 'conv4_1'],
help='Comma-separated list of layer names to be used for the content loss')
parser.add_argument('--nstyle-w', dest='neural_style_weight', type=float,
- default=0.5, help='Weight for neural style loss between A\' and B\'')
- parser.add_argument('-nstyle-layers', dest='neural_style_layers', action=CommaSplitAction,
+ default=0.0, help='Weight for neural style loss between A\' and B\'')
+ parser.add_argument('--nstyle-layers', dest='neural_style_layers', action=CommaSplitAction,
default=['conv2_1', 'conv3_1', 'conv4_1', 'conv5_1'],
help='Comma-separated list of layer names to be used for the neural style')
parser.add_argument('--patch-size', dest='patch_size', type=int,
@@ -0,0 +1,24 @@
+'''This is from the keras neural style example.'''
+from keras import backend as K
+
+
+# the gram matrix of an image tensor (feature-wise outer product)
+def gram_matrix(x):
+ assert K.ndim(x) == 3
+ features = K.batch_flatten(x)
+ gram = K.dot(features, K.transpose(features))
+ return gram
+
+
+# the "style loss" is designed to maintain
+# the style of the reference image in the generated image.
+# It is based on the gram matrices (which capture style) of
+# feature maps from the style reference image
+# and from the generated image
+def neural_style_loss(style, combination, num_channels, img_width, img_height):
+ assert K.ndim(style) == 3
+ assert K.ndim(combination) == 3
+ S = gram_matrix(style)
+ C = gram_matrix(combination)
+ size = img_width * img_height
+ return K.sum(K.square(S - C)) / (4. * (num_channels ** 2) * (size ** 2))
@@ -5,6 +5,7 @@
from image_analogy.losses.core import content_loss
from image_analogy.losses.nnf import nnf_analogy_loss, NNFState, PatchMatcher
+from image_analogy.losses.neural_style import neural_style_loss
from .base import BaseModel
@@ -95,4 +96,14 @@ def build_loss(self, a_image, ap_image, b_image):
cl = content_loss(bp_features, b_features)
loss += self.args.b_bp_content_weight / len(self.args.b_content_layers) * cl
+ if self.args.neural_style_weight != 0.0:
+ for layer_name in self.args.neural_style_layers:
+ ap_image_features = K.variable(all_ap_image_features[layer_name][0])
+ layer_features = self.get_layer_output(layer_name)
+ layer_shape = self.get_layer_output_shape(layer_name)
+ # current combined output
+ combination_features = layer_features[0, :, :, :]
+ nsl = neural_style_loss(ap_image_features, combination_features, 3, self.output_shape[-2], self.output_shape[-1])
+ loss += (self.args.neural_style_weight / len(self.args.neural_style_layers)) * nsl
+
return loss
View
@@ -4,7 +4,7 @@
setup(
name='neural-image-analogies',
- version='0.0.8',
+ version='0.0.9',
description='Generate image analogies with a deep neural network.',
author='Adam Wentz',
author_email='adam@adamwentz.com',

0 comments on commit c064d5d

Please sign in to comment.