-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
272 lines (215 loc) · 10.2 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import random
import math
import PIL.Image
from scipy.ndimage.filters import gaussian_filter
import inception5h
model = inception5h.Inception5h()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
session = tf.Session(graph=model.graph,
config=tf.ConfigProto(gpu_options=gpu_options))
def load_image(filename):
image = PIL.Image.open(filename)
return np.float32(image)
def save_image(image, filename):
# Ensure the pixel-values are between 0 and 255.
image = np.clip(image, 0.0, 255.0)
# Convert to bytes.
image = image.astype(np.uint8)
# Write the image-file in jpeg-format.
with open(filename, 'wb') as file:
PIL.Image.fromarray(image).save(file, 'jpeg')
def plot_image(image):
# Assume the pixel-values are scaled between 0 and 255.
if False:
# Convert the pixel-values to the range between 0.0 and 1.0
image = np.clip(image/255.0, 0.0, 1.0)
# Plot using matplotlib.
plt.imshow(image, interpolation='lanczos')
plt.show()
else:
# Ensure the pixel-values are between 0 and 255.
image = np.clip(image, 0.0, 255.0)
# Convert pixels to bytes.
image = image.astype(np.uint8)
# Convert to a PIL-image and display it.
plt.imshow(image, interpolation='lanczos')
plt.show()
def normalize_image(x):
# Get the min and max values for all pixels in the input.
x_min = x.min()
x_max = x.max()
# Normalize so all values are between 0.0 and 1.0
x_norm = (x - x_min) / (x_max - x_min)
return x_norm
def plot_gradient(gradient):
# Normalize the gradient so it is between 0.0 and 1.0
gradient_normalized = normalize_image(gradient)
# Plot the normalized gradient.
plt.imshow(gradient_normalized, interpolation='bilinear')
plt.show()
def get_tile_size(num_pixels, tile_size=400):
"""
num_pixels is the number of pixels in a dimension of the image.
tile_size is the desired tile-size.
"""
# How many times can we repeat a tile of the desired size.
num_tiles = int(round(num_pixels / tile_size))
# Ensure that there is at least 1 tile.
num_tiles = max(1, num_tiles)
# The actual tile-size.
actual_tile_size = math.ceil(num_pixels / num_tiles)
return actual_tile_size
def tiled_gradient(gradient, image, tile_size=400):
# Allocate an array for the gradient of the entire image.
grad = np.zeros_like(image)
# Number of pixels for the x- and y-axes.
x_max, y_max, _ = image.shape
# Tile-size for the x-axis.
x_tile_size = get_tile_size(num_pixels=x_max, tile_size=tile_size)
# 1/4 of the tile-size.
x_tile_size4 = x_tile_size // 4
# Tile-size for the y-axis.
y_tile_size = get_tile_size(num_pixels=y_max, tile_size=tile_size)
# 1/4 of the tile-size
y_tile_size4 = y_tile_size // 4
# Random start-position for the tiles on the x-axis.
# The random value is between -3/4 and -1/4 of the tile-size.
# This is so the border-tiles are at least 1/4 of the tile-size,
# otherwise the tiles may be too small which creates noisy gradients.
x_start = random.randint(-3*x_tile_size4, -x_tile_size4)
while x_start < x_max:
# End-position for the current tile.
x_end = x_start + x_tile_size
# Ensure the tile's start- and end-positions are valid.
x_start_lim = max(x_start, 0)
x_end_lim = min(x_end, x_max)
# Random start-position for the tiles on the y-axis.
# The random value is between -3/4 and -1/4 of the tile-size.
y_start = random.randint(-3*y_tile_size4, -y_tile_size4)
while y_start < y_max:
# End-position for the current tile.
y_end = y_start + y_tile_size
# Ensure the tile's start- and end-positions are valid.
y_start_lim = max(y_start, 0)
y_end_lim = min(y_end, y_max)
# Get the image-tile.
img_tile = image[x_start_lim:x_end_lim,
y_start_lim:y_end_lim, :]
# Create a feed-dict with the image-tile.
feed_dict = model.create_feed_dict(image=img_tile)
# Use TensorFlow to calculate the gradient-value.
g = session.run(gradient, feed_dict=feed_dict)
# Normalize the gradient for the tile. This is
# necessary because the tiles may have very different
# values. Normalizing gives a more coherent gradient.
g /= (np.std(g) + 1e-8)
# Store the tile's gradient at the appropriate location.
grad[x_start_lim:x_end_lim,
y_start_lim:y_end_lim, :] = g
# Advance the start-position for the y-axis.
y_start = y_end
# Advance the start-position for the x-axis.
x_start = x_end
return grad
def optimize_image(layer_tensor, image,
num_iterations=10, step_size=3.0, tile_size=400,
show_gradient=False):
"""
Use gradient ascent to optimize an image so it maximizes the
mean value of the given layer_tensor.
Parameters:
layer_tensor: Reference to a tensor that will be maximized.
image: Input image used as the starting point.
num_iterations: Number of optimization iterations to perform.
step_size: Scale for each step of the gradient ascent.
tile_size: Size of the tiles when calculating the gradient.
show_gradient: Plot the gradient in each iteration.
"""
# Copy the image so we don't overwrite the original image.
img = image.copy()
print("Processing image: ")
# Use TensorFlow to get the mathematical function for the
# gradient of the given layer-tensor with regard to the
# input image. This may cause TensorFlow to add the same
# math-expressions to the graph each time this function is called.
# It may use a lot of RAM and could be moved outside the function.
gradient = model.get_gradient(layer_tensor)
for i in range(num_iterations):
# Calculate the value of the gradient.
# This tells us how to change the image so as to
# maximize the mean of the given layer-tensor.
grad = tiled_gradient(gradient=gradient, image=img)
# Blur the gradient with different amounts and add
# them together. The blur amount is also increased
# during the optimization. This was found to give
# nice, smooth images. You can try and change the formulas.
# The blur-amount is called sigma (0=no blur, 1=low blur, etc.)
# We could call gaussian_filter(grad, sigma=(sigma, sigma, 0.0))
# which would not blur the colour-channel. This tends to
# give psychadelic / pastel colours in the resulting images.
# When the colour-channel is also blurred the colours of the
# input image are mostly retained in the output image.
sigma = (i * 4.0) / num_iterations + 0.5
grad_smooth1 = gaussian_filter(grad, sigma=sigma)
grad_smooth2 = gaussian_filter(grad, sigma=sigma*2)
grad_smooth3 = gaussian_filter(grad, sigma=sigma*0.5)
grad = (grad_smooth1 + grad_smooth2 + grad_smooth3)
step_size_scaled = step_size / (np.std(grad) + 1e-8)
# Update the image by following the gradient.
img = img + (grad * step_size_scaled)
if show_gradient:
# Print statistics for the gradient.
msg = "Gradient min: {0:>9.6f}, max: {1:>9.6f}, stepsize: {2:>9.2f}"
print(msg.format(grad.min(), grad.max(), step_size_scaled))
# Plot the gradient.
plot_gradient(grad)
else:
# Otherwise show a little progress-indicator.
print(". ", end="")
return img
def recursive_optimize(layer_tensor, image,
num_repeats=4, rescale_factor=0.7, blend=0.2,
num_iterations=10, step_size=3.0,
tile_size=400):
"""
Recursively blur and downscale the input image.
Each downscaled image is run through the optimize_image()
function to amplify the patterns that the Inception model sees.
Parameters:
image: Input image used as the starting point.
rescale_factor: Downscaling factor for the image.
num_repeats: Number of times to downscale the image.
blend: Factor for blending the original and processed images.
Parameters passed to optimize_image():
layer_tensor: Reference to a tensor that will be maximized.
num_iterations: Number of optimization iterations to perform.
step_size: Scale for each step of the gradient ascent.
tile_size: Size of the tiles when calculating the gradient.
"""
# Do a recursive step?
if num_repeats > 0:
# Blur the input image to prevent artifacts when downscaling.
# The blur amount is controlled by sigma. Note that the
# colour-channel is not blurred as it would make the image gray.
sigma = 0.5
img_blur = gaussian_filter(image, sigma=(sigma, sigma, 0.0))
img_result = recursive_optimize(layer_tensor=layer_tensor,
image=img_blur,
num_repeats=num_repeats-1,
rescale_factor=rescale_factor,
blend=blend,
num_iterations=num_iterations,
step_size=step_size,
tile_size=tile_size)
image = blend * image + (1.0 - blend) * img_result
print("\nRecursive level:", num_repeats)
# Process the image using the DeepDream algorithm.
img_result = optimize_image(layer_tensor=layer_tensor,
image=image,
num_iterations=num_iterations,
step_size=step_size,
tile_size=tile_size)
return img_result