Strange gradients.

[airalcorn2]

Hi. My colleague (@czw0078) and I have been using your neural renderer, and we've noticed some pretty strange behavior during optimization. A minimal working example demonstrating the odd behavior can be found below.

import chainer
import neural_renderer as nr
import numpy as np
import scipy.misc

from chainer import Chain


class Model(Chain):
    def __init__(self, input_obj, initial_z):
        super(Model, self).__init__()

        # Load object.
        (vertices, faces) = nr.load_obj(input_obj)

        vertices = vertices[None, :, :]
        faces = faces[None, :, :]
        texture_size = 2
        textures = np.ones((1, faces.shape[1], texture_size, texture_size, texture_size, 3), "float32")

        chainer.cuda.get_device_from_id(0).use()
        self.vertices = chainer.cuda.to_gpu(vertices)
        self.faces = chainer.cuda.to_gpu(faces)
        self.textures = chainer.cuda.to_gpu(textures)

        self.x = self.vertices[0][:, 0]
        self.y = self.vertices[0][:, 1]
        self.z = self.vertices[0][:, 2]

        # Camera parameters.
        camera_distance = 2.732
        elevation = 0
        azimuth = 0
        self.camera_position = np.array(nr.get_points_from_angles(camera_distance, elevation, azimuth), dtype=np.float32)

        # Adjust renderer.
        renderer = nr.Renderer()
        renderer.camera_direction = np.array(renderer.camera_direction, dtype=np.float32)
        renderer.camera_direction = chainer.cuda.to_gpu(renderer.camera_direction)
        renderer.camera_mode = "look"
        renderer.eye = self.camera_position
        renderer.eye = chainer.cuda.to_gpu(renderer.eye)
        renderer.viewing_angle = 8.123
        self.renderer = renderer

        # Optimization direction.
        with self.init_scope():
            self.z_delta = chainer.Parameter(np.array([initial_z], dtype=np.float32))

    def __call__(self):
        new_z = chainer.functions.add(self.z, chainer.functions.repeat(self.z_delta, len(self.z)))

        vertices = chainer.functions.stack((self.x, self.y, new_z), axis=1)[None, :, :]
        image = self.renderer.render(vertices, self.faces, self.textures)

        return (new_z, image)


def main():
    input_obj = "./examples/data/teapot.obj"
    model = Model(input_obj, 4)
    model.to_gpu(0)

    gen_teapot = False

    for sign in [1, -1]:
        model.cleargrads()
        (new_z, images) = model()

        if gen_teapot and sign == 1:
            image = images.data.get()[0].transpose((1, 2, 0))
            scipy.misc.toimage(image, cmin=0, cmax=1).save("my_teapot.png")

        print("Current z_delta: {0}".format(model.z_delta.data.get()[0]))
        assert model.z_delta.grad is None

        loss = sign * chainer.functions.batch_l2_norm_squared(images)[0]
        loss.backward(retain_grad=True)
        print("Loss: {0}".format(loss.data.get().item()))

        print("z_delta derivative after .backward(): {0}".format(model.z_delta.grad.get()[0]))


if __name__ == "__main__":
    main()

The example code produces the following output:

Current z_delta: 4.0
Loss: 21549.5859375
z_delta derivative after .backward(): -68343.7109375
Current z_delta: 4.0
Loss: -21549.5859375
z_delta derivative after .backward(): -13344.5039062

when using the image below as a starting point.

As you can see, the gradients have the same sign despite using opposite loss functions. Any insight you could provide on this behavior would be greatly appreciated. Thank you.

The following Dockerfile was used to generate the environment used for the code above.

FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
RUN apt-get update && apt-get install -y --no-install-recommends \
         wget \
         build-essential \
         cmake \
         nano \
         less \
         git \
         curl \
         libjpeg-dev \
         libpng-dev \
         imagemagick \
         python \
         python-dev \
         python-setuptools \
         python-pip \
         python-wheel
RUN pip install chainer cupy scikit-image tqdm imageio ipython==5.8.0

[airalcorn2]

@nkolot suggested commenting out the two instances of:

if (diff_grad <= 0)
    continue;

in the rasterizer's backward CUDA kernel, which seems to work.