Merge pull request #10 from avik-pal/paper-updates

Update the paper as per the review

Manifest.toml

@@ -41,9 +41,9 @@ version = "0.8.10"
 
 [[BinaryProvider]]
 deps = ["Libdl", "Logging", "SHA"]
-git-tree-sha1 = "8153fd64131cd00a79544bb23788877741f627bb"
+git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648"
 uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
-version = "0.5.5"
+version = "0.5.6"
 
 [[CSTParser]]
 deps = ["Tokenize"]
@@ -124,10 +124,10 @@ uuid = "dc8bdbbb-1ca9-579f-8c36-e416f6a65cce"
 version = "0.2.0"
 
 [[DataStructures]]
-deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"]
-git-tree-sha1 = "ca971f03e146cf144a9e2f2ce59674f5bf0e8038"
+deps = ["InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "0809951a1774dc724da22d26e4289bbaab77809a"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.15.0"
+version = "0.17.0"
 
 [[Dates]]
 deps = ["Printf"]
@@ -184,9 +184,9 @@ version = "0.6.1"
 
 [[Flux]]
 deps = ["AbstractTrees", "Adapt", "CodecZlib", "Colors", "DelimitedFiles", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "Requires", "SHA", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"]
-git-tree-sha1 = "5142e80eba87bf7b6d549c7b588c5cdde27769c1"
-repo-rev = "sf/zygote_updated"
-repo-url = "https://github.com/thebhatman/Flux.jl"
+git-tree-sha1 = "c5234e2a0e6c5ae0faf86ade929fbee0df26a463"
+repo-rev = "zygote"
+repo-url = "https://github.com/FluxML/Flux.jl.git"
 uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 version = "0.8.3"
 
@@ -242,9 +242,9 @@ version = "0.6.3"
 
 [[ImageMagick]]
 deps = ["BinaryProvider", "ColorTypes", "FileIO", "FixedPointNumbers", "ImageCore", "InteractiveUtils", "Libdl", "Pkg", "Random"]
-git-tree-sha1 = "2bb39ea007cb81672547d171de3f40b4d6b8ba32"
+git-tree-sha1 = "6e1efa8bd5123e18a082c5580c7b8d4a287e1af4"
 uuid = "6218d12a-5da1-5696-b52f-db25d2ecc6d1"
-version = "0.7.4"
+version = "0.7.5"
 
 [[ImageMetadata]]
 deps = ["AxisArrays", "ColorVectorSpace", "Colors", "FixedPointNumbers", "ImageAxes", "ImageCore", "IndirectArrays"]

paper/header.tex

@@ -1,9 +1,10 @@
 % **************GENERATED FILE, DO NOT EDIT**************
 
-\title{RayTracer.jl: A General Purpose Differentiable Renderer}
+\title{RayTracer.jl: A Differentiable Renderer that supports Parameter Optimization for Scene Reconstruction}
 
 \author[1]{Avik Pal}
-\affil[1]{Indian Institute of Technology, Kanpur}
+\affil[1]{Indian Institute of Technology Kanpur}
+
+\keywords{Julia, Differentiable Programming, Automatic Differentiation, Inverse Graphics, Differentiable Rendering}
 
-\keywords{Julia, Differentiable Programming, Deep Learning, Automatic Differentiation, Inverse Graphics}
 

paper/paper.bbl

@@ -9,36 +9,43 @@ Jeff Bezanson, Alan Edelman, Stefan Karpinski, and Viral~B Shah.
 Julia: A fresh approach to numerical computing.
 {\em SIAM review}, 59(1):65--98, 2017.
 
+\bibitem{retinaface}
+Jiankang Deng, Jia Guo, Yuxiang Zhou, Jinke Yu, Irene Kotsia, and Stefanos
+  Zafeiriou.
+Retinaface: Single-stage dense face localisation in the wild.
+{\em CoRR}, abs/1905.00641, 2019.
+
 \bibitem{DBLP:journals/corr/abs-1810-07951}
 Michael Innes.
-Don't unroll adjoint: Differentiating ssa-form programs.
+Don't {U}nroll {A}djoint: {D}ifferentiating {SSA-F}orm {P}rograms.
 {\em CoRR}, abs/1810.07951, 2018.
 
 \bibitem{Flux.jl-2018}
 Michael Innes, Elliot Saba, Keno Fischer, Dhairya Gandhi, Marco~Concetto
   Rudilosso, Neethu~Mariya Joy, Tejan Karmali, Avik Pal, and Viral Shah.
-Fashionable modelling with flux.
+Fashionable {M}odelling with {F}lux.
 {\em CoRR}, abs/1811.01457, 2018.
 
 \bibitem{Kay:1986:RTC:15922.15916}
 Timothy~L. Kay and James~T. Kajiya.
-Ray tracing complex scenes.
+Ray {T}racing {C}omplex {S}cenes.
 In {\em Proceedings of the 13th Annual Conference on Computer Graphics and
   Interactive Techniques}, SIGGRAPH '86, pages 269--278, New York, NY, USA,
   1986. ACM.
 
 \bibitem{kingma2014adam}
-Diederik~P. Kingma and Jimmy Ba.
-Adam: A method for stochastic optimization, 2014.
+Diederik Kingma and Jimmy Ba.
+Adam: {A} {M}ethod for {S}tochastic {O}ptimization.
+{\em International Conference on Learning Representations}, 12 2014.
 
 \bibitem{Li:2018:DMC}
 Tzu-Mao Li, Miika Aittala, Fr{\'e}do Durand, and Jaakko Lehtinen.
-Differentiable monte carlo ray tracing through edge sampling.
+Differentiable {M}onte {C}arlo {R}ay {T}racing through {E}dge {S}ampling.
 {\em ACM Trans. Graph. (Proc. SIGGRAPH Asia)}, 37(6):222:1--222:11, 2018.
 
 \bibitem{RayTracer.jl}
 Avik Pal.
-Raytracer.jl.
+Ray{T}racer.jl.
 \url{https://github.com/avik-pal/RayTracer.jl}, 2019.
 
 \end{thebibliography}

paper/paper.yml

@@ -1,4 +1,4 @@
-title: "RayTracer.jl: A General Purpose Differentiable Renderer"
+title: "RayTracer.jl: A Differentiable Renderer that supports Parameter Optimization for Scene Reconstruction"
 keywords:
   - Julia
   - Differentiable Programming

paper/ref.bib

@@ -10,7 +10,7 @@ @article{bezanson2017julia
 }
 
 @article{Li:2018:DMC,
-    title = {Differentiable Monte Carlo Ray Tracing through Edge Sampling},
+    title = {Differentiable {M}onte {C}arlo {R}ay {T}racing through {E}dge {S}ampling},
     author = {Li, Tzu-Mao and Aittala, Miika and Durand, Fr{\'e}do and Lehtinen, Jaakko},
     journal = {ACM Trans. Graph. (Proc. SIGGRAPH Asia)},
     volume = {37},
@@ -19,18 +19,18 @@ @article{Li:2018:DMC
     year = {2018}
 }
 
-@misc{kingma2014adam,
-    title={Adam: A Method for Stochastic Optimization},
-    author={Diederik P. Kingma and Jimmy Ba},
-    year={2014},
-    eprint={1412.6980},
-    archivePrefix={arXiv},
-    primaryClass={cs.LG}
+@article{kingma2014adam,
+    author = {Kingma, Diederik and Ba, Jimmy},
+    year = {2014},
+    month = {12},
+    pages = {},
+    title = {Adam: {A} {M}ethod for {S}tochastic {O}ptimization},
+    journal = {International Conference on Learning Representations}
 }
 
 @misc{RayTracer.jl,
   author = {Avik Pal},
-  title = {RayTracer.jl},
+  title = {Ray{T}racer.jl},
   year = {2019},
   publisher = {GitHub},
   journal = {GitHub repository},
@@ -57,7 +57,7 @@ @article{Flux.jl-2018
                Tejan Karmali and
                Avik Pal and
                Viral Shah},
-  title     = {Fashionable Modelling with Flux},
+  title     = {Fashionable {M}odelling with {F}lux},
   journal   = {CoRR},
   volume    = {abs/1811.01457},
   year      = {2018},
@@ -71,7 +71,7 @@ @article{Flux.jl-2018
 
 @inproceedings{Kay:1986:RTC:15922.15916,
  author = {Kay, Timothy L. and Kajiya, James T.},
- title = {Ray Tracing Complex Scenes},
+ title = {Ray {T}racing {C}omplex {S}cenes},
  booktitle = {Proceedings of the 13th Annual Conference on Computer Graphics and Interactive Techniques},
  series = {SIGGRAPH '86},
  year = {1986},
@@ -87,7 +87,7 @@ @inproceedings{Kay:1986:RTC:15922.15916
 
 @article{DBLP:journals/corr/abs-1810-07951,
   author    = {Michael Innes},
-  title     = {Don't Unroll Adjoint: Differentiating SSA-Form Programs},
+  title     = {Don't {U}nroll {A}djoint: {D}ifferentiating {SSA-F}orm {P}rograms},
   journal   = {CoRR},
   volume    = {abs/1810.07951},
   year      = {2018},
@@ -98,3 +98,39 @@ @article{DBLP:journals/corr/abs-1810-07951
   biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1810-07951},
   bibsource = {dblp computer science bibliography, https://dblp.org}
 }
+
+@article{softraster,
+  author    = {Shichen Liu and
+               Tianye Li and
+               Weikai Chen and
+               Hao Li},
+  title     = {Soft Rasterizer: {A} Differentiable Renderer for Image-based 3D Reasoning},
+  journal   = {CoRR},
+  volume    = {abs/1904.01786},
+  year      = {2019},
+  url       = {http://arxiv.org/abs/1904.01786},
+  archivePrefix = {arXiv},
+  eprint    = {1904.01786},
+  timestamp = {Wed, 24 Apr 2019 12:21:25 +0200},
+  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1904-01786},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+
+@article{retinaface,
+  author    = {Jiankang Deng and
+               Jia Guo and
+               Yuxiang Zhou and
+               Jinke Yu and
+               Irene Kotsia and
+               Stefanos Zafeiriou},
+  title     = {RetinaFace: Single-stage Dense Face Localisation in the Wild},
+  journal   = {CoRR},
+  volume    = {abs/1905.00641},
+  year      = {2019},
+  url       = {http://arxiv.org/abs/1905.00641},
+  archivePrefix = {arXiv},
+  eprint    = {1905.00641},
+  timestamp = {Mon, 27 May 2019 13:15:00 +0200},
+  biburl    = {https://dblp.org/rec/bib/journals/corr/abs-1905-00641},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}

src/gradients/numerical.jl

@@ -18,8 +18,7 @@ results be sure to use `Float64` as `Float32` is too numerically unstable.
 function ngradient(f, xs::AbstractArray...)
     grads = zero.(xs)
     for (x, Δ) in zip(xs, grads), i in 1:length(x)
-        # This gives reasonable results
-        δ = 1.0e-13
+        δ = 1.0e-11
         tmp = x[i]
         x[i] = tmp - δ/2
         y1 = f(xs...)
@@ -41,12 +40,15 @@ end
 Get the parameters from a struct that can be tuned. The output is in
 the form of an array.
 """
-get_params(x::T) where {T<:AbstractArray} = x
+get_params(x::T, typehelp = Float32) where {T<:AbstractArray} = x
 
-get_params(x::T) where {T<:Real} = [x]
+get_params(x::T, typehelp = Float32) where {T<:Real} = [x]
 
-get_params(x::T) where {T} = foldl((a, b) -> [a; b],
-                                   [map(i -> get_params(getfield(x, i)), fieldnames(T))...])
+get_params(::Nothing, typehelp = Float32) = typehelp[]
+
+get_params(x::T, typehelp = Float32) where {T} =
+    foldl((a, b) -> [a; b], [map(i -> get_params(getfield(x, i), typehelp),
+                                 fieldnames(T))...])
 
 """
     set_params!(x, y::AbstractArray)
@@ -89,6 +91,7 @@ set_params!(x::T, y::AbstractArray) where {T<:Real} = set_params!([x], y)
 function set_params!(x, y::AbstractArray)
     start = 1
     for i in 1:nfields(x)
+        isnothing(getfield(x, i)) && continue
         start += set_params!(getfield(x, i), y[start:end])
     end
     return start - 1

src/objects/triangle.jl

@@ -52,10 +52,9 @@ end
 function get_normal(t::Triangle, pt, dir)
     # normal not expanded
     normal_nexp = normalize(cross(t.v2 - t.v1, t.v3 - t.v1))
-    # direction = -sign.(dot(normal_nexp, dir))
-    # normal = Vec3(repeat(normal_nexp.x, inner = size(pt.x)),
-    #               repeat(normal_nexp.y, inner = size(pt.y)),
-    #               repeat(normal_nexp.z, inner = size(pt.z)))
-    # return normal * direction
-    return normal_nexp
+    direction = -sign.(dot(normal_nexp, dir))
+    normal = Vec3(repeat(normal_nexp.x, inner = size(pt.x)),
+                  repeat(normal_nexp.y, inner = size(pt.y)),
+                  repeat(normal_nexp.z, inner = size(pt.z)))
+    return normal * direction
 end

src/optimize.jl

@@ -37,7 +37,7 @@ update!(opt, x::T, Δ::T) where {T<:Real} = (update!(opt, [x], [Δ]))[]
 # This makes sure we donot end up optimizing the value of the material.
 # We cannot do this update in a stable manner for now. So it is wise
 # to just avoid it for now.
-update!(opt, x::Material, Δ::Material) = x
+# update!(opt, x::Material, Δ::Material) = x
 
 update!(opt, x::FixedParams, Δ::FixedParams) = x 
 

test/gradcheck.jl

@@ -13,23 +13,35 @@ end
 
 screen_size = (w = 400, h = 300)
 
-light = PointLight(Vec3(1.0f0), 1000.0f0, Vec3(0.15f0, 0.5f0, -110.5f0))
+light = PointLight(
+    Vec3(1.0),
+    1000.0,
+    Vec3(0.15, 0.5, -110.5)
+)
 
-eye_pos = Vec3(0.0f0, 0.0f0, -5.0f0)
+eye_pos = Vec3(0.0, 0.0, -5.0)
 
-cam = Camera(eye_pos, Vec3(0.0f0), Vec3(0.0f0, 1.0f0, 0.0f0), 45.0f0, 1.0f0, screen_size...)
+cam = Camera(eye_pos, Vec3(0.0), Vec3(0.0, 1.0, 0.0), 45.0, 1.0, screen_size...)
 
 origin, direction = get_primary_rays(cam);
 
 @testset "Triangle" begin
     scene = [
-        Triangle(Vec3(-1.7f0, 1.0f0, 0.0f0), Vec3(1.0f0, 1.0f0, 0.0f0), Vec3(1.0f0, -1.0f0, 0.0f0),
-                 Material())
+        Triangle(Vec3(-1.7, 1.0, 0.0), Vec3(1.0, 1.0, 0.0), Vec3(1.0, -1.0, 0.0),
+                 Material(color_ambient = Vec3(1.0),
+                          color_diffuse = Vec3(1.0),
+                          color_specular = Vec3(1.0),
+                          specular_exponent = 50.0,
+                          reflection = 1.0))
     ]
 
     scene_new = [
-        Triangle(Vec3(-1.9f0, 1.3f0, 0.1f0), Vec3(1.2f0, 1.1f0, 0.3f0), Vec3(0.8f0, -1.2f0, -0.15f0),
-                 Material())
+        Triangle(Vec3(-1.9, 1.3, 0.1), Vec3(1.2, 1.1, 0.3), Vec3(0.8, -1.2, -0.15),
+                 Material(color_ambient = Vec3(1.0),
+                          color_diffuse = Vec3(1.0),
+                          color_specular = Vec3(1.0),
+                          specular_exponent = 50.0,
+                          reflection = 1.0))
     ]
 
     color = raytrace(origin, direction, scene, light, eye_pos, 0)
@@ -38,17 +50,26 @@ origin, direction = get_primary_rays(cam);
 
     numerical_grads = get_params(numderiv(x -> loss_fn([x], color), scene_new[1]))
 
-    # Ignore the Material Gradients
-    @test isapprox(numerical_grads[1:end-4], zygote_grads[1:end-4], rtol = 1.0e-1)
+    @test isapprox(numerical_grads, zygote_grads, rtol = 1.0e-3)
 end
 
 @testset "Sphere" begin
     scene = [
-        Sphere(Vec3(-1.7f0, 1.0f0, 0.0f0), [0.6f0], Material())
+        Sphere(Vec3(-1.7, 1.0, 0.0), [0.6],
+               Material(color_ambient = Vec3(1.0),
+                        color_diffuse = Vec3(1.0),
+                        color_specular = Vec3(1.0),
+                        specular_exponent = 50.0,
+                        reflection = 1.0))
     ]
 
     scene_new = [
-        Sphere(Vec3(-1.9f0, 1.3f0, 0.1f0), [0.9f0], Material())
+        Sphere(Vec3(-1.9, 1.3, 0.1), [0.9],
+	       Material(color_ambient = Vec3(1.0),
+                        color_diffuse = Vec3(1.0),
+                        color_specular = Vec3(1.0),
+                        specular_exponent = 50.0,
+                        reflection = 1.0))
     ]
 
     color = raytrace(origin, direction, scene, light, eye_pos, 0)
@@ -57,6 +78,5 @@ end
 
     numerical_grads = get_params(numderiv(x -> loss_fn([x], color), scene_new[1]))
 
-    # Ignore the Material Gradients
-    @test isapprox(numerical_grads[1:end-5], zygote_grads[1:end-5], rtol = 1.0e-1)
+    @test isapprox(numerical_grads, zygote_grads, rtol = 1.0e-3)
 end

test/runtests.jl

@@ -8,7 +8,6 @@ end
 
 @testset "Differentiable Ray Tracing" begin
     @testset "Gradient Checks" begin
-        # Numerical Gradients are presently broken
-        # include("gradcheck.jl")
+        include("gradcheck.jl")
     end
 end