add Camera API (#599)

Signed-off-by: Clement Fuji Tsang <cfujitsang@nvidia.com>

Co-authored-by: operel <operel@nvidia.com>

docs/kaolin_ext.py

@@ -48,7 +48,16 @@ def run_apidoc(_):
             "kaolin/render/mesh/utils.py",
             "kaolin/render/spc/raytrace.py",
             "kaolin/rep/spc.py",
-            "kaolin/visualize/timelapse.py"
+            "kaolin/visualize/timelapse.py",
+            "kaolin/framework/*",
+            "kaolin/render/camera/camera.py",
+            "kaolin/render/camera/coordinates.py",
+            "kaolin/render/camera/extrinsics_backends.py",
+            "kaolin/render/camera/extrinsics.py",
+            "kaolin/render/camera/intrinsics_ortho.py",
+            "kaolin/render/camera/intrinsics_pinhole.py",
+            "kaolin/render/camera/intrinsics.py",
+            "kaolin/render/camera/legacy.py"
         ]
     ]
 

docs/modules/package.rst_t

@@ -22,11 +22,11 @@ API
 {{ pkgname | e | heading }}
 {%- endif -%}
 
-{%- if subpackages or submodules -%}
+{%- if subpackages or submodules %}
 .. toctree::
    :maxdepth: {{ maxdepth }}
    :titlesonly:
-{%- endif -%}
+{% endif -%}
 {{ unfoldtree(subpackages + submodules) }}
 
 {% if not is_namespace -%}

docs/notes/tutorial_index.rst

@@ -64,4 +64,14 @@ Simple Recipes
     * `spc_trilinear_interp.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/spc/spc_trilinear_interp.py>`_: computing trilinear interpolation of a point cloud on an SPC
 * Visualization:
     * `visualize_main.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/tutorial/visualize_main.py>`_: using Timelapse API to write mock 3D checkpoints
+* Camera:
+    * `camera_differentiable.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_differentiable.py>`_: optimize a camera position
+    * `camera_transforms.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_transforms.py>`_: using :func:`Camera.transform()` function
+    * `camera_ray_tracing.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_ray_tracing.py>`_: how to design a ray generating function using :class:`Camera` objects
+    * `camera_properties.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_properties.py>`_: exposing some the camera attributes and properties
+    * `camera_opengl_shaders.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_opengl_shaders.py>`_: Using the camera with glumpy
+    * `camera_movement.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_movement.py>`_: Manipulating a camera position and zoom
+    * `camera_init_simple.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_init_simple.py>`_: Making Camera objects with the flexible :func:`Camera.from_args()` constructor
+    * `camera_init_explicit.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_init_explicit.py>`_: Making :class:`CameraIntrinsics` and :class:`CameraExtrinsics` with all the different constructors available
+    * `camera_coordinate_systems.py <https://github.com/NVIDIAGameWorks/kaolin/blob/master/examples/recipes/camera_coordinate_systems.py>`_: Changing coordinate system in a :class:`Camera` object
 

examples/recipes/camera/camera_coordinate_systems.py

@@ -0,0 +1,25 @@
+# ==============================================================================================================
+# The following snippet demonstrates how to change the coordinate system of the camera.
+# ==============================================================================================================
+
+import math
+import torch
+import numpy as np
+from kaolin.render.camera import Camera, blender_coords
+
+device = 'cuda'
+
+camera = Camera.from_args(
+    eye=torch.tensor([4.0, 4.0, 4.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    fov=30 * np.pi / 180,  # In radians
+    width=800, height=800,
+    device=device
+)
+
+print(camera.basis_change_matrix)
+camera.change_coordinate_system(blender_coords())
+print(camera.basis_change_matrix)
+camera.reset_coordinate_system()
+print(camera.basis_change_matrix)

examples/recipes/camera/camera_init_explicit.py

@@ -0,0 +1,88 @@
+# ==============================================================================================================
+# The following snippet demonstrates how to initialize instances of kaolin's pinhole / ortho cameras
+# explicitly.
+# Also review `camera_init_simple` which greatly simplifies the construction methods shown here.
+# ==============================================================================================================
+
+import math
+import torch
+from kaolin.render.camera import Camera, CameraExtrinsics, PinholeIntrinsics, OrthographicIntrinsics
+
+#################################################################
+#   Camera 1: from eye, at, up and focal length (Perspective)   #
+#################################################################
+# Build the camera extrinsics object from lookat
+eye = torch.tensor([0.0, 0.0, -1.0], device='cuda') # Camera positioned here in world coords
+at = torch.tensor([0.0, 0.0, 0.0], device='cuda')   # Camera observing this world point
+up = torch.tensor([0.0, 1.0, 0.0], device='cuda')   # Camera up direction vector
+extrinsics = CameraExtrinsics.from_lookat(eye, at, up)
+
+# Build a pinhole camera's intrinsics. This time we use focal length (other useful args: focal_y, x0, y0)
+intrinsics = PinholeIntrinsics.from_focal(width=800, height=600, focal_x=1.0, device='cuda')
+
+# Combine extrinsics and intrinsics to obtain the full camera object
+camera_1 = Camera(extrinsics=extrinsics, intrinsics=intrinsics)
+print('--- Camera 1 ---')
+print(camera_1)
+
+########################################################################
+#   Camera 2: from camera position, orientation and fov (Perspective)  #
+########################################################################
+# Build the camera extrinsics object from lookat
+cam_pos = torch.tensor([0.0, 0.0, -1.0], device='cuda')
+cam_dir = torch.tensor([[1.0, 0.0, 0.0],
+                        [0.0, 1.0, 0.0],
+                        [0.0, 0.0, 1.0]], device='cuda')  # 3x3 orientation within the world
+extrinsics = CameraExtrinsics.from_camera_pose(cam_pos=cam_pos, cam_dir=cam_dir)
+
+# Use pinhole camera intrinsics, construct using field-of-view (other useful args: camera_fov_direction, x0, y0)
+intrinsics = PinholeIntrinsics.from_fov(width=800, height=600, fov=math.radians(45.0), device='cuda')
+camera_2 = Camera(extrinsics=extrinsics, intrinsics=intrinsics)
+
+print('--- Camera 2 ---')
+print(camera_2)
+
+####################################################################
+#   Camera 3: camera view matrix, (Orthographic)                   #
+####################################################################
+# Build the camera extrinsics object from lookat
+world2cam = torch.tensor([[1.0, 0.0, 0.0, 0.5],
+                          [0.0, 1.0, 0.0, 0.5],
+                          [0.0, 0.0, 1.0, 0.5],
+                          [0.0, 0.0, 0.0, 1.0]], device='cuda')  # 3x3 orientation within the world
+extrinsics = CameraExtrinsics.from_view_matrix(view_matrix=world2cam)
+
+# Use pinhole camera intrinsics, construct using field-of-view (other useful args: camera_fov_direction, x0, y0)
+intrinsics = OrthographicIntrinsics.from_frustum(width=800, height=600, near=-800, far=800,
+                                                 fov_distance=1.0, device='cuda')
+camera_3 = Camera(extrinsics=extrinsics, intrinsics=intrinsics)
+
+print('--- Camera 3 ---')
+print(camera_3)
+
+
+##########################################################
+#   Camera 4: Combining cameras                          #
+##########################################################
+# Must be of the same intrinsics type, and non params fields such as width, height, near, far
+# (currently we don't perform validation)
+camera_4 = Camera.cat((camera_1, camera_2))
+
+print('--- Camera 4 ---')
+print(camera_4)
+
+
+##########################################################
+#   Camera 5: constructing a batch of cameras together   #
+##########################################################
+
+# Extrinsics are created using batched tensors. The intrinsics are automatically broadcasted.
+camera_5 = Camera.from_args(
+    eye=torch.tensor([[4.0, 4.0, 4.0], [4.0, 4.0, 4.0]]),
+    at=torch.tensor([[0.0, 0.0, 0.0], [4.0, 4.0, 4.0]]),
+    up=torch.tensor([[0.0, 1.0, 0.0], [4.0, 4.0, 4.0]]),
+    width=800, height=600, focal_x=300.0
+)
+
+print('--- Camera 5 ---')
+print(camera_5)

examples/recipes/camera/camera_init_simple.py

@@ -0,0 +1,65 @@
+# ==============================================================================================================
+# The following snippet demonstrates how to initialize instances of kaolin's pinhole / ortho cameras.
+# ==============================================================================================================
+
+import math
+import torch
+import numpy as np
+from kaolin.render.camera import Camera
+
+device = 'cuda'
+
+perspective_camera_1 = Camera.from_args(
+    eye=torch.tensor([4.0, 4.0, 4.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    fov=30 * np.pi / 180,  # In radians
+    x0=0.0, y0=0.0,
+    width=800, height=800,
+    near=1e-2, far=1e2,
+    dtype=torch.float64,
+    device=device
+)
+
+print('--- Perspective Camera 1 ---')
+print(perspective_camera_1)
+
+perspective_camera_2 = Camera.from_args(
+    eye=torch.tensor([4.0, 4.0, 4.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    fov=30 * np.pi / 180,  # In radians
+    width=800, height=800,
+    device=device
+)
+
+print('--- Perspective Camera 2 ---')
+print(perspective_camera_2)
+
+ortho_camera_1 = Camera.from_args(
+    eye=torch.tensor([4.0, 4.0, 4.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    width=800, height=800,
+    near=-800, far=800,
+    fov_distance=1.0,
+    dtype=torch.float64,
+    device=device
+)
+
+print('--- Orthographic Camera 1 ---')
+print(ortho_camera_1)
+
+
+ortho_camera_2 = Camera.from_args(
+    view_matrix=torch.tensor([[1.0, 0.0, 0.0, 0.5],
+                              [0.0, 1.0, 0.0, 0.5],
+                              [0.0, 0.0, 1.0, 0.5],
+                              [0.0, 0.0, 0.0, 1.0]]),
+    width=800, height=800,
+    dtype=torch.float64,
+    device=device
+)
+
+print('--- Orthographic Camera 2 ---')
+print(ortho_camera_2)

examples/recipes/camera/camera_movement.py

@@ -0,0 +1,27 @@
+# ==============================================================================================================
+# The following snippet demonstrates how to manipulate kaolin's camera.
+# ==============================================================================================================
+
+import torch
+from kaolin.render.camera import Camera
+
+
+camera = Camera.from_args(
+    eye=torch.tensor([0.0, 0.0, -1.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    width=800, height=600,
+    fov=1.0,
+    device='cuda'
+)
+
+# Extrinsic rigid transformations managed by CameraExtrinsics
+camera.move_forward(amount=10.0)               # Translate forward in world coordinates (this is wisp's mouse zoom)
+camera.move_right(amount=-5.0)                 # Translate left in world coordinates
+camera.move_up(amount=5.0)                     # Translate up in world coordinates
+camera.rotate(yaw=0.1, pitch=0.02, roll=1.0)   # Rotate the camera
+
+# Intrinsic lens transformations managed by CameraIntrinsics
+# Zoom in to decrease field of view - for Orthographic projection the internal implementation differs
+# as there is no acual fov or depth concept (hence we use a "made up" fov distance parameter, see the projection matrix)
+camera.zoom(amount=0.5)

examples/recipes/camera/camera_opengl_shaders.py

@@ -0,0 +1,57 @@
+# ==============================================================================================================
+# The following snippet demonstrates how to use the camera for generating a view-projection matrix
+# as used in opengl shaders.
+# ==============================================================================================================
+
+import torch
+import numpy as np
+from kaolin.render.camera import Camera
+
+# !!! This example is not runnable -- it is minimal to contain integration between the opengl shader and !!!
+# !!! the camera matrix                                                                                  !!!
+try:
+    from glumpy import gloo
+except:
+    class DummyGloo(object):
+        def Program(self, vertex, fragment):
+            # see: https://glumpy.readthedocs.io/en/latest/api/gloo-shader.html#glumpy.gloo.Program
+            return dict([])
+    gloo = DummyGloo()
+
+
+device = 'cuda'
+
+camera = Camera.from_args(
+    eye=torch.tensor([4.0, 4.0, 4.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    fov=30 * np.pi / 180,  # In radians
+    x0=0.0, y0=0.0,
+    width=800, height=800,
+    near=1e-2, far=1e2,
+    dtype=torch.float64,
+    device=device
+)
+
+
+vertex = """
+            uniform mat4   u_viewprojection;
+            attribute vec3 position;
+            attribute vec4 color;
+            varying vec4 v_color;
+            void main()
+            {
+                v_color = color;
+                gl_Position = u_viewprojection * vec4(position, 1.0f);
+            } """
+
+fragment = """
+            varying vec4 v_color;
+            void main()
+            {
+                gl_FragColor = v_color;
+            } """
+
+# Compile GL program
+gl_program = gloo.Program(vertex, fragment)
+gl_program["u_viewprojection"] = camera.view_projection_matrix()[0].cpu().numpy().T

examples/recipes/camera/camera_properties.py

@@ -0,0 +1,47 @@
+# ==============================================================================================================
+# The following snippet demonstrates various camera properties
+# ==============================================================================================================
+
+import math
+import torch
+import numpy as np
+from kaolin.render.camera import Camera
+
+device = 'cuda'
+
+camera = Camera.from_args(
+    eye=torch.tensor([4.0, 4.0, 4.0]),
+    at=torch.tensor([0.0, 0.0, 0.0]),
+    up=torch.tensor([0.0, 1.0, 0.0]),
+    fov=30 * np.pi / 180,  # In radians
+    width=800, height=800,
+    dtype=torch.float32,
+    device=device
+)
+
+print(camera.width)
+print(camera.height)
+print(camera.lens_type)
+
+print(camera.device)
+camera = camera.cpu()
+print(camera.device)
+
+# Create a batched camera and view single element
+camera = Camera.cat((camera, camera))
+print(camera)
+camera = camera[0]
+print(camera)
+
+print(camera.dtype)
+camera = camera.half()
+print(camera.dtype)
+camera = camera.double()
+print(camera.dtype)
+camera = camera.float()
+print(camera.dtype)
+
+print(camera.extrinsics.requires_grad)
+print(camera.intrinsics.requires_grad)
+
+print(camera.to('cuda', torch.float64))