fix usd documentation and some CI issues (#721)

* fix usd documentation and some CI issues

Signed-off-by: Clement Fuji Tsang <cfujitsang@nvidia.com>

Signed-off-by: Clement Fuji Tsang <cfujitsang@nvidia.com>

ci/gitlab_jenkins_templates/core_ci.jenkins

@@ -482,11 +482,12 @@ def prepareWindowsJob(configName, cudaVer, pythonVer, torchVer, archsToTest,
       updateGitlabCommitStatus(name: "build-${configName}", state: "pending")
       if (buildWheel.toBoolean()) {
         updateGitlabCommitStatus(name: "test-${configName}", state: "pending")
-      } else {
-        for (arch in archsToTest.split(';')) {
-          updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
-        }
-      }
+      } //else {
+      //  for (arch in archsToTest.split(';')) {
+      //    updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
+      //  }
+      //}
+
       build job: "windows_build_template_CI",
       parameters: [
         string(name: 'configName', value: "${configName}"),

ci/gitlab_jenkins_templates/ubuntu_test_CI.jenkins

@@ -87,7 +87,11 @@ spec:
           stage("Pytest") {
             sh '''
             export KAOLIN_TEST_NVIDIFFRAST=1
-            pytest --import-mode=importlib -s --cov=/kaolin/kaolin /kaolin/tests/python/kaolin/
+            export KAOLIN_TEST_SHAPENETV1_PATH=/mnt/data/ci_shapenetv1
+            export KAOLIN_TEST_SHAPENETV2_PATH=/mnt/data/ci_shapenetv2
+            export KAOLIN_TEST_MODELNET_PATH=/mnt/data/ModelNet
+            export KAOLIN_TEST_SHREC16_PATH=/mnt/data/ci_shrec16
+            pytest --import-mode=importlib -rs --cov=/kaolin/kaolin /kaolin/tests/python/kaolin/
             '''
           }
         } catch(e) {
@@ -96,7 +100,7 @@ spec:
         }
         try {
           stage("Dash3D") {
-            sh '''
+            '''
             pytest -s --cov=/kaolin/kaolin /kaolin/tests/integration/
             '''
           }
@@ -167,6 +171,14 @@ spec:
           build_passed = false
           echo e.toString()
         }
+        try {
+          stage("Interactive Visualizer") {
+            sh 'cd /kaolin/examples/tutorial && ipython interactive_visualizer.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
         try {
           stage("SPC from Pointcloud Recipe") {
             sh 'cd /kaolin/examples/recipes/dataload/ && python spc_from_pointcloud.py'

ci/gitlab_jenkins_templates/windows_build_CI.jenkins

@@ -124,18 +124,18 @@ spec:
       jobMap = [:]
       if (buildWheel.toBoolean()) {
         jobMap["test"] = prepareWindowsWheelTestJob()
-      } else { 
-        for (arch in archsToTest.split(';')) {
-          jobMap["${arch}"] = prepareWindowsTestJob(arch)
-        }
-      }
+      } //else { 
+      //  for (arch in archsToTest.split(';')) {
+      //    jobMap["${arch}"] = prepareWindowsTestJob(arch)
+      //  }
       parallel jobMap
     }
   }
 }
 
 }  // gitlabCommitStatus
 
+/*
 def prepareWindowsTestJob(arch) {
   return {
     stage("Test ${arch}") {
@@ -155,6 +155,7 @@ def prepareWindowsTestJob(arch) {
     }
   }
 }
+*/
 
 def prepareWindowsWheelTestJob() {
   return {

examples/tutorial/interactive_visualizer.ipynb

@@ -54,8 +54,8 @@
    "outputs": [],
    "source": [
     "# Here replace the \"root\" by the path where you are storing shapenet\n",
-    "ds = kal.io.shapenet.ShapeNetV2(root='/data/ShapeNetCore.v2/',\n",
-    "                                categories=['car'],\n",
+    "ds = kal.io.shapenet.ShapeNetV2(root='/mnt/data/ci_shapenetv2/',\n",
+    "                                categories=['plane'],\n",
     "                                train=True, split=1.,\n",
     "                                with_materials=True,\n",
     "                                output_dict=True)\n",
@@ -569,7 +569,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.10.8"
   }
  },
  "nbformat": 4,

kaolin/io/__init__.py

@@ -8,3 +8,5 @@
 from . import modelnet
 from . import shrec
 from . import utils
+
+__all__ = [k for k in locals().keys() if not k.startswith('__')]

kaolin/io/usd/__init__.py

@@ -2,3 +2,5 @@
 from .mesh import *
 from .pointcloud import *
 from .voxelgrid import *
+
+__all__ = [k for k in locals().keys() if not k.startswith('__')]

kaolin/io/usd/mesh.py

@@ -75,18 +75,21 @@ def get_raw_mesh_prim_geometry(mesh_prim, time=None, with_normals=False, with_uv
         time (convertible to float, optional): positive integer indicating the time at which to retrieve parameters
 
     Returns:
-    dictionary of some or all values:
+
+    (dict):
         - **vertices** (torch.FloatTensor): vertex positions with any transforms already applied, of shape (N, 3)
         - **transform** (torch.FloatTensor): applied transform of shape (4, 4)
         - **faces** (torch.LongTensor): face vertex indices of original shape saved in the USD
         - **face_sizes** (torch.LongTensor): face vertex counts of original shape saved in the USD
-        if `with_normals=True`:
-        - **normals** (torch.FloatTensor): normal values of original shape saved in the USD
-        - **normals_interpolation** (string): normal interpolation type saved in the USD, such as "faceVarying"
-        if `with_uvs=True`:
-        - **uvs** (torch.FloatTensor): raw UV values saved in the USD
-        - **face_uvs_idx** (torch.LongTensor): raw indices into the UV for every vertex of every face
-        - **uv_interpolation** (string): UV interpolation type saved in the USD, such as "faceVarying"
+        - **normals** (torch.FloatTensor, optional):
+            if `with_normals=True`, normal values of original shape saved in the USD
+        - **normals_interpolation** (string, optional):
+            if `with_normals=True`, normal interpolation type saved in the USD, such as "faceVarying"
+        - **uvs** (torch.FloatTensor, optional): if ``with_uvs=True``, raw UV values saved in the USD
+        - **face_uvs_idx** (torch.LongTensor, optional):
+            if ``with_uvs=True``, raw indices into the UV for every vertex of every face
+        - **uv_interpolation** (string, optional):
+            if ``with_uvs=True``, UV interpolation type saved in the USD, such as "faceVarying"
     """
     if time is None:
         time = Usd.TimeCode.Default()

tests/python/kaolin/io/test_modelnet.py

@@ -24,7 +24,7 @@
 from kaolin.io.off import return_type
 from kaolin.io.modelnet import ModelNet
 
-MODELNET_PATH = '/data/ModelNet'
+MODELNET_PATH = os.getenv('KAOLIN_TEST_MODELNET_PATH')
 MODELNET_TEST_CATEGORY_LABELS = ['bathtub']
 MODELNET_TEST_CATEGORY_LABELS_2 = ['desk']
 MODELNET_TEST_CATEGORY_LABELS_MULTI = ['bathtub', 'desk']
@@ -36,7 +36,8 @@
 ]
 
 # Skip test in a CI environment
-@pytest.mark.skipif(os.getenv('CI') == 'true', reason="CI does not have dataset")
+@pytest.mark.skipif(MODELNET_PATH is None,
+                    reason="'KAOLIN_TEST_MODELNET_PATH' environment variable is not set.")
 @pytest.mark.parametrize('categories', ALL_CATEGORIES)
 @pytest.mark.parametrize('split', ['train', 'test'])
 @pytest.mark.parametrize('index', [0, -1])

tests/python/kaolin/io/test_shapenet.py

@@ -24,22 +24,28 @@
 from kaolin.io.dataset import KaolinDatasetItem
 from kaolin.io import shapenet
 
-SHAPENETV1_PATH = '/data/ShapeNetCore.v1'
-SHAPENETV2_PATH = '/data/ShapeNetCore.v2'
+SHAPENETV1_PATH = os.getenv('KAOLIN_TEST_SHAPENETV1_PATH')
+SHAPENETV2_PATH = os.getenv('KAOLIN_TEST_SHAPENETV2_PATH')
 SHAPENET_TEST_CATEGORY_SYNSETS = ['02933112']
 SHAPENET_TEST_CATEGORY_LABELS = ['dishwasher']
 SHAPENET_TEST_CATEGORY_MULTI = ['mailbox', '04379243']
 
 ALL_CATEGORIES = [
-    None,
     SHAPENET_TEST_CATEGORY_SYNSETS,
     SHAPENET_TEST_CATEGORY_LABELS,
     SHAPENET_TEST_CATEGORY_MULTI
 ]
 
-# Skip test in a CI environment
-@pytest.mark.skipif(os.getenv('CI') == 'true', reason="CI does not have dataset")
-@pytest.mark.parametrize('version', ['v1', 'v2'])
+@pytest.mark.parametrize('version', [
+    pytest.param('v1', marks=pytest.mark.skipif(
+        SHAPENETV1_PATH is None,
+        reason="'KAOLIN_TEST_SHAPENETV1_PATH' environment variable is not set."
+    )),
+    pytest.param('v2', marks=pytest.mark.skipif(
+        SHAPENETV2_PATH is None,
+        reason="'KAOLIN_TEST_SHAPENETV2_PATH' environment variable is not set."
+    ))
+])
 @pytest.mark.parametrize('categories', ALL_CATEGORIES)
 @pytest.mark.parametrize('with_materials', [True, False])
 @pytest.mark.parametrize('output_dict', [True, False])

tests/python/kaolin/io/test_shrec.py

@@ -24,7 +24,7 @@
 from kaolin.io.dataset import KaolinDatasetItem
 from kaolin.io.shrec import SHREC16
 
-SHREC16_PATH = '/data/shrec16/'
+SHREC16_PATH = os.getenv('KAOLIN_TEST_SHREC16_PATH')
 SHREC16_TEST_CATEGORY_SYNSETS = ['02691156']
 SHREC16_TEST_CATEGORY_LABELS = ['airplane']
 SHREC16_TEST_CATEGORY_SYNSETS_2 = ['02958343']
@@ -33,7 +33,6 @@
 SHREC16_TEST_CATEGORY_LABELS_MULTI = ['airplane', 'car']
 
 ALL_CATEGORIES = [
-    None,
     SHREC16_TEST_CATEGORY_SYNSETS,
     SHREC16_TEST_CATEGORY_LABELS,
     SHREC16_TEST_CATEGORY_SYNSETS_2,
@@ -44,7 +43,8 @@
 
 
 # Skip test in a CI environment
-@pytest.mark.skipif(os.getenv('CI') == 'true', reason="CI does not have dataset")
+@pytest.mark.skipif(SHREC16_PATH is None,
+                    reason="'KAOLIN_TEST_SHREC16_PATH' environment variable is not set.")
 @pytest.mark.parametrize('categories', ALL_CATEGORIES)
 @pytest.mark.parametrize('split', ['train', 'val', 'test'])
 @pytest.mark.parametrize('use_transform', [True, False])

tools/linux/Dockerfile.base

@@ -1,8 +1,8 @@
 ARG CUDA_VERSION=11.6.1
 ARG CUDNN_VERSION=8
-FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu22.04
+FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu20.04
 
-ARG PYTHON_VERSION=3.9
+ARG PYTHON_VERSION=3.8
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 ARG PYTORCH_VERSION=1.13.1
 ENV PYTORCH_VERSION=${PYTORCH_VERSION}

tools/linux/Dockerfile.base_cpuonly

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 # used for cross-compilation in docker build
 #
 ARG PYTHON_VERSION=3.9