pin tornado version. pip is stupid and decides to upgrade to 6.2 due …

…to requirements in ci_requirements.txt (#630) Signed-off-by: Alex Zook <azook@nvidia.com> try using same image pull secret as linux Signed-off-by: Alex Zook <azook@nvidia.com> test change to path syntax without anything else Signed-off-by: Alex Zook <azook@nvidia.com> debug messages Signed-off-by: Alex Zook <azook@nvidia.com> Revert "debug messages" This reverts commit 5a904920a23d26b830b26a008b4733cd21ff62d9. fix paths used in assert statements. attempting to add volume mount to windows Signed-off-by: Alex Zook <azook@nvidia.com> Signed-off-by: Alex Zook <azook@nvidia.com> Co-authored-by: Alex Zook <azook@nvidia.com>
NVIDIAGameWorks · Oct 5, 2022 · 5456fe8 · 5456fe8
1 parent bb0a2eb
commit 5456fe8
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 7 deletions.
diff --git a/ci/gitlab_jenkins_templates/windows_test_CI.jenkins b/ci/gitlab_jenkins_templates/windows_test_CI.jenkins
@@ -20,6 +20,10 @@ yaml: """
 apiVersion: v1
 kind: Pod
 spec:
+  volumes:
+  - name: pvc-mount
+    persistentVolumeClaim:
+      claimName: 'kaolin-pvc'
   containers:
   - name: jnlp
     image: jenkins/jnlp-agent:latest-windows
@@ -34,8 +38,11 @@ spec:
     restartPolicy: Never
     backoffLimit: 4
     tty: true
+    volumeMounts:
+      - mountPath: c:/mnt
+        name: pvc-mount
   imagePullSecrets:
-  - name: test-secret-config2
+  - name: gitlabregcred
   nodeSelector:
     kubernetes.io/os: windows
     nvidia.com/node_type: ${arch}
@@ -63,6 +70,14 @@ spec:
         c:\\data\\bandwidthTest.exe
         '''
       }
+      stage("Check mount") {
+        catchError(stageResult: "failure") {
+          powershell '''
+            dir c:\\
+            dir c:\\mnt
+          '''
+        }
+      }
       stage("Fix paging memory") {
         // addresses this error on Windows with pytorch consuming too much paging memory: https://stackoverflow.com/a/69489193
         powershell '''
@@ -136,7 +151,7 @@ spec:
         catchError(stageResult: "failure") {
           powershell '''
             cd c:\\kaolin\\examples\\recipes\\preprocess
-            python fast_mesh_sampling.py --shapenet-dir=/mnt/data/ci_shapenetv2/
+            python fast_mesh_sampling.py --shapenet-dir=c:/mnt/data/ci_shapenetv2/
           '''
         }
       }

diff --git a/kaolin/io/dataset.py b/kaolin/io/dataset.py
@@ -94,9 +94,10 @@ def _get_saving_actions(dataset, cache_dir, save_on_disk=False,
         else:
             # If the number of folder (len(dataset)) is different, 
             # something is probably wrong with the existing data
+            #   note: reporting error directory with POSIX path to ease regex matching without raising encoding errors due to Windows backslashes
             if len(cached_ids) > 0 and len(cached_ids) != len(dataset):
                 raise RuntimeError(f"{len(cached_ids)} files already exist on "
-                                   f"{cache_dir} this dataset as {len(dataset)} files "
+                                   f"{cache_dir.resolve().as_posix()} this dataset as {len(dataset)} files "
                                    "so caching is too ambiguous and error-prone "
                                    "please force rewriting by setting 'force_overwrite'")
 
@@ -107,11 +108,13 @@ def _get_saving_actions(dataset, cache_dir, save_on_disk=False,
             #                   since we are not avoiding to run the preprocessing
             for k, v in _data.items():
                 if k in save_on_disk:
-                    path = cache_dir / f'0/{k}.pt'
+                    path:Path = cache_dir / '0' / f'{k}.pt'
+                    # note: reporting error directory with POSIX path to ease regex matching without raising encoding errors due to Windows backslashes
+                    path_str = path.resolve().as_posix()
                     if path.exists(): # There is already a file for a given key 
                         # Is the value stored the same than the one from the data?
                         assert ignore_diff_error or contained_torch_equal(v, torch.load(path)), \
-                            f"file '{cache_dir / f'0/{k}.pt'}' is different than " \
+                            f"file '{path_str}' is different than " \
                             "its matching field from the input dataset, set 'force_overwriting' " \
                             "to True to overwrite the files cached."
                         to_not_save.add(k)

diff --git a/tests/python/kaolin/io/test_dataset.py b/tests/python/kaolin/io/test_dataset.py
@@ -14,6 +14,7 @@
 
 import os
 import glob
+from pathlib import Path
 import time
 import shutil
 
@@ -240,8 +241,12 @@ def test_fail_larger_dataset(self, dataset_size, dummy_dataset, larger_dataset,
         time.sleep(0.01) # This is just in case I/O is so fast that files get greated in the same time
 
         # An error should be raised in the size of the dataset is different than the number of cached subfolder
+        # using a POSIX path to avoid regex issues with Windows path backslashes.
+        #   this is only a problem for the `pytest.raises` regex logic, not a problem with file saving.
+        cache_check_str = Path(CACHE_DIR).resolve().as_posix()
         with pytest.raises(RuntimeError,
-                           match=f"{len(dummy_dataset)} files already exist on {CACHE_DIR} "
+                        #    match=f"{len(dummy_dataset)} files already exist "):
+                           match=f"{len(dummy_dataset)} files already exist on {cache_check_str} "
                                  f"this dataset as {len(larger_dataset)} files so caching "
                                  "is too ambiguous and error-prone please force rewriting "
                                  "by setting 'force_overwrite'"):
@@ -280,6 +285,8 @@ def test_fail_diff_val_dataset(self, dummy_dataset, diff_val_dataset, use_transf
                                    num_workers, save_on_disk, cache_at_runtime):
         assert not os.path.exists(CACHE_DIR)
 
+
+
         transform = DummyTransform(10, True) if use_transform else None
         ds = CachedDataset(dummy_dataset,
                            transform=transform,
@@ -294,8 +301,13 @@ def test_fail_diff_val_dataset(self, dummy_dataset, diff_val_dataset, use_transf
         expected_files = glob.glob(os.path.join(CACHE_DIR, '*', '*.pt'))
         files_mtime = {path: os.path.getmtime(path) for path in expected_files}
         time.sleep(0.01) # This is just in case I/O is so fast that files get greated in the same time
+        f"file '{os.path.join(CACHE_DIR, '0')}"
+
+        # using a POSIX path to avoid regex issues with Windows path backslashes.
+        #   this is only a problem for the `pytest.raises` regex logic, not a problem with file saving.
+        cache_check_str = (Path(CACHE_DIR) / "0").resolve().as_posix()
         with pytest.raises(AssertionError,
-                           match=f"file '{os.path.join(CACHE_DIR, '0')}"
+                           match=f"file '{cache_check_str}"
                                  r"/.\.pt' is different than its matching field from the "
                                  "input dataset, set 'force_overwriting' to True "
                                  "to overwrite the files cached."):

diff --git a/tools/ci_requirements.txt b/tools/ci_requirements.txt
@@ -6,3 +6,4 @@ flake8-pyi==19.3.0
 pytest==7.1.0
 pytest-cov==3.0.0
 jupyter
+tornado==6.1