added check for nans and a comment

mlcommons · May 22, 2023 · a5ea57a · a5ea57a
1 parent a4eb233
commit a5ea57a
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/GANDLF/cli/patch_extraction.py b/GANDLF/cli/patch_extraction.py
@@ -20,6 +20,12 @@
 def parse_gandlf_csv(fpath):
     df, _ = parseTrainingCSV(fpath, train=False)
     df = df.drop_duplicates()
+    # nans can be easily removed using df.dropna(axis=1, how='all')
+    # we want to keep them because we want the user to check the CSV instead
+    # there might be cases where labels are accidentally removed for some subjects, but not all
+    assert (
+        df.isnull().values.any() == False
+    ), "Data CSV contains null/nan values, please check."
     for _, row in df.iterrows():
         if "Label" in row:
             yield row["SubjectID"], row["Channel_0"], row["Label"]