Merge 2a6f0a2 into 8bed922

alexandrainst · Mar 15, 2021 · c08024c · c08024c
2 parents 8bed922 + 2a6f0a2
commit c08024c
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/danlp/datasets/dkhate.py b/danlp/datasets/dkhate.py
@@ -29,7 +29,7 @@ def load_with_pandas(self):
         
         :return: a dataframe for test data and a dataframe from train data
         """
-        df_test = pd.read_csv(os.path.join(self.dataset_dir, self.dataset_name + '.test' + self.file_extension), sep='\t', index_col=0, encoding='utf-8')
-        df_train = pd.read_csv(os.path.join(self.dataset_dir, self.dataset_name + '.train' + self.file_extension), sep='\t', index_col=0, encoding='utf-8')
+        df_test = pd.read_csv(os.path.join(self.dataset_dir, self.dataset_name + '.test' + self.file_extension), sep='\t', index_col=0, encoding='utf-8').dropna()
+        df_train = pd.read_csv(os.path.join(self.dataset_dir, self.dataset_name + '.train' + self.file_extension), sep='\t', index_col=0, encoding='utf-8').dropna()
 
         return df_test, df_train
diff --git a/tests/test_datasets.py b/tests/test_datasets.py
@@ -173,7 +173,7 @@ def test_dkhate(self):
         dkhate = DKHate() 
         test, train = dkhate.load_with_pandas()
         self.assertEqual(len(test), 329)
-        self.assertEqual(len(train), 2961)
+        self.assertEqual(len(train), 2960)
         self.assertEqual(set(test['subtask_a'].to_list()), {'NOT', 'OFF'})
 
 class TestDannetDataset(unittest.TestCase):