Setting a time limit of 5 seconds to every test

pyproject.toml

@@ -11,27 +11,24 @@ packages = [{include='pysus'}]
 cffi = "1.15.1"
 dbfread = "2.0.7"
 fastparquet = "^0.8.1"
-geocoder = "^1.38.1"
-jupyterlab = "^3.4.5"
 numpy = "1.23.2"
-pandas = "1.4.3"
+pandas = ">=1.5.3"
 pyarrow = ">=11.0.0"
 pycparser = "2.21"
 pyreaddbc = "1.0.0"
 python = "^3.9"
 python-dateutil = "2.8.2"
-pytz = "2022.2.1"
-six = "1.16.0"
 tqdm = "4.64.0"
 wget = "^3.2"
 loguru = "^0.6.0"
 Unidecode = "^1.3.6"
 sqlalchemy = "<2.0.0"
 elasticsearch = "7.16.2"
-ipykernel = "^6.22.0"
-geobr = "^0.2.0"
 dateparser = "^1.1.8"
 urwid = "^2.1.2"
+# Machine Learn Packages
+geobr = { version = "^0.2.0", extras=["preprocessing"] }
+geocoder = { version = "^1.38.1", extras=["ml"] }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=6.1.0"
@@ -46,6 +43,9 @@ folium = "^0.14.0"
 seaborn = "^0.12.2"
 descartes = "^1.1.0"
 keplergl = "^0.3.2"
+jupyterlab = "^4.0.5"
+ipykernel = "^6.25.1"
+pytest-timeout = "^2.1.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -70,5 +70,8 @@ testpaths = [
     "tests"
 ]
 
-
 exclude = ["*.git", "docs/"]
+
+[tool.poetry.extras]
+preprocessing = ["geobr", "geocoder"]
+

pysus/tests/test_SIA.py

@@ -1,11 +1,13 @@
 import unittest
-from ftplib import FTP
+import pytest
+
 import pandas as pd
 from pysus.online_data.SIA import download
 from pysus.online_data import parquets_to_dataframe as to_df
 
 class SIATestCase(unittest.TestCase):
-    @unittest.skip  # Takes a long time to complete
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_large_PA(self):
         res = to_df(download('SP', 2020, 12, group='PA'))
         if isinstance(res, pd.DataFrame):

pysus/tests/test_cnes.py

@@ -1,4 +1,5 @@
 import unittest
+import pytest
 
 import pandas as pd
 
@@ -13,10 +14,8 @@ def test_fetch_estabelecimentos(self):
         self.assertIsInstance(df, pd.DataFrame)
         # self.assertEqual(True, False)  # add assertion here
 
+    @pytest.mark.timeout(5)
     def test_fetch_equipamentos(self):
         df = to_df(download(group="EQ", states="RO", years=2021, months=9))
         self.assertIsInstance(df, pd.DataFrame)
 
-
-# if __name__ == "__main__":
-#     unittest.main()

pysus/tests/test_data/test_Infodengue.py

@@ -1,10 +1,12 @@
 import unittest
+import pytest
 
 import pandas as pd
 from pysus.online_data.Infodengue import download, search_string, geocode_by_cities, normalize
 
 
 class InfoDengueTestCase(unittest.TestCase):
+    @pytest.mark.timeout(1)
     def test_search_string(self):
         get_from_dict = search_string("Curitiba")
         cites_mathes = {
@@ -71,6 +73,7 @@ def test_search_string(self):
         self.assertIn("Rio de Janeiro", pattern_city_names.keys())
         self.assertIn(4204806, get_from_dict.values() )
 
+    @pytest.mark.timeout(1)
     def test_normalize(self):
         normalized_str = normalize("Rio das Ostras")
 
@@ -80,6 +83,7 @@ def test_normalize(self):
         # self.assertEqual(substr_list, ['rio', 'das', 'ostras'])
         self.assertEqual(normalized_str, "rio das ostras")
 
+    @pytest.mark.timeout(1)
     def test_download(self):
         df = download(
             "dengue",

pysus/tests/test_data/test_Infogripe.py

@@ -1,9 +1,12 @@
 import unittest
+import pytest
 
 from pysus.online_data.Infogripe import download, DATASETS
 
 
 class InfoGripeTestCase(unittest.TestCase):
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download(self):
         for ds in DATASETS.keys():
             df = download(ds)

pysus/tests/test_data/test_PNI.py

@@ -1,21 +1,25 @@
 import unittest
+import pytest
 
 import pandas as pd
 
 from pysus.online_data.PNI import *
 from pysus.online_data import parquets_to_dataframe
-
+ 
 
 class PNITestCase(unittest.TestCase):
+    @pytest.mark.timeout(1)
     def test_get_available_years(self):
         res = get_available_years("AC")
         self.assertIsInstance(res, list)
         self.assertIn('2000', res)
 
+    @pytest.mark.timeout(1)
     def test_get_available_docs(self):
         res = available_docs()
         self.assertIsInstance(res, list)
 
+    @pytest.mark.timeout(1)
     def test_download(self):
         files = download("RO", 2000)
         df = parquets_to_dataframe(files)

pysus/tests/test_data/test_ciha.py

@@ -1,6 +1,7 @@
 __author__ = "fccoelho"
 
 import unittest
+import pytest
 
 import pandas as pd
 
@@ -11,13 +12,17 @@
 
 
 class SIHTestCase(unittest.TestCase):
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_CIH(self):
         files = download("mg", 2011, 7)
         df = parquets_to_dataframe(files)
         self.assertGreater(len(df), 0)
         self.assertIn("DIAG_PRINC", df.columns)
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_CIHA(self):
         files = download("MG", 2013, 10)
         df = parquets_to_dataframe(files)

pysus/tests/test_data/test_sia.py

@@ -2,15 +2,18 @@
 
 from pathlib import Path
 import unittest
+import pytest
 
 import pandas as pd
 from pysus.online_data.SIA import download
 from pysus.online_data import parquets_to_dataframe
 
 unittest.skip("too slow to run om travis")
-
+ 
 
 class SIATestCase(unittest.TestCase):
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_after_2008(self):
         parquerts = download("to", 2015, 12)
         df = parquets_to_dataframe(parquerts)
@@ -19,24 +22,31 @@ def test_download_after_2008(self):
         self.assertIsInstance(df, pd.DataFrame)
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_before_2008(self):
         parquets = download("mg", 2005, 8)
         # self.assertWarns(UserWarning)
         df = parquets_to_dataframe(parquets)
         self.assertIn("PA_CODUNI", df.columns)
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.timeout(5)
     @unittest.expectedFailure
     def test_download_before_1994(self):
         file = download("RS", 1993, 12)
         self.assertTrue(Path(file).exists())
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_one(self):
         file = download("se", 2020, 10, group="PS")
         df = parquets_to_dataframe(file)
         self.assertIn("CNS_PAC", df.columns)
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_many(self):
         files = []
         groups = ["aq", "AM", "atd"]
@@ -57,6 +67,8 @@ def test_download_many(self):
         self.assertIn("AM_PESO", df2.columns)
         self.assertIn("ATD_CARACT", df3.columns)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_missing(self):
         dfs = parquets_to_dataframe(download("MS", 2006, 5))
         self.assertIsNotNone(dfs)

pysus/tests/test_data/test_sih.py

@@ -1,6 +1,7 @@
 __author__ = "fccoelho"
 
 import unittest
+import pytest
 
 import pandas as pd
 
@@ -11,6 +12,8 @@
 
 
 class SIHTestCase(unittest.TestCase):
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download(self):
         df = to_df(download("to", 2009, 12))
         df2 = to_df(download("AC", 2013, 10))

pysus/tests/test_data/test_sinan.py

@@ -1,3 +1,5 @@
+import pytest
+
 import datetime
 import os
 from pathlib import Path
@@ -39,60 +41,74 @@ class TestSINANClass(unittest.TestCase):
         'RAIVBR19.parquet',
     ]
 
+    @pytest.mark.timeout(5)
     def test_list_all_diseases(self):
         all_diseases = list(FTP_SINAN.diseases.keys())
         self.assertIn('Dengue', all_diseases)
         self.assertIn('Zika', all_diseases)
         self.assertIn('Chikungunya', all_diseases)
 
+    @pytest.mark.timeout(5)
     def test_download(self):
         files = download(self.d1, [7,8,9], data_path=self.data_path)
         self.assertEqual(len(files), 3)
 
+    @pytest.mark.timeout(5)
     def test_read_dataframe(self):
         df = parquets_to_dataframe(Path(self.data_path)/self.r1[0])
         self.assertIsInstance(df, pd.DataFrame)
         self.assertEqual(df.shape, (110, 94))
 
+    @pytest.mark.timeout(5)
     def test_metadata_dataframe(self):
         df = metadata_df('Raiva Humana')
         self.assertIsInstance(df, pd.DataFrame)
         self.assertEqual(df.shape, (68, 7))
 
 
 class TestSINANDownload(unittest.TestCase):
+    @pytest.mark.timeout(5)
     def test_download(self):
         df = parquets_to_dataframe(download(years=2007, disease='Botulismo'))
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.timeout(5)
     def test_filename_only(self):
         fname = download(years=2015, disease='Botulismo')
         self.assertIsInstance(fname, str)
         self.assertTrue(os.path.exists(fname))
         shutil.rmtree(fname, ignore_errors=True)
 
+    @pytest.mark.timeout(5)
     def test_fetch_viol_dom(self):
         df = parquets_to_dataframe(download(years=2011, disease='Hantavirose'))
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.timeout(5)
     def test_fetch_cancer_prelim(self):
         df = parquets_to_dataframe(download(years=2022, disease='Cancer'))
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_fetch_sifilis(self):
         self.assertRaises(
             Exception, download(years=2021, disease='Sífilis Adquirida')
         )
 
+    @pytest.mark.timeout(5)
     def test_fetch_sifilis_gestante(self):
         df = parquets_to_dataframe(download(years=2021, disease='Sífilis em Gestante'))
         self.assertIsInstance(df, pd.DataFrame)
 
+    @pytest.mark.timeout(5)
     def test_lista_agravos(self):
         lista = list_diseases()
         self.assertIsInstance(lista, list)
         self.assertGreater(len(lista), 0)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_chunked_df_size(self):
         df1 = parquets_to_dataframe(download(years=2018, disease='Chikungunya'))
         s1 = len(df1)
@@ -110,6 +126,7 @@ def test_chunked_df_size(self):
 class TestSinanDBF(unittest.TestCase):
     dbf_name = PATH_ROOT / 'EPR-2016-06-01-2016.dbf'
 
+    @pytest.mark.timeout(5)
     def test_read_dbf(self):
         df = read_sinan_dbf(self.dbf_name, encoding='latin-1')
         self.assertTrue(self.dbf_name.exists())
@@ -131,6 +148,7 @@ def test_read_dbf(self):
                     ),
                 )
 
+    @pytest.mark.timeout(5)
     def test_type_convertion(self):
         df = read_sinan_dbf(self.dbf_name, encoding='latin-1')
         self.assertTrue(self.dbf_name.exists())

pysus/tests/test_data/test_sinasc.py

@@ -1,22 +1,28 @@
 __author__ = "fccoelho"
 
 import unittest
+import pytest
 
 from pysus.online_data.sinasc import download, get_available_years
 from pysus.online_data import parquets_to_dataframe as to_df
 
 
 class TestDownload(unittest.TestCase):
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_new(self):
         df = to_df(download("SE", 2015))
         self.assertIn("IDADEMAE", df.columns)
         self.assertGreater(len(df), 0)
 
+    @pytest.mark.skip(reason="This test takes too long")
+    @pytest.mark.timeout(5)
     def test_download_old(self):
         df = to_df(download("AL", 1994)[0]) #[0] bc there is a file duplicity in the ftp sever
         self.assertIn("IDADE_MAE", df.columns)
         self.assertGreater(len(df), 0)
 
+    @pytest.mark.timeout(5)
     def test_get_available_years(self):
         yrs = get_available_years("AC")
         self.assertIn("1996", yrs)

pysus/tests/test_data/test_vaccine.py

@@ -1,11 +1,13 @@
 import unittest
+import pytest
 
 import pandas as pd
 
 from pysus.online_data.vaccine import download_covid
 
 
 class VaccineTestCase(unittest.TestCase):
+    pytest.mark.timeout(5)
     def test_Download(self):
         """Careful! this download can take a long time"""
         df = download_covid("BA", only_header=True)