Improve get_temp_dir to allow subdirectory

Fix for breaking change in Tabulator

requirements.txt

@@ -9,6 +9,6 @@ python-dateutil==2.8.1
 ratelimit==2.2.1
 six==1.14.0
 sshtunnel==0.1.5
-tabulator==1.34.0
+tabulator==1.35.0
 typing==3.7.4.1
 yamlloader==0.5.5

setup.py

@@ -19,7 +19,7 @@
                 'ratelimit',
                 'six>=1.14.0',
                 'sshtunnel',
-                'tabulator>=1.34.0',
+                'tabulator>=1.35.0',
                 'typing',
                 'yamlloader'
                 ]

src/hdx/utilities/downloader.py

@@ -404,11 +404,6 @@ def get_tabular_rows(self, url, headers=1, dict_form=False, ignore_blank_rows=Tr
         """
         if headers is None:
             raise DownloadError('Argument headers cannot be None!')
-        if ignore_blank_rows:
-            skip_rows = kwargs.get('skip_rows', list())
-            if '' not in skip_rows:
-                skip_rows.append('')
-            kwargs['skip_rows'] = skip_rows
         stream = self.get_tabular_stream(url, headers=headers, **kwargs)
         origheaders = stream.headers
         if header_insertions is None or origheaders is None:
@@ -417,14 +412,19 @@ def get_tabular_rows(self, url, headers=1, dict_form=False, ignore_blank_rows=Tr
             headers = copy.deepcopy(origheaders)
             for position, header in header_insertions:
                 headers.insert(position, header)
-        if row_function is None:
+        if row_function is None and ignore_blank_rows is False:
             return headers, stream.iter(keyed=dict_form)
 
         def get_next():
             for row in stream.iter(keyed=dict_form):
-                processed_row = row_function(origheaders, row)
-                if processed_row is not None:
-                    yield processed_row
+                if ignore_blank_rows is True and not row:
+                    continue
+                if row_function:
+                    processed_row = row_function(origheaders, row)
+                    if processed_row is not None:
+                        yield processed_row
+                else:
+                    yield row
 
         return headers, get_next()
 

src/hdx/utilities/path.py

@@ -51,15 +51,23 @@ def script_dir_plus_file(filename, pyobject, follow_symlinks=True):
     return join(script_dir(pyobject, follow_symlinks), filename)
 
 
-def get_temp_dir():
-    # type: () -> str
+def get_temp_dir(folder=None):
+    # type: (Optional[str]) -> str
     """Get a temporary directory. Looks for environment variable TEMP_DIR and falls
     back on os.gettempdir.
 
+    Args:
+        folder (Optional[str]): Folder to create in temporary folder. Defaults to None.
+
     Returns:
         str: A temporary directory
     """
-    return getenv('TEMP_DIR', gettempdir())
+    tempdir = getenv('TEMP_DIR', gettempdir())
+    if folder:
+        tempdir = join(tempdir, folder)
+        if not exists(tempdir):
+            makedirs(tempdir)
+    return tempdir
 
 
 @contextlib.contextmanager
@@ -75,11 +83,7 @@ def temp_dir(folder=None, delete_on_success=True, delete_on_failure=True):
     Returns:
         str: A temporary directory
     """
-    tempdir = get_temp_dir()
-    if folder:
-        tempdir = join(tempdir, folder)
-    if not exists(tempdir):
-        makedirs(tempdir)
+    tempdir = get_temp_dir(folder)
     try:
         yield tempdir
         if folder and delete_on_success:

src/hdx/utilities/version.txt

@@ -1 +1 @@
-2.2.1
+2.2.2

tests/hdx/utilities/test_path.py

@@ -16,6 +16,7 @@ def mytestdir(self):
 
     def test_get_temp_dir(self, monkeypatch, mytestdir):
         assert get_temp_dir() == gettempdir()
+        assert get_temp_dir('TEST') == join(gettempdir(), 'TEST')
         monkeypatch.setenv('TEMP_DIR', mytestdir)
         assert get_temp_dir() == mytestdir
         monkeypatch.delenv('TEMP_DIR')