Use blank rows handling of Tabulator

requirements.txt

@@ -4,11 +4,11 @@ colorlog==4.1.0
 email_validator==1.0.5
 html5lib==1.0.1
 psycopg2-binary==2.8.4
-pyaml==19.12.0
+pyaml==20.3.1
 python-dateutil==2.8.1
 ratelimit==2.2.1
 six==1.14.0
 sshtunnel==0.1.5
-tabulator==1.35.0
+tabulator==1.36.0
 typing==3.7.4.1
 yamlloader==0.5.5

setup.py

@@ -19,7 +19,7 @@
                 'ratelimit',
                 'six>=1.14.0',
                 'sshtunnel',
-                'tabulator>=1.35.0',
+                'tabulator>=1.36.0',
                 'typing',
                 'yamlloader'
                 ]

src/hdx/utilities/downloader.py

@@ -404,6 +404,11 @@ def get_tabular_rows(self, url, headers=1, dict_form=False, ignore_blank_rows=Tr
         """
         if headers is None:
             raise DownloadError('Argument headers cannot be None!')
+        if ignore_blank_rows:
+            skip_rows = kwargs.get('skip_rows', list())
+            if {'type': 'preset', 'value': 'blank'} not in skip_rows:
+                skip_rows.append({'type': 'preset', 'value': 'blank'})
+            kwargs['skip_rows'] = skip_rows
         stream = self.get_tabular_stream(url, headers=headers, **kwargs)
         origheaders = stream.headers
         if header_insertions is None or origheaders is None:
@@ -417,13 +422,6 @@ def get_tabular_rows(self, url, headers=1, dict_form=False, ignore_blank_rows=Tr
 
         def get_next():
             for row in stream.iter(keyed=dict_form):
-                if ignore_blank_rows is True:
-                    if dict_form:
-                        if not any(row.values()):
-                            continue
-                    else:
-                        if not row:
-                            continue
                 if row_function:
                     processed_row = row_function(origheaders, row)
                     if processed_row is not None:

src/hdx/utilities/version.txt

@@ -1 +1 @@
-2.2.3
+2.2.4

tests/hdx/utilities/test_downloader.py

@@ -178,7 +178,7 @@ def test_setup_stream(self, fixtureurl, fixturenotexistsurl, getfixtureurl, post
         with Download() as downloader:
             downloader.setup(fixtureurl)
             headers = downloader.response.headers
-            assert headers['Content-Length'] == '742'
+            assert headers['Content-Length'] == '728'
         with Download() as downloader:
             downloader.setup(postfixtureurl, post=True)
             headers = downloader.response.headers
@@ -244,7 +244,7 @@ def test_download(self, fixtureurl, fixturenotexistsurl, getfixtureurl, postfixt
             downloader.download(fixturenotexistsurl)
         with Download() as downloader:
             result = downloader.download(fixtureurl)
-            assert result.headers['Content-Length'] == '742'
+            assert result.headers['Content-Length'] == '728'
             downloader.download('%s?id=10&lala=a' % getfixtureurl, post=False,
                                 parameters=OrderedDict([('b', '4'), ('d', '3')]))
             assert downloader.get_json()['args'] == OrderedDict([('b', '4'), ('d', '3'), ('id', '10'), ('lala', 'a')])