HDXDSYS-927 Allow xlsx2csv option in datasetinfo used by reader (which is used by configurable scrapers)

documentation/main.md

@@ -1089,6 +1089,30 @@ same HXL hashtags.
         use_hxl: True
         should_overwrite_sources: True
 
+The xlsx2csv option below converts the xlsx to csv before processing.
+
+      idps_somalia:
+        dataset: "somalia-internally-displaced-persons-idps"
+        format: "xlsx"
+        xlsx2csv: True
+        filter_cols:
+          - "Reason"
+          - "Year"
+        prefilter: "'drought' in Reason.lower() and int(Year) in (self.today.year - 1, self.today.year)"
+        admin:
+          - value: "SOM"
+          - "Current (Arrival) District"
+        input:
+          - "Number of Individuals"
+        sum:
+          - formula: "int(Number of Individuals)"
+            mustbepopulated: True
+        output:
+          - "IDPs"
+        output_hxl:
+          - "#affected+idps+ind"
+        source_date_format: "%Y-%m-%d"
+
 ## Population Data
 
 Population data is treated as a special class of data. By default, configurable and

pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-api>=6.3.1",
+    "hdx-python-api>=6.3.2",
     "hdx-python-country>=3.7.6",
-    "hdx-python-utilities>=3.7.2",
+    "hdx-python-utilities>=3.7.3",
     "gspread",
     "regex",
 ]

requirements.txt

@@ -8,7 +8,7 @@ attrs==23.2.0
     #   jsonlines
     #   jsonschema
     #   referencing
-cachetools==5.3.3
+cachetools==5.4.0
     # via google-auth
 certifi==2024.7.4
     # via requests
@@ -26,7 +26,7 @@ click==8.1.7
     # via typer
 coverage==7.6.0
     # via pytest-cov
-cryptography==42.0.8
+cryptography==43.0.0
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
@@ -56,13 +56,13 @@ google-auth-oauthlib==1.2.1
     # via gspread
 gspread==6.1.2
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-api==6.3.1
+hdx-python-api==6.3.2
     # via hdx-python-scraper (pyproject.toml)
-hdx-python-country==3.7.6
+hdx-python-country==3.7.7
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
-hdx-python-utilities==3.7.2
+hdx-python-utilities==3.7.3
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   hdx-python-api
@@ -119,7 +119,7 @@ nodeenv==1.9.1
     # via pre-commit
 num2words==0.5.13
     # via quantulum3
-numpy==2.0.0
+numpy==2.0.1
     # via pandas
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -141,7 +141,7 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.7.1
+pre-commit==3.8.0
     # via hdx-python-scraper (pyproject.toml)
 pyasn1==0.6.0
     # via
@@ -159,13 +159,13 @@ pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
-pyopenssl==24.1.0
+pyopenssl==24.2.1
     # via
     #   hdx-python-api
     #   ndg-httpsclient
 pyphonetics==0.5.3
     # via hdx-python-country
-pytest==8.2.2
+pytest==8.3.2
     # via
     #   hdx-python-scraper (pyproject.toml)
     #   pytest-cov
@@ -198,7 +198,7 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2024.5.15
+regex==2024.7.24
     # via hdx-python-scraper (pyproject.toml)
 requests==2.32.3
     # via
@@ -216,7 +216,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.7.1
     # via typer
-rpds-py==0.19.0
+rpds-py==0.19.1
     # via
     #   jsonschema
     #   referencing
@@ -226,7 +226,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==70.3.0
+setuptools==72.1.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -245,7 +245,7 @@ sphinxcontrib-napoleon==0.7
     # via defopt
 stringcase==1.2.0
     # via frictionless
-structlog==24.2.0
+structlog==24.4.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -284,6 +284,8 @@ xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
+xlsx2csv==0.8.3
+    # via hdx-python-utilities
 xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0

src/hdx/scraper/utilities/reader.py

@@ -215,6 +215,7 @@ def read_tabular(
                 kwargs["fill_merged_cells"] = True
             elif "fill_merged_cells" not in kwargs:
                 kwargs["fill_merged_cells"] = False
+            kwargs["xlsx2csv"] = datasetinfo.get("xlsx2csv", False)
         if sheet:
             kwargs["sheet"] = sheet
         kwargs["headers"] = headers

tests/config/project_configuration.yaml

@@ -1049,10 +1049,11 @@ scraper_subnational:
   idps_somalia:
     dataset: "somalia-internally-displaced-persons-idps"
     format: "xlsx"
+    xlsx2csv: True
     filter_cols:
       - "Reason"
       - "Year"
-    prefilter: "'drought' in Reason.lower() and Year in (self.today.year - 1, self.today.year)"
+    prefilter: "'drought' in Reason.lower() and int(Year) in (self.today.year - 1, self.today.year)"
     admin:
       - value: "SOM"
       - "Current (Arrival) District"

tests/hdx/scraper/test_scrapers_subnational.py

@@ -353,10 +353,11 @@ def test_get_subnational(self, configuration):
     def test_fixed_country(self, configuration):
         BaseScraper.population_lookup = {}
         today = parse_date("2020-10-01")
-        adminlevel = AdminLevel(configuration["admin1"])
-        adminlevel.setup_from_admin_info(configuration["admin1"]["admin_info"])
         level = "subnational"
         scraper_configuration = configuration[f"scraper_{level}"]
+
+        adminlevel = AdminLevel(configuration["admin1"])
+        adminlevel.setup_from_admin_info(configuration["admin1"]["admin_info"])
         runner = Runner(("SOM",), today)
         runner.add_configurables(
             scraper_configuration, level, adminlevel=adminlevel