Use pdbufr to read DWD radar data in bufr format

.github/workflows/install.sh

@@ -10,7 +10,7 @@ fi
 echo "Installing package and requirements for ${flavor}"
 
 if [[ "${flavor}" = "testing" ]]; then
-  poetry install --no-interaction --extras=sql --extras=export --extras=restapi --extras=explorer
+  poetry install --no-interaction --extras=sql --extras=export --extras=restapi --extras=explorer --extras=bufr
 
 elif [[ "${flavor}" = "docs" ]]; then
   poetry install --no-interaction --extras=docs

.github/workflows/tests.yml

@@ -53,6 +53,12 @@ jobs:
         path: ${{ steps.poetry-cache-dir.outputs.dir }}
         key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}-${{ env.CACHE_NUMBER }}
 
+    - name: Install eccodes (Mac only)
+    - run: |
+      if ${{ matrix.os }} == "macos-latest"; then
+        brew install eccodes
+      fi
+
     - name: Install dependencies
       #run: poetry install --no-interaction --no-root --extras=sql --extras=export --extras=restapi --extras=explorer
       run: .github/workflows/install.sh testing

CHANGELOG.rst

@@ -9,6 +9,7 @@ Added
 
 - Enable selecting a parameter precisely from a dataset by passing a tuple like [("precipitation_height", "kl")] or
   [("precipitation_height", "precipitation_more")], or for cli/restapi use "precipitation_height/kl"
+- Allow parsing DWD radar data in bufr format to a pandas DataFrame
 
 Changed
 =======

pyproject.toml

@@ -104,7 +104,7 @@ click-params = "^0.1.1"
 cloup = "^0.8.0"
 
 # Optional dependencies aka. "extras"
-matplotlib =  { version = "^3.3.2", optional = true }
+matplotlib                      =  { version = "^3.3.2", optional = true }
 
 openpyxl                        = { version = "^3.0.7", optional = true }
 pyarrow                         = { version = "^3.0.0", optional = true, markers = "sys_platform != 'darwin' or (sys_platform == 'darwin' and platform_machine != 'arm64')" }
@@ -119,23 +119,26 @@ psycopg2-binary                 = { version = "^2.8.6", optional = true }
 # HTTP REST API service
 fastapi                         = { version = "^0.61.1", optional = true }
 uvicorn                         = { version = "^0.13.3", optional = true }
+
+# Radar
 wradlib                         = { version = "^1.9.0", optional = true }
+eccodes                         = { version = "^1.2.0", optional = true }
+pdbufr                          = { version = "^0.9.0", optional = true, extras = ["eccodes"] }
 
 # Explorer UI service
 plotly                          = { version = "^4.14.3", optional = true }
 dash                            = { version = "^1.19.0", optional = true }
 dash-bootstrap-components       = { version = "^0.12.0", optional = true }
 
-sphinx = { version = "^3.2.1", optional = true }
-sphinx-material = { version = "^0.0.30", optional = true }
-sphinx-autodoc-typehints = { version = "^1.11.0", optional = true }
-sphinxcontrib-svg2pdfconverter = { version = "^1.1.0", optional = true }
-tomlkit = { version = "^0.7.0", optional = true }
-ipython = { version = "^7.10.1", optional = true }
-ipython-genutils = { version = "^0.2.0", optional = true }
-zarr = { version = "^2.7.0", optional = true, markers = "sys_platform != 'darwin' or (sys_platform == 'darwin' and platform_machine != 'arm64')" }  # not supported through numcodecs
-xarray = { version = "^0.17.0", optional = true }
-
+sphinx                          = { version = "^3.2.1", optional = true }
+sphinx-material                 = { version = "^0.0.30", optional = true }
+sphinx-autodoc-typehints        = { version = "^1.11.0", optional = true }
+sphinxcontrib-svg2pdfconverter  = { version = "^1.1.0", optional = true }
+tomlkit                         = { version = "^0.7.0", optional = true }
+ipython                         = { version = "^7.10.1", optional = true }
+ipython-genutils                = { version = "^0.2.0", optional = true }
+zarr                            = { version = "^2.7.0", optional = true, markers = "sys_platform != 'darwin' or (sys_platform == 'darwin' and platform_machine != 'arm64')" }  # not supported through numcodecs
+xarray                          = { version = "^0.17.0", optional = true }
 
 [tool.poetry.dev-dependencies]
 black = "^20.8b1"
@@ -183,8 +186,8 @@ influxdb = ["influxdb", "influxdb-client"]
 cratedb = ["crate"]
 mysql = ["mysqlclient"]
 postgresql = ["psycopg2-binary"]
-radar = ["wradlib", "pybufrkit", "h5py"]
-bufr = ["pybufrkit"]
+radar = ["wradlib", "pybufrkit", "h5py", "pdbufr"]
+bufr = ["pybufrkit", "pdbufr"]
 
 [tool.poetry.scripts]
 wetterdienst = 'wetterdienst.ui.cli:cli'

tests/__init__.py

@@ -10,3 +10,7 @@
 mac_arm64_unsupported = pytest.mark.skipif(
     mac_arm64, reason="can't be tested under mac arm64 due to h5py incompatibility"
 )
+mac_only = pytest.mark.skipif(
+    sys.platform.startswith("win") or sys.platform.startswith("lin"),
+    reason="can't be tested under windows due to eccodes incompatibility",
+)

tests/provider/dwd/radar/test_api_historic.py

@@ -11,6 +11,7 @@
 import pytest
 import requests
 
+from tests import mac_only
 from tests.provider.dwd.radar import (
     station_reference_pattern_de,
     station_reference_pattern_sorted,
@@ -404,6 +405,44 @@ def test_radar_request_site_historic_pe_bufr():
     decoder.process(payload, info_only=True)
 
 
+@mac_only
+@pytest.mark.remote
+def test_radar_request_site_historic_pe_bufr_dataframe():
+    """
+    Verify acquisition of radar/site/PE_ECHO_TOP data works
+    when using a specific date.
+
+    This time, we will use the BUFR data format.
+    """
+
+    # Acquire data from yesterday at this time.
+    timestamp = datetime.utcnow() - timedelta(days=1)
+
+    request = DwdRadarValues(
+        parameter=DwdRadarParameter.PE_ECHO_TOP,
+        start_date=timestamp,
+        site=DwdRadarSite.BOO,
+        fmt=DwdRadarDataFormat.BUFR,
+        read_bufr=True,
+    )
+
+    results = list(request.query())
+
+    if len(results) == 0:
+        raise pytest.skip("Data currently not available")
+
+    df = results[0].data
+
+    assert not df.empty
+
+    assert df.columns == [
+        "station_id",
+        "latitude",
+        "longitude",
+        "horizontalReflectivity",
+    ]
+
+
 @pytest.mark.remote
 @pytest.mark.parametrize(
     "format",

wetterdienst/provider/dwd/radar/access.py

@@ -8,11 +8,12 @@
 from dataclasses import dataclass
 from datetime import datetime
 from io import BytesIO
-from typing import Generator, Optional, Tuple
+from typing import Generator, Optional, Tuple, Union
 
 import pandas as pd
 
 from wetterdienst.exceptions import FailedDownload
+from wetterdienst.metadata.columns import Columns
 from wetterdienst.metadata.extension import Extension
 from wetterdienst.metadata.period import Period
 from wetterdienst.metadata.resolution import Resolution
@@ -46,7 +47,7 @@ class RadarResult:
     Currently, this will relate to exactly one radar data file.
     """
 
-    data: BytesIO
+    data: Union[BytesIO, pd.DataFrame]
     timestamp: datetime = None
     url: str = None
     filename: str = None
@@ -79,6 +80,7 @@ def collect_radar_data(
     start_date: Optional[datetime] = None,
     end_date: Optional[datetime] = None,
     verify: Optional[bool] = True,
+    read_bufr: bool = False,
 ) -> RadarResult:
     """
     Collect radar data for given parameters.
@@ -95,10 +97,14 @@ def collect_radar_data(
     :param start_date:      Start date
     :param end_date:        End date
     :param verify:          Whether to verify the response
+    :param read_bufr:       read bufr to pd.DataFrame
 
     :return:                ``RadarResult`` item
     """
 
+    if read_bufr:
+        import pdbufr
+
     # Find latest file.
     if start_date == DwdRadarDate.LATEST:
 
@@ -212,6 +218,28 @@ def collect_radar_data(
                         if result.timestamp is None:
                             result.timestamp = date_time
 
+                        if fmt == DwdRadarDataFormat.BUFR:
+                            df = pdbufr.read_bufr(
+                                result.data,
+                                columns=(
+                                    "stationNumber",
+                                    "latitude",
+                                    "longitude",
+                                    "horizontalReflectivity",
+                                ),
+                            )
+
+                            df = df.rename(
+                                columns={
+                                    "stationNumber": Columns.STATION_ID.value,
+                                    "latitude": Columns.LATITUDE.value,
+                                    "longitude": Columns.LONGITUDE.value,
+                                }
+                            )
+
+                            # rewrite data to result
+                            result.data = df
+
                         if verify:
                             if fmt == DwdRadarDataFormat.HDF5:
                                 verify_hdf5(result.data)