From 2e0a0095a56cb934a9a2079a670349235d14c5f1 Mon Sep 17 00:00:00 2001 From: benjamin Date: Sat, 22 Jan 2022 02:00:57 +0100 Subject: [PATCH] Format code with line length 120 Add more linting checks --- example/observations_sql.py | 5 +- example/radar/radar_composite_rw.py | 4 +- example/radar/radar_radolan_cdc.py | 4 +- example/radar/radar_radolan_rw.py | 4 +- example/radar/radar_scan_precip.py | 5 +- example/radar/radar_scan_volume.py | 5 +- poetry.lock | 152 +++++++++++-- pyproject.toml | 95 ++++---- tests/example/test_notebook_examples.py | 5 +- tests/provider/dwd/forecast/test_api_data.py | 4 +- .../provider/dwd/observation/test_api_data.py | 22 +- .../dwd/observation/test_api_stations_geo.py | 4 +- tests/provider/dwd/observation/test_io.py | 67 ++---- .../dwd/observation/test_metaindex.py | 4 +- .../provider/dwd/observation/util/__init__.py | 6 +- tests/provider/dwd/radar/__init__.py | 6 +- tests/provider/dwd/radar/test_api_current.py | 4 +- tests/provider/dwd/radar/test_api_historic.py | 92 ++------ tests/provider/dwd/radar/test_api_latest.py | 17 +- .../dwd/radar/test_api_most_recent.py | 7 +- tests/provider/dwd/radar/test_api_recent.py | 2 +- tests/provider/dwd/radar/test_index.py | 55 ++--- tests/provider/dwd/radar/test_util.py | 8 +- tests/provider/dwd/test_date.py | 8 +- tests/provider/dwd/test_index.py | 3 +- tests/provider/dwd/test_util.py | 5 +- tests/provider/eccc/test_api.py | 4 +- tests/test_api.py | 4 +- tests/ui/explorer/conftest.py | 4 +- tests/ui/test_cli.py | 69 ++---- tests/ui/test_restapi.py | 16 +- tests/util/test_geo.py | 4 +- wetterdienst/__init__.py | 8 +- wetterdienst/api.py | 4 +- wetterdienst/core/process.py | 20 +- wetterdienst/core/scalar/export.py | 28 +-- wetterdienst/core/scalar/request.py | 119 +++------- wetterdienst/core/scalar/result.py | 11 +- wetterdienst/core/scalar/values.py | 144 +++---------- wetterdienst/metadata/parameter.py | 204 +++++------------- wetterdienst/metadata/period.py | 10 +- wetterdienst/metadata/provider.py | 3 +- wetterdienst/metadata/unit.py | 8 +- wetterdienst/provider/dwd/forecast/access.py | 27 +-- wetterdienst/provider/dwd/forecast/api.py | 53 ++--- wetterdienst/provider/dwd/index.py | 4 +- wetterdienst/provider/dwd/observation/api.py | 175 +++++---------- .../provider/dwd/observation/download.py | 8 +- .../provider/dwd/observation/fields.py | 23 +- .../provider/dwd/observation/fileindex.py | 25 +-- .../dwd/observation/metadata/dataset.py | 4 +- .../dwd/observation/metadata/parameter.py | 4 +- .../provider/dwd/observation/metaindex.py | 38 +--- .../provider/dwd/observation/parser.py | 29 +-- .../dwd/observation/util/parameter.py | 22 +- wetterdienst/provider/dwd/radar/access.py | 30 +-- wetterdienst/provider/dwd/radar/api.py | 28 +-- wetterdienst/provider/dwd/radar/index.py | 39 ++-- .../provider/dwd/radar/metadata/parameter.py | 2 +- wetterdienst/provider/dwd/radar/sites.py | 20 +- wetterdienst/provider/eccc/observation/api.py | 27 +-- wetterdienst/provider/eumetnet/opera/sites.py | 17 +- wetterdienst/provider/noaa/ghcn/api.py | 23 +- wetterdienst/ui/cli.py | 8 +- wetterdienst/ui/core.py | 7 +- wetterdienst/ui/explorer/app.py | 23 +- .../explorer/layout/observations_germany.py | 14 +- wetterdienst/ui/explorer/library.py | 8 +- wetterdienst/ui/restapi.py | 20 +- wetterdienst/util/cache.py | 16 +- wetterdienst/util/datetime.py | 4 +- wetterdienst/util/enumeration.py | 8 +- wetterdienst/util/geo.py | 8 +- wetterdienst/util/io.py | 2 +- wetterdienst/util/logging.py | 2 +- wetterdienst/util/network.py | 16 +- 76 files changed, 648 insertions(+), 1339 deletions(-) diff --git a/example/observations_sql.py b/example/observations_sql.py index fad5a98b7..d680a2abf 100644 --- a/example/observations_sql.py +++ b/example/observations_sql.py @@ -41,10 +41,7 @@ def sql_example(): stations = request.filter_by_station_id(station_id=(1048,)) - sql = ( - "SELECT * FROM data WHERE " - "parameter='temperature_air_mean_200' AND value < -7.0;" - ) + sql = "SELECT * FROM data WHERE " "parameter='temperature_air_mean_200' AND value < -7.0;" log.info(f"Invoking SQL query '{sql}'") # Acquire observation values and filter with SQL. diff --git a/example/radar/radar_composite_rw.py b/example/radar/radar_composite_rw.py index 5a864ae91..a32e68ca7 100644 --- a/example/radar/radar_composite_rw.py +++ b/example/radar/radar_composite_rw.py @@ -70,9 +70,7 @@ def plot_radolan(data: np.ndarray, attrs: dict, grid: np.dstack, clabel: str = N cb.set_label(clabel) plt.xlabel("x [km]") plt.ylabel("y [km]") - plt.title( - "{0} Product\n{1}".format(attrs["producttype"], attrs["datetime"].isoformat()) - ) + plt.title("{0} Product\n{1}".format(attrs["producttype"], attrs["datetime"].isoformat())) plt.xlim((x[0, 0], x[-1, -1])) plt.ylim((y[0, 0], y[-1, -1])) plt.grid(color="r") diff --git a/example/radar/radar_radolan_cdc.py b/example/radar/radar_radolan_cdc.py index ed3302804..0285fb618 100644 --- a/example/radar/radar_radolan_cdc.py +++ b/example/radar/radar_radolan_cdc.py @@ -95,9 +95,7 @@ def plot_radolan(data: np.ndarray, attrs: dict, grid: np.dstack, clabel: str = N cb.set_label(clabel) plt.xlabel("x [km]") plt.ylabel("y [km]") - plt.title( - "{0} Product\n{1}".format(attrs["producttype"], attrs["datetime"].isoformat()) - ) + plt.title("{0} Product\n{1}".format(attrs["producttype"], attrs["datetime"].isoformat())) plt.xlim((x[0, 0], x[-1, -1])) plt.ylim((y[0, 0], y[-1, -1])) plt.grid(color="r") diff --git a/example/radar/radar_radolan_rw.py b/example/radar/radar_radolan_rw.py index 27069f590..c0a1cf238 100644 --- a/example/radar/radar_radolan_rw.py +++ b/example/radar/radar_radolan_rw.py @@ -94,9 +94,7 @@ def plot_radolan(data: np.ndarray, attrs: dict, grid: np.dstack, clabel: str = N cb.set_label(clabel) plt.xlabel("x [km]") plt.ylabel("y [km]") - plt.title( - "{0} Product\n{1}".format(attrs["producttype"], attrs["datetime"].isoformat()) - ) + plt.title("{0} Product\n{1}".format(attrs["producttype"], attrs["datetime"].isoformat())) plt.xlim((x[0, 0], x[-1, -1])) plt.ylim((y[0, 0], y[-1, -1])) plt.grid(color="r") diff --git a/example/radar/radar_scan_precip.py b/example/radar/radar_scan_precip.py index 36e25a7e5..6c27d653b 100644 --- a/example/radar/radar_scan_precip.py +++ b/example/radar/radar_scan_precip.py @@ -94,10 +94,7 @@ def radar_scan_precip(): subset=DwdRadarDataSubset.POLARIMETRIC, ) - log.info( - f"Acquiring radar SWEEP_PCP data for {DwdRadarSite.ESS} at " - f"{request_velocity.start_date}" - ) + log.info(f"Acquiring radar SWEEP_PCP data for {DwdRadarSite.ESS} at " f"{request_velocity.start_date}") # Submit requests. results = chain(request_velocity.query(), request_reflectivity.query()) diff --git a/example/radar/radar_scan_volume.py b/example/radar/radar_scan_volume.py index b28860783..7663378e4 100644 --- a/example/radar/radar_scan_volume.py +++ b/example/radar/radar_scan_volume.py @@ -94,10 +94,7 @@ def radar_scan_volume(): subset=DwdRadarDataSubset.POLARIMETRIC, ) - log.info( - f"Acquiring radar SWEEP_VOL data for {DwdRadarSite.ESS} at " - f"{request_velocity.start_date}" - ) + log.info(f"Acquiring radar SWEEP_VOL data for {DwdRadarSite.ESS} at " f"{request_velocity.start_date}") # Submit requests. results = chain(request_velocity.query(), request_reflectivity.query()) diff --git a/poetry.lock b/poetry.lock index 680ea23c7..e1da0a025 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,6 +605,14 @@ category = "dev" optional = false python-versions = ">=2.7" +[[package]] +name = "eradicate" +version = "2.0.0" +description = "Removes commented-out code." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "et-xmlfile" version = "1.1.0" @@ -664,6 +672,18 @@ mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.8.0,<2.9.0" pyflakes = ">=2.4.0,<2.5.0" +[[package]] +name = "flake8-2020" +version = "1.6.1" +description = "flake8 plugin which checks for misuse of `sys.version` or `sys.version_info`" +category = "dev" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +flake8 = ">=3.7" +importlib-metadata = {version = ">=0.9", markers = "python_version < \"3.8\""} + [[package]] name = "flake8-bandit" version = "2.1.2" @@ -732,6 +752,39 @@ python-versions = ">=3.7" flake8 = ">=3.0,<3.2.0 || >3.2.0" importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +[[package]] +name = "flake8-copyright" +version = "0.2.2" +description = "Adds copyright checks to flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "flake8-docstrings" +version = "1.6.0" +description = "Extension for flake8 which uses pydocstyle to check docstrings" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +flake8 = ">=3" +pydocstyle = ">=2.1" + +[[package]] +name = "flake8-eradicate" +version = "1.2.0" +description = "Flake8 plugin to find commented out code" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +attrs = "*" +eradicate = ">=2.0,<3.0" +flake8 = ">=3.5,<5" + [[package]] name = "flake8-isort" version = "4.1.1" @@ -748,6 +801,14 @@ testfixtures = ">=6.8.0,<7" [package.extras] test = ["pytest-cov"] +[[package]] +name = "flake8-plugin-utils" +version = "1.3.2" +description = "The package provides base classes and utils for flake8 plugin writing" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + [[package]] name = "flake8-polyfill" version = "1.0.2" @@ -773,32 +834,36 @@ pycodestyle = "*" six = "*" [[package]] -name = "flakehell" -version = "0.8.0" +name = "flake8-return" +version = "1.1.3" +description = "Flake8 plugin that checks return values" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +flake8-plugin-utils = ">=1.0,<2.0" + +[[package]] +name = "flakeheaven" +version = "0.11.0" description = "Flake8 wrapper to make it nice and configurable" category = "dev" optional = false python-versions = ">=3.5" -develop = false [package.dependencies] colorama = "*" entrypoints = "*" -flake8 = ">=3.8.0" +flake8 = ">=4.0.1" pygments = "*" toml = "*" urllib3 = "*" [package.extras] -dev = ["dlint", "flake8-2020", "flake8-aaa", "flake8-absolute-import", "flake8-alfred", "flake8-annotations-complexity", "flake8-bandit", "flake8-black", "flake8-broken-line", "flake8-bugbear", "flake8-builtins", "flake8-coding", "flake8-cognitive-complexity", "flake8-commas", "flake8-comprehensions", "flake8-debugger", "flake8-django", "flake8-docstrings", "flake8-eradicate", "flake8-executable", "flake8-expression-complexity", "flake8-fixme", "flake8-functions", "flake8-future-import", "flake8-import-order", "flake8-isort", "flake8-logging-format", "flake8-mock", "flake8-mutable", "flake8-mypy", "flake8-pep3101", "flake8-pie", "flake8-print", "flake8-printf-formatting", "flake8-pyi", "flake8-pytest-style", "flake8-pytest", "flake8-quotes", "flake8-requirements", "flake8-rst-docstrings", "flake8-scrapy", "flake8-spellcheck", "flake8-sql", "flake8-strict", "flake8-string-format", "flake8-tidy-imports", "flake8-todo", "flake8-use-fstring", "flake8-variables-names", "mccabe", "pandas-vet", "pep8-naming", "pylint", "typing-extensions", "wemake-python-styleguide", "pytest", "isort"] +dev = ["dlint", "flake8-2020", "flake8-aaa", "flake8-absolute-import", "flake8-alfred", "flake8-annotations-complexity", "flake8-bandit", "flake8-black", "flake8-broken-line", "flake8-bugbear", "flake8-builtins", "flake8-coding", "flake8-cognitive-complexity", "flake8-commas", "flake8-comprehensions", "flake8-debugger", "flake8-django", "flake8-docstrings", "flake8-eradicate", "flake8-executable", "flake8-expression-complexity", "flake8-fixme", "flake8-functions", "flake8-future-import", "flake8-import-order", "flake8-isort", "flake8-logging-format", "flake8-mock", "flake8-mutable", "flake8-mypy", "flake8-pep3101", "flake8-pie", "flake8-print", "flake8-printf-formatting", "flake8-pyi", "flake8-pytest-style", "flake8-pytest", "flake8-quotes", "flake8-requirements", "flake8-rst-docstrings", "flake8-scrapy", "flake8-spellcheck", "flake8-sql", "flake8-strict", "black (==21.10b0)", "flake8-string-format", "flake8-tidy-imports", "flake8-todo", "flake8-use-fstring", "flake8-variables-names", "mccabe", "pandas-vet", "pep8-naming", "pylint", "typing-extensions", "wemake-python-styleguide", "mypy", "pytest", "isort"] docs = ["alabaster", "pygments-github-lexers", "recommonmark", "sphinx"] -[package.source] -type = "git" -url = "https://github.com/mcarans/flakehell.git" -reference = "master" -resolved_reference = "1b84f4dd6c16232b5c0c6206511427676ab55f5b" - [[package]] name = "flask" version = "2.0.2" @@ -1183,7 +1248,7 @@ format_nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- [[package]] name = "jupyter-client" -version = "7.1.1" +version = "7.1.2" description = "Jupyter protocol implementation and client libraries" category = "dev" optional = false @@ -1216,7 +1281,7 @@ traitlets = "*" [[package]] name = "jupyter-server" -version = "1.13.3" +version = "1.13.4" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." category = "dev" optional = false @@ -1850,6 +1915,20 @@ typing-extensions = ">=3.7.4.3" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pydocstyle" +version = "6.1.1" +description = "Python docstring style checker" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +snowballstemmer = "*" + +[package.extras] +toml = ["toml"] + [[package]] name = "pyflakes" version = "2.4.0" @@ -2848,7 +2927,7 @@ sql = ["duckdb"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "e073da8b224a91579fea249bdf234656e26134ef38d460c5e94354aaf8b330af" +content-hash = "d475e9b420dcb4109bc8652386939fdba84bc667b8fe63f76cc38673a66ff895" [metadata.files] aenum = [ @@ -3352,6 +3431,9 @@ entrypoints = [ {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, ] +eradicate = [ + {file = "eradicate-2.0.0.tar.gz", hash = "sha256:27434596f2c5314cc9b31410c93d8f7e8885747399773cd088d3adea647a60c8"}, +] et-xmlfile = [ {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, @@ -3372,6 +3454,10 @@ flake8 = [ {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, ] +flake8-2020 = [ + {file = "flake8_2020-1.6.1-py2.py3-none-any.whl", hash = "sha256:efcc056fb723e1ea5307e3b663c7c328f1c23a5ff0a0fd3be695a918d8245c3a"}, + {file = "flake8_2020-1.6.1.tar.gz", hash = "sha256:db523e3383befc17c895219551ff6c9b2f6e0a5cae4c7739ea65a2238bdc6f74"}, +] flake8-bandit = [ {file = "flake8_bandit-2.1.2.tar.gz", hash = "sha256:687fc8da2e4a239b206af2e54a90093572a60d0954f3054e23690739b0b0de3b"}, ] @@ -3391,10 +3477,27 @@ flake8-comprehensions = [ {file = "flake8-comprehensions-3.8.0.tar.gz", hash = "sha256:8e108707637b1d13734f38e03435984f6b7854fa6b5a4e34f93e69534be8e521"}, {file = "flake8_comprehensions-3.8.0-py3-none-any.whl", hash = "sha256:9406314803abe1193c064544ab14fdc43c58424c0882f6ff8a581eb73fc9bb58"}, ] +flake8-copyright = [ + {file = "flake8-copyright-0.2.2.tar.gz", hash = "sha256:5c3632dd8c586547b25fff4272282005fdbcba56eeb77b7487564aa636b6e533"}, + {file = "flake8_copyright-0.2.2-py2.py3-none-any.whl", hash = "sha256:616a960c9602ad2d0136bf3f12564e253caffe82f151d2982f78a12a42e1faa0"}, + {file = "flake8_copyright-0.2.2-py3-none-any.whl", hash = "sha256:dbad92ee5f51398722cd571b6e36cc3651914bf1b286b0e638bba1f4af0b6f5b"}, +] +flake8-docstrings = [ + {file = "flake8-docstrings-1.6.0.tar.gz", hash = "sha256:9fe7c6a306064af8e62a055c2f61e9eb1da55f84bb39caef2b84ce53708ac34b"}, + {file = "flake8_docstrings-1.6.0-py2.py3-none-any.whl", hash = "sha256:99cac583d6c7e32dd28bbfbef120a7c0d1b6dde4adb5a9fd441c4227a6534bde"}, +] +flake8-eradicate = [ + {file = "flake8-eradicate-1.2.0.tar.gz", hash = "sha256:acaa1b6839ff00d284b805c432fdfa6047262bd15a5504ec945797e87b4de1fa"}, + {file = "flake8_eradicate-1.2.0-py3-none-any.whl", hash = "sha256:51dc660d0c1c1ed93af0f813540bbbf72ab2d3466c14e3f3bac371c618b6042f"}, +] flake8-isort = [ {file = "flake8-isort-4.1.1.tar.gz", hash = "sha256:d814304ab70e6e58859bc5c3e221e2e6e71c958e7005239202fee19c24f82717"}, {file = "flake8_isort-4.1.1-py3-none-any.whl", hash = "sha256:c4e8b6dcb7be9b71a02e6e5d4196cefcef0f3447be51e82730fb336fff164949"}, ] +flake8-plugin-utils = [ + {file = "flake8-plugin-utils-1.3.2.tar.gz", hash = "sha256:20fa2a8ca2decac50116edb42e6af0a1253ef639ad79941249b840531889c65a"}, + {file = "flake8_plugin_utils-1.3.2-py3-none-any.whl", hash = "sha256:1fe43e3e9acf3a7c0f6b88f5338cad37044d2f156c43cb6b080b5f9da8a76f06"}, +] flake8-polyfill = [ {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, {file = "flake8_polyfill-1.0.2-py2.py3-none-any.whl", hash = "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9"}, @@ -3403,7 +3506,14 @@ flake8-print = [ {file = "flake8-print-4.0.0.tar.gz", hash = "sha256:5afac374b7dc49aac2c36d04b5eb1d746d72e6f5df75a6ecaecd99e9f79c6516"}, {file = "flake8_print-4.0.0-py3-none-any.whl", hash = "sha256:6c0efce658513169f96d7a24cf136c434dc711eb00ebd0a985eb1120103fe584"}, ] -flakehell = [] +flake8-return = [ + {file = "flake8-return-1.1.3.tar.gz", hash = "sha256:13a31edeb0c6157dd4f77166cdaa6141703d2b8b24def5558ae659852a003cb4"}, + {file = "flake8_return-1.1.3-py3-none-any.whl", hash = "sha256:4a266191f7ed69aa26b835ec90c5a5522fa8f79f5cf6363a877ac499f8eb418b"}, +] +flakeheaven = [ + {file = "flakeheaven-0.11.0-py3-none-any.whl", hash = "sha256:7c13bce95cfa496c47e46532f4ed0a020a6532d88f037e79bb9e342d9446992d"}, + {file = "flakeheaven-0.11.0.tar.gz", hash = "sha256:002f2de2bbc2ae72c2920cfc91cf068c89b7a561c61f63efe4a06c4af146c2dd"}, +] flask = [ {file = "Flask-2.0.2-py3-none-any.whl", hash = "sha256:cb90f62f1d8e4dc4621f52106613488b5ba826b2e1e10a33eac92f723093ab6a"}, {file = "Flask-2.0.2.tar.gz", hash = "sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2"}, @@ -3583,16 +3693,16 @@ jsonschema = [ {file = "jsonschema-4.4.0.tar.gz", hash = "sha256:636694eb41b3535ed608fe04129f26542b59ed99808b4f688aa32dcf55317a83"}, ] jupyter-client = [ - {file = "jupyter_client-7.1.1-py3-none-any.whl", hash = "sha256:f0c576cce235c727e30b0a0da88c2755d0947d0070fa1bc45f195079ffd64e66"}, - {file = "jupyter_client-7.1.1.tar.gz", hash = "sha256:540ca35e57e83c5ece81abd9b781a57cba39a37c60a2a30c8c1b2f6663544343"}, + {file = "jupyter_client-7.1.2-py3-none-any.whl", hash = "sha256:d56f1c57bef42ff31e61b1185d3348a5b2bcde7c9a05523ae4dbe5ee0871797c"}, + {file = "jupyter_client-7.1.2.tar.gz", hash = "sha256:4ea61033726c8e579edb55626d8ee2e6bf0a83158ddf3751b8dd46b2c5cd1e96"}, ] jupyter-core = [ {file = "jupyter_core-4.9.1-py3-none-any.whl", hash = "sha256:1c091f3bbefd6f2a8782f2c1db662ca8478ac240e962ae2c66f0b87c818154ea"}, {file = "jupyter_core-4.9.1.tar.gz", hash = "sha256:dce8a7499da5a53ae3afd5a9f4b02e5df1d57250cf48f3ad79da23b4778cd6fa"}, ] jupyter-server = [ - {file = "jupyter_server-1.13.3-py3-none-any.whl", hash = "sha256:3608129b90cfdcfb7dd275f15a1113d119b7c19e8356303b14312ac5c216c42a"}, - {file = "jupyter_server-1.13.3.tar.gz", hash = "sha256:4d622161f4d378ff28548b49cc180024ce102d25ba5805821fcc17ab1bc5c754"}, + {file = "jupyter_server-1.13.4-py3-none-any.whl", hash = "sha256:3a1df2e27a322e84c028e52272e6ff72fd875f9a74c84409263c5c2f1afbf6fa"}, + {file = "jupyter_server-1.13.4.tar.gz", hash = "sha256:5fb5a219385338b1d13a013a68f54688b6a69ecff4e757fd230e27ecacdbf212"}, ] jupyter-server-mathjax = [ {file = "jupyter_server_mathjax-0.2.3-py3-none-any.whl", hash = "sha256:740de2ed0d370f1856faddfaf8c09a6d7435d09d3672f24826451467b268969d"}, @@ -4322,6 +4432,10 @@ pydantic = [ {file = "pydantic-1.9.0-py3-none-any.whl", hash = "sha256:085ca1de245782e9b46cefcf99deecc67d418737a1fd3f6a4f511344b613a5b3"}, {file = "pydantic-1.9.0.tar.gz", hash = "sha256:742645059757a56ecd886faf4ed2441b9c0cd406079c2b4bee51bcc3fbcd510a"}, ] +pydocstyle = [ + {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, + {file = "pydocstyle-6.1.1.tar.gz", hash = "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc"}, +] pyflakes = [ {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, diff --git a/pyproject.toml b/pyproject.toml index ec26e8fe5..182e2e9b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,33 +141,52 @@ timezonefinder = "^5.2.0" [tool.poetry.dev-dependencies] +poethepoet = "^0.9.0" + +pip-licenses = "^3.3.0" + +# Formatting/Linting black = "^20.8b1" -flakehell = { git = "https://github.com/mcarans/flakehell.git" }# "^0.7.1" + isort = "^5.7.0" + +flakeheaven = "^0.11.0" +flake8-bandit = "^2.1.2" flake8-black = "^0.2.1" flake8-bugbear = "^20.1.4" -flake8-bandit = "^2.1.2" +flake8-builtins = "^1.5.3" +flake8-comprehensions = "^3.7.0" +flake8-copyright = "^0.2.2" +flake8-docstrings = "^1.6.0" +flake8-eradicate = "^1.2.0" flake8-isort = "^4.0.0" -coverage = { version = "^5.3", extras = ["toml"] } +flake8-print = "^4.0.0" +flake8-return = "^1.1.3" +flake8-2020 = "^1.6.1" + +# Testing pytest = "^6.0.2" pytest-cov = "^2.10.1" -pytest-notebook = "^0.6.1" pytest-dictsdiff = "^0.5.8" +pytest-notebook = "^0.6.1" +pytest-xdist = "^2.2.1" + mock = "^4.0.2" +freezegun = "^1.1.0" + +# Test required libraries surrogate = "^0.1" pybufrkit = "^0.2.17" -freezegun = "^1.1.0" -poethepoet = "^0.9.0" -pip-licenses = "^3.3.0" -sphinx-autobuild = "^2020.9.1" selenium = "^3.141.0" percy = "^2.0.2" h5py = "^3.1.0" h5netcdf = "^0.11.0" -pytest-xdist = "^2.2.1" -flake8-print = "^4.0.0" -flake8-builtins = "^1.5.3" -flake8-comprehensions = "^3.7.0" + +# Coverage +coverage = { version = "^5.3", extras = ["toml"] } + +# Docs +sphinx-autobuild = "^2020.9.1" [tool.poetry.extras] mpl = ["matplotlib"] @@ -201,62 +220,53 @@ wddump = 'wetterdienst.provider.dwd.radar.cli:wddump' from = {format = "poetry", path = "pyproject.toml"} to = {format = "pip", path = "requirements.txt"} +[tool.black] +line-length = 120 + [tool.isort] profile = "black" multi_line_output = 3 -[tool.flakehell] +[tool.flakeheaven] exclude = [ "example/climate_observations.ipynb" ] -max-line-length = 88 +format = "grouped" +max-line-length = 120 +show_source = true extended_default_ignore = [] -[tool.flakehell.plugins] -pycodestyle = ["+*", "-E203", "-W503"] +[tool.flakeheaven.plugins] +pycodestyle = ["+*", "-E203", "-W503", "-E501", "-E231"] pyflakes = ["+*"] flake8-bandit = ["+*"] flake8-black = ["+*"] flake8-bugbear = ["+*"] -flake8-isort = ["+*"] -flake8-print = ["+*"] flake8-builtins = ["+*", "-A003"] flake8-comprehensions = ["+*"] +flake8-copyright = ["+*"] +flake8-docstrings = ["+*"] +flake8-eradicate = ["+*"] +flake8-isort = ["+*"] +flake8-print = ["+*"] +flake8-return = ["+*"] +flake8-2020 = ["+*"] -[tool.flakehell.exceptions."**/__init__.py"] +[tool.flakeheaven.exceptions."**/__init__.py"] pyflakes = ["-F401"] - -[tool.flakehell.exceptions."example/"] +[tool.flakeheaven.exceptions."example/"] flake8-print = ["-*"] - -[tool.flakehell.exceptions."tests/"] +[tool.flakeheaven.exceptions."tests/"] flake8-bandit = ["-S101", "-S106"] -[tool.flakehell.exceptions."tests/provider/dwd/radar/test_index.py"] -pycodestyle = ["-E501", "-B950"] -[tool.flakehell.exceptions."tests/provider/dwd/observation/util/test_parameter.py"] -pycodestyle = ["-E501", "-B950"] - -[tool.flakehell.exceptions."wetterdienst/ui/cli.py"] -pycodestyle = ["-E501", "-B950"] -[tool.flakehell.exceptions."wetterdienst/ui/restapi.py"] +[tool.flakeheaven.exceptions."wetterdienst/ui/restapi.py"] flake8-bugbear = ["-B008"] -[tool.flakehell.exceptions."wetterdienst/core/scalar/export.py"] -pycodestyle = ["-E501", "-B950"] -[tool.flakehell.exceptions."wetterdienst/provider/dwd/observation/metadata/field_types.py"] -pycodestyle = ["-E501", "-B950"] -[tool.flakehell.exceptions."wetterdienst/provider/dwd/observation/metadata/parameter.py"] -pycodestyle = ["-E501", "-B950"] -[tool.flakehell.exceptions."wetterdienst/provider/dwd/forecast/metadata/field_types.py"] -pycodestyle = ["-E501", "-B950"] -[tool.flakehell.exceptions."tests/provider/dwd/observation/test_api_metadata.py"] -pycodestyle = ["-E501", "-B950"] [tool.poe.tasks] install_dev = "poetry install -E mpl -E ipython -E docs -E sql -E export -E duckdb -E influxdb -E cratedb -E mysql -E postgresql -E radar -E bufr -E restapi -E explorer" black = "black wetterdienst example tests" isort = "isort wetterdienst example tests" format = ["black", "isort"] -lint = "flakehell lint wetterdienst example tests" +lint = "flakeheaven lint wetterdienst example tests" docs = { shell = "cd docs && poetry run make html" } test = "pytest -vvv tests" # TODO: When parallel testing works on CI, use this: @@ -268,6 +278,7 @@ coverage-parallel = "pytest --cov=wetterdienst --numprocesses=4 -m 'not (explore export_requirements = "poetry export --without-hashes --dev --output requirements.txt" export_licenses = "pip-licenses --from=mixed --format=plain-vertical --with-authors --with-urls --with-license-file --no-license-path --ignore-packages wetterdienst --output-file=THIRD_PARTY_NOTICES" export = ["export_requirements", "export_licenses"] +update = ["poetry update"] [tool.pytest.ini_options] markers = [ diff --git a/tests/example/test_notebook_examples.py b/tests/example/test_notebook_examples.py index bb1f40f91..f1b1e32d5 100644 --- a/tests/example/test_notebook_examples.py +++ b/tests/example/test_notebook_examples.py @@ -10,10 +10,7 @@ @pytest.mark.slow -@pytest.mark.xfail( - reason="nbconvert stack has problems, see " - "https://github.com/jupyter/jupyter_client/issues/637" -) +@pytest.mark.xfail(reason="nbconvert stack has problems, see " "https://github.com/jupyter/jupyter_client/issues/637") def test_jupyter_example(): """ Test for climate_observations jupyter notebook """ fixture = NBRegressionFixture( diff --git a/tests/provider/dwd/forecast/test_api_data.py b/tests/provider/dwd/forecast/test_api_data.py index d4f65d5a8..c0afce132 100644 --- a/tests/provider/dwd/forecast/test_api_data.py +++ b/tests/provider/dwd/forecast/test_api_data.py @@ -12,9 +12,7 @@ def test_dwd_mosmix_l(): Test some details of a typical MOSMIX-L response. """ - request = DwdMosmixRequest( - parameter="large", mosmix_type=DwdMosmixType.LARGE, humanize=False - ).filter_by_station_id( + request = DwdMosmixRequest(parameter="large", mosmix_type=DwdMosmixType.LARGE, humanize=False).filter_by_station_id( station_id=["01001"], ) response = next(request.values.query()) diff --git a/tests/provider/dwd/observation/test_api_data.py b/tests/provider/dwd/observation/test_api_data.py index 2d40b1aeb..ba72b87f8 100644 --- a/tests/provider/dwd/observation/test_api_data.py +++ b/tests/provider/dwd/observation/test_api_data.py @@ -42,7 +42,7 @@ def test_dwd_observation_data_api(): assert request.parameter == [ ( - DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.PRECIPITATION_HEIGHT, # Noqa: E501, B950 + DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.PRECIPITATION_HEIGHT, DwdObservationDataset.CLIMATE_SUMMARY, ) ] @@ -118,9 +118,7 @@ def test_dwd_observation_data_parameter(): period=["recent", "historical"], ) - assert request.parameter == [ - (DwdObservationDataset.CLIMATE_SUMMARY, DwdObservationDataset.CLIMATE_SUMMARY) - ] + assert request.parameter == [(DwdObservationDataset.CLIMATE_SUMMARY, DwdObservationDataset.CLIMATE_SUMMARY)] def test_dwd_observation_data_parameter_dataset_pairs(): @@ -131,9 +129,7 @@ def test_dwd_observation_data_parameter_dataset_pairs(): period=["recent", "historical"], ) - assert request.parameter == [ - (DwdObservationDataset.CLIMATE_SUMMARY, DwdObservationDataset.CLIMATE_SUMMARY) - ] + assert request.parameter == [(DwdObservationDataset.CLIMATE_SUMMARY, DwdObservationDataset.CLIMATE_SUMMARY)] request = DwdObservationRequest( parameter=[("precipitation_height", "precipitation_more")], @@ -700,9 +696,7 @@ def test_dwd_observation_data_result_tidy_metric(): def test_dwd_observation_data_10_minutes_result_tidy(): """ Test for actual values (tidy) in metric units """ request = DwdObservationRequest( - parameter=[ - DwdObservationDatasetTree.MINUTE_10.TEMPERATURE_AIR.PRESSURE_AIR_SITE - ], + parameter=[DwdObservationDatasetTree.MINUTE_10.TEMPERATURE_AIR.PRESSURE_AIR_SITE], resolution=DwdObservationResolution.MINUTE_10, start_date="1999-12-31 22:00", end_date="1999-12-31 23:00", @@ -743,9 +737,7 @@ def test_dwd_observation_data_10_minutes_result_tidy(): ], errors="coerce", ).astype(float), - "quality": pd.to_numeric( - [1, 1, 1, 1, 1, 1, pd.NA], errors="coerce" - ).astype(float), + "quality": pd.to_numeric([1, 1, 1, 1, 1, 1, pd.NA], errors="coerce").astype(float), }, ), # Needed since pandas 1.2? @@ -862,9 +854,7 @@ def test_tidy_up_data(): -7.9, -11.4, ], - "quality": pd.Series( - [10, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], dtype=float - ), + "quality": pd.Series([10, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], dtype=float), } ) diff --git a/tests/provider/dwd/observation/test_api_stations_geo.py b/tests/provider/dwd/observation/test_api_stations_geo.py index 2417a894f..191b2a665 100644 --- a/tests/provider/dwd/observation/test_api_stations_geo.py +++ b/tests/provider/dwd/observation/test_api_stations_geo.py @@ -162,9 +162,7 @@ def test_dwd_observation_stations_bbox(): datetime(2020, 1, 1), datetime(2020, 1, 20), ) - nearby_station = request.filter_by_bbox( - left=8.7862, bottom=49.9195, right=8.993, top=50.0899 - ) + nearby_station = request.filter_by_bbox(left=8.7862, bottom=49.9195, right=8.993, top=50.0899) nearby_station = nearby_station.df.drop("to_date", axis="columns") pd.testing.assert_frame_equal( diff --git a/tests/provider/dwd/observation/test_io.py b/tests/provider/dwd/observation/test_io.py index 54d8ef658..f0f47af86 100644 --- a/tests/provider/dwd/observation/test_io.py +++ b/tests/provider/dwd/observation/test_io.py @@ -77,9 +77,7 @@ def test_filter_by_date(): def test_filter_by_date_interval(): - df = filter_by_date_and_resolution( - df_data, "2019-12-27/2019-12-29", Resolution.HOURLY - ) + df = filter_by_date_and_resolution(df_data, "2019-12-27/2019-12-29", Resolution.HOURLY) assert not df.empty df = filter_by_date_and_resolution(df_data, "2020/2022", Resolution.HOURLY) @@ -124,14 +122,10 @@ def test_filter_by_date_annual(): } ) - df = filter_by_date_and_resolution( - df, date="2019-05/2019-09", resolution=Resolution.ANNUAL - ) + df = filter_by_date_and_resolution(df, date="2019-05/2019-09", resolution=Resolution.ANNUAL) assert not df.empty - df = filter_by_date_and_resolution( - df, date="2020/2022", resolution=Resolution.ANNUAL - ) + df = filter_by_date_and_resolution(df, date="2020/2022", resolution=Resolution.ANNUAL) assert df.empty df = filter_by_date_and_resolution(df, date="2020", resolution=Resolution.ANNUAL) @@ -178,10 +172,7 @@ def test_format_csv(): output = ExportMixin(df=df_data).to_csv().strip() assert "station_id,dataset,parameter,date,value,quality" in output - assert ( - "01048,climate_summary,temperature_air_max_200,2019-12-28T00-00-00,1.3," - in output - ) + assert "01048,climate_summary,temperature_air_max_200,2019-12-28T00-00-00,1.3," in output def test_request(): @@ -291,11 +282,7 @@ def test_export_spreadsheet(tmpdir_factory): (1.5,), ] - last_record = list( - worksheet.iter_cols( - min_row=worksheet.max_row, max_row=worksheet.max_row, values_only=True - ) - ) + last_record = list(worksheet.iter_cols(min_row=worksheet.max_row, max_row=worksheet.max_row, values_only=True)) assert last_record == [ ("01048",), ("climate_summary",), @@ -377,13 +364,9 @@ def test_export_parquet(tmpdir_factory): # Validate content. data = table.to_pydict() - assert data["date"][0] == datetime.datetime( - 2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc - ) + assert data["date"][0] == datetime.datetime(2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) assert data["temperature_air_min_005"][0] == 1.5 - assert data["date"][-1] == datetime.datetime( - 2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc - ) + assert data["date"][-1] == datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) assert data["temperature_air_min_005"][-1] == -4.6 os.unlink(filename) @@ -447,13 +430,9 @@ def test_export_zarr(tmpdir_factory): # Validate content. data = group - assert data["date"][0] == np.datetime64( - datetime.datetime(2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) - ) + assert data["date"][0] == np.datetime64(datetime.datetime(2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)) assert data["temperature_air_min_005"][0] == 1.5 - assert data["date"][-1] == np.datetime64( - datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) - ) + assert data["date"][-1] == np.datetime64(datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)) assert data["temperature_air_min_005"][-1] == -4.6 shutil.rmtree(filename) @@ -515,13 +494,9 @@ def test_export_feather(tmpdir_factory): # Validate content. data = table.to_pydict() - assert data["date"][0] == datetime.datetime( - 2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc - ) + assert data["date"][0] == datetime.datetime(2019, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) assert data["temperature_air_min_005"][0] == 1.5 - assert data["date"][-1] == datetime.datetime( - 2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc - ) + assert data["date"][-1] == datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) assert data["temperature_air_min_005"][-1] == -4.6 os.unlink(filename) @@ -639,9 +614,7 @@ def test_export_duckdb(): ).filter_by_station_id(station_id=[1048]) mock_connection = mock.MagicMock() - with mock.patch( - "duckdb.connect", side_effect=[mock_connection], create=True - ) as mock_connect: + with mock.patch("duckdb.connect", side_effect=[mock_connection], create=True) as mock_connect: df = request.values.all().df ExportMixin(df=df).to_target("duckdb:///test.duckdb?table=testdrive") @@ -793,13 +766,9 @@ def test_export_influxdb2_tabular(): ): df = request.values.all().df - ExportMixin(df=df).to_target( - "influxdb2://orga:token@localhost/?database=dwd&table=weather" - ) + ExportMixin(df=df).to_target("influxdb2://orga:token@localhost/?database=dwd&table=weather") - mock_connect.assert_called_once_with( - url="http://localhost:8086", org="orga", token="token" - ) + mock_connect.assert_called_once_with(url="http://localhost:8086", org="orga", token="token") @surrogate("influxdb_client.InfluxDBClient") @@ -828,10 +797,6 @@ def test_export_influxdb2_tidy(): ): df = request.values.all().df - ExportMixin(df=df).to_target( - "influxdb2://orga:token@localhost/?database=dwd&table=weather" - ) + ExportMixin(df=df).to_target("influxdb2://orga:token@localhost/?database=dwd&table=weather") - mock_connect.assert_called_once_with( - url="http://localhost:8086", org="orga", token="token" - ) + mock_connect.assert_called_once_with(url="http://localhost:8086", org="orga", token="token") diff --git a/tests/provider/dwd/observation/test_metaindex.py b/tests/provider/dwd/observation/test_metaindex.py index b3d8d05a7..5533fa26f 100644 --- a/tests/provider/dwd/observation/test_metaindex.py +++ b/tests/provider/dwd/observation/test_metaindex.py @@ -46,9 +46,7 @@ def test_meta_index_1mph_creation(): Period.HISTORICAL, ) - assert meta_index_1mph.loc[ - meta_index_1mph[Columns.STATION_ID.value] == "00003", : - ].values.tolist() == [ + assert meta_index_1mph.loc[meta_index_1mph[Columns.STATION_ID.value] == "00003", :].values.tolist() == [ [ "00003", "18910101", diff --git a/tests/provider/dwd/observation/util/__init__.py b/tests/provider/dwd/observation/util/__init__.py index 51424f995..91805216a 100644 --- a/tests/provider/dwd/observation/util/__init__.py +++ b/tests/provider/dwd/observation/util/__init__.py @@ -3,12 +3,10 @@ # Distributed under the MIT License. See LICENSE for more info. station_reference_pattern_unsorted = ( - "(asb,)?(boo,)?ros,hnr,umd,pro,ess,fld,drs," - "neu,(nhb,)?oft,eis,(tur,)?(isn,)?fbg,mem" + "(asb,)?(boo,)?ros,hnr,umd,pro,ess,fld,drs," "neu,(nhb,)?oft,eis,(tur,)?(isn,)?fbg,mem" ) station_reference_pattern_sorted = ( - "(asb,)?(boo,)?drs,eis,ess,fbg,fld,hnr,(isn,)?" - "mem,neu,(nhb,)?oft,pro,ros,(tur,)?umd" + "(asb,)?(boo,)?drs,eis,ess,fbg,fld,hnr,(isn,)?" "mem,neu,(nhb,)?oft,pro,ros,(tur,)?umd" ) station_reference_pattern_de = ( "(deasb,)?(deboo,)?dedrs,deeis,deess,(defbg,)?defld,dehnr," diff --git a/tests/provider/dwd/radar/__init__.py b/tests/provider/dwd/radar/__init__.py index da230756a..b6831f854 100644 --- a/tests/provider/dwd/radar/__init__.py +++ b/tests/provider/dwd/radar/__init__.py @@ -3,12 +3,10 @@ # Distributed under the MIT License. See LICENSE for more info. station_reference_pattern_unsorted = ( - "(asb,)?(boo,)?(ros,)?(hnr,)?umd,pro,ess,fld,(drs,)?" - "(neu,)?(nhb,)?(oft,)?eis,(tur,)?(isn,)?(fbg,)?(mem)?" + "(asb,)?(boo,)?(ros,)?(hnr,)?umd,pro,ess,fld,(drs,)?" "(neu,)?(nhb,)?(oft,)?eis,(tur,)?(isn,)?(fbg,)?(mem)?" ) station_reference_pattern_sorted = ( - "(asb,)?(boo,)?(drs,)?eis,ess,(fbg,)?fld,(hnr,)?(isn,)?" - "(mem,)?(neu,)?(nhb,)?oft,pro,(ros,)?(tur,)?umd" + "(asb,)?(boo,)?(drs,)?eis,ess,(fbg,)?fld,(hnr,)?(isn,)?" "(mem,)?(neu,)?(nhb,)?oft,pro,(ros,)?(tur,)?umd" ) station_reference_pattern_de = ( "(deasb,)?(deboo,)?(dedrs,)?deeis,deess,(defbg,)?defld,(dehnr,)?" diff --git a/tests/provider/dwd/radar/test_api_current.py b/tests/provider/dwd/radar/test_api_current.py index 2fe627536..0317f8813 100644 --- a/tests/provider/dwd/radar/test_api_current.py +++ b/tests/provider/dwd/radar/test_api_current.py @@ -43,7 +43,7 @@ def test_radar_request_site_current_sweep_pcp_v_hdf5(): assert payload.startswith(b"\x89HDF\r\n") # Verify more details. - # wddump ras07-stqual-pcpng01_sweeph5onem_vradh_00-2020093000403400-boo-10132-hd5 # noqa:E501,B950 + # wddump ras07-stqual-pcpng01_sweeph5onem_vradh_00-2020093000403400-boo-10132-hd5 hdf = h5py.File(buffer, "r") @@ -86,7 +86,7 @@ def test_radar_request_site_current_sweep_vol_v_hdf5_full(): assert payload.startswith(b"\x89HDF\r\n") # Verify more details. - # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 + # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 hdf = h5py.File(buffer, "r") diff --git a/tests/provider/dwd/radar/test_api_historic.py b/tests/provider/dwd/radar/test_api_historic.py index b121d37e9..1fb189c95 100644 --- a/tests/provider/dwd/radar/test_api_historic.py +++ b/tests/provider/dwd/radar/test_api_historic.py @@ -47,9 +47,7 @@ def test_radar_request_radolan_cdc_hourly_alignment_1(): start_date="2019-08-08 00:53:53", ) - assert request.start_date == datetime( - year=2019, month=8, day=8, hour=0, minute=50, second=0 - ) + assert request.start_date == datetime(year=2019, month=8, day=8, hour=0, minute=50, second=0) def test_radar_request_radolan_cdc_hourly_alignment_2(): @@ -68,9 +66,7 @@ def test_radar_request_radolan_cdc_hourly_alignment_2(): start_date="2019-08-08 00:42:42", ) - assert request.start_date == datetime( - year=2019, month=8, day=7, hour=23, minute=50, second=0 - ) + assert request.start_date == datetime(year=2019, month=8, day=7, hour=23, minute=50, second=0) @pytest.mark.remote @@ -173,7 +169,7 @@ def test_radar_request_composite_historic_fx_yesterday(): date_time = request.start_date.strftime("%d%H%M") month_year = request.start_date.strftime("%m%y") header = ( - f"FX{date_time}10000{month_year}BY.......VS 3SW 2.12.0PR E-01INT 5GP 900x 900VV 000MF 00000002MS " # noqa:E501,B950 + f"FX{date_time}10000{month_year}BY.......VS 3SW 2.12.0PR E-01INT 5GP 900x 900VV 000MF 00000002MS " f"..<{station_reference_pattern_unsorted}>" ) @@ -205,9 +201,7 @@ def test_radar_request_composite_historic_fx_timerange(): # Verify all timestamps are properly propagated from the tarfile. assert all( - request.start_date == result.timestamp - or request.start_date + timedelta(minutes=5) - for result in results + request.start_date == result.timestamp or request.start_date + timedelta(minutes=5) for result in results ) @@ -269,10 +263,7 @@ def test_radar_request_composite_historic_radolan_rw_yesterday(): # radar locations can change over time -> check if at least 10 radar locations # were found and at least 5 of them match with the provided one assert len(requested_attrs["radarlocations"]) >= 10 - assert ( - len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) - >= 5 - ) + assert len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) >= 5 skip_attrs = ["radarid", "datasize", "maxrange", "radarlocations", "radolanversion"] for attr in skip_attrs: @@ -308,12 +299,8 @@ def test_radar_request_composite_historic_radolan_rw_timerange(): requested_header = wrl.io.read_radolan_header(buffer) requested_attrs = wrl.io.parse_dwd_composite_header(requested_header) - assert request.start_date.strftime("m%y") == requested_attrs["datetime"].strftime( - "m%y" - ) - assert request.start_date.strftime("%d%H%M") == requested_attrs[ - "datetime" - ].strftime("%d%H%M") + assert request.start_date.strftime("m%y") == requested_attrs["datetime"].strftime("m%y") + assert request.start_date.strftime("%d%H%M") == requested_attrs["datetime"].strftime("%d%H%M") attrs = { "producttype": "RW", @@ -346,10 +333,7 @@ def test_radar_request_composite_historic_radolan_rw_timerange(): # radar locations can change over time -> check if at least 10 radar locations # were found and at least 5 of them match with the provided one assert len(requested_attrs["radarlocations"]) >= 10 - assert ( - len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) - >= 5 - ) + assert len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) >= 5 skip_attrs = [ "datetime", @@ -394,12 +378,8 @@ def test_radar_request_site_historic_dx_yesterday(): requested_attrs = wrl.io.radolan.parse_dx_header(requested_header) timestamp_aligned = round_minutes(timestamp, 5) - assert timestamp_aligned.strftime("%m%y") == requested_attrs["datetime"].strftime( - "%m%y" - ) - assert timestamp_aligned.strftime("%d%H%M") == requested_attrs["datetime"].strftime( - "%d%H%M" - ) + assert timestamp_aligned.strftime("%m%y") == requested_attrs["datetime"].strftime("%m%y") + assert timestamp_aligned.strftime("%d%H%M") == requested_attrs["datetime"].strftime("%d%H%M") attrs = { "producttype": "DX", @@ -450,12 +430,8 @@ def test_radar_request_site_historic_dx_timerange(): requested_attrs = wrl.io.radolan.parse_dx_header(requested_header) timestamp_aligned = round_minutes(timestamp, 5) - assert timestamp_aligned.strftime("%m%y") == requested_attrs["datetime"].strftime( - "%m%y" - ) - assert timestamp_aligned.strftime("%d%H%M") == requested_attrs["datetime"].strftime( - "%d%H%M" - ) + assert timestamp_aligned.strftime("%m%y") == requested_attrs["datetime"].strftime("%m%y") + assert timestamp_aligned.strftime("%d%H%M") == requested_attrs["datetime"].strftime("%d%H%M") attrs = { "producttype": "DX", @@ -506,7 +482,7 @@ def test_radar_request_site_historic_pe_binary_yesterday(): date_time = request.start_date.strftime("%d%H") month_year = request.start_date.strftime("%m%y") header = ( - f"PE{date_time}..10132{month_year}BY ....?VS 1LV12 1.0 2.0 3.0 4.0 5.0 " # noqa:E501,B950 + f"PE{date_time}..10132{month_year}BY ....?VS 1LV12 1.0 2.0 3.0 4.0 5.0 " f"6.0 7.0 8.0 9.0 10.0 11.0 12.0CO0CD0CS0ET 5.0FL....MS" ) @@ -845,14 +821,8 @@ def test_radar_request_site_historic_sweep_pcp_v_hdf5_yesterday(): assert hdf["/dataset1/data1/data"].shape == (360, 600) timestamp = round_minutes(request.start_date, 5) - assert hdf["/what"].attrs.get("date") == bytes( - timestamp.strftime("%Y%m%d"), encoding="ascii" - ) - assert ( - hdf["/what"] - .attrs.get("time") - .startswith(bytes(timestamp.strftime("%H%M"), encoding="ascii")) - ) + assert hdf["/what"].attrs.get("date") == bytes(timestamp.strftime("%Y%m%d"), encoding="ascii") + assert hdf["/what"].attrs.get("time").startswith(bytes(timestamp.strftime("%H%M"), encoding="ascii")) @pytest.mark.remote @@ -916,7 +886,7 @@ def test_radar_request_site_historic_sweep_vol_v_hdf5_yesterday(): assert payload.startswith(b"\x89HDF\r\n") # Verify more details. - # h5dump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 + # h5dump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 hdf = h5py.File(buffer, "r") @@ -930,14 +900,8 @@ def test_radar_request_site_historic_sweep_vol_v_hdf5_yesterday(): assert hdf["/dataset1/data1/data"].shape in ((360, 180), (360, 720), (361, 720)) timestamp = round_minutes(request.start_date, 5) - assert hdf["/what"].attrs.get("date") == bytes( - timestamp.strftime("%Y%m%d"), encoding="ascii" - ) - assert ( - hdf["/what"] - .attrs.get("time") - .startswith(bytes(timestamp.strftime("%H%M"), encoding="ascii")) - ) + assert hdf["/what"].attrs.get("date") == bytes(timestamp.strftime("%Y%m%d"), encoding="ascii") + assert hdf["/what"].attrs.get("time").startswith(bytes(timestamp.strftime("%H%M"), encoding="ascii")) # Verify that the second file is the second scan / elevation level. buffer = results[1].data @@ -946,14 +910,8 @@ def test_radar_request_site_historic_sweep_vol_v_hdf5_yesterday(): assert hdf["/dataset1/how"].attrs.get("scan_index") == 2 timestamp = round_minutes(request.start_date, 5) - assert hdf["/what"].attrs.get("date") == bytes( - timestamp.strftime("%Y%m%d"), encoding="ascii" - ) - assert ( - hdf["/what"] - .attrs.get("time") - .startswith(bytes(timestamp.strftime("%H%M"), encoding="ascii")) - ) + assert hdf["/what"].attrs.get("date") == bytes(timestamp.strftime("%Y%m%d"), encoding="ascii") + assert hdf["/what"].attrs.get("time").startswith(bytes(timestamp.strftime("%H%M"), encoding="ascii")) @pytest.mark.remote @@ -1050,10 +1008,7 @@ def test_radar_request_radvor_re_yesterday(): # radar locations can change over time -> check if at least 10 radar locations # were found and at least 5 of them match with the provided one assert len(requested_attrs["radarlocations"]) >= 10 - assert ( - len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) - >= 5 - ) + assert len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) >= 5 skip_attrs = [ "radarid", @@ -1164,10 +1119,7 @@ def test_radar_request_radvor_rq_yesterday(): # radar locations can change over time -> check if at least 10 radar locations # were found and at least 5 of them match with the provided one assert len(requested_attrs["radarlocations"]) >= 10 - assert ( - len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) - >= 5 - ) + assert len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) >= 5 skip_attrs = [ "datasize", diff --git a/tests/provider/dwd/radar/test_api_latest.py b/tests/provider/dwd/radar/test_api_latest.py index 2ef1d4643..bcf915ba0 100644 --- a/tests/provider/dwd/radar/test_api_latest.py +++ b/tests/provider/dwd/radar/test_api_latest.py @@ -35,8 +35,8 @@ def test_radar_request_composite_latest_rx_reflectivity(): month_year = datetime.utcnow().strftime("%m%y") header = ( - f"RX......10000{month_year}BY 8101..VS 3SW ......PR E\\+00INT 5GP 900x 900MS " # noqa:E501,B950 - f"..<{station_reference_pattern_unsorted}>" # noqa:E501,B950 + f"RX......10000{month_year}BY 8101..VS 3SW ......PR E\\+00INT 5GP 900x 900MS " + f"..<{station_reference_pattern_unsorted}>" ) assert re.match(bytes(header, encoding="ascii"), payload[:160]) @@ -64,9 +64,7 @@ def test_radar_request_composite_latest_rw_reflectivity(): requested_attrs = wrl.io.parse_dwd_composite_header(requested_header) # Verify data. - assert datetime.utcnow().strftime("%m%y") == requested_attrs["datetime"].strftime( - "%m%y" - ) + assert datetime.utcnow().strftime("%m%y") == requested_attrs["datetime"].strftime("%m%y") attrs = { "producttype": "RW", @@ -99,10 +97,7 @@ def test_radar_request_composite_latest_rw_reflectivity(): # radar locations can change over time -> check if at least 10 radar locations # were found and at least 5 of them match with the provided one assert len(requested_attrs["radarlocations"]) >= 10 - assert ( - len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) - >= 5 - ) + assert len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) >= 5 skip_attrs = [ "radarid", @@ -138,9 +133,7 @@ def test_radar_request_site_latest_dx_reflectivity(): # Verify data. timestamp_aligned = round_minutes(datetime.utcnow(), 5) - assert timestamp_aligned.strftime("%m%y") == requested_attrs["datetime"].strftime( - "%m%y" - ) + assert timestamp_aligned.strftime("%m%y") == requested_attrs["datetime"].strftime("%m%y") attrs = { "producttype": "DX", diff --git a/tests/provider/dwd/radar/test_api_most_recent.py b/tests/provider/dwd/radar/test_api_most_recent.py index eddd1f18e..47116753b 100644 --- a/tests/provider/dwd/radar/test_api_most_recent.py +++ b/tests/provider/dwd/radar/test_api_most_recent.py @@ -99,7 +99,7 @@ def test_radar_request_site_most_recent_sweep_vol_v_hdf5(): assert payload.startswith(b"\x89HDF\r\n") # Verify more details. - # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 + # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 hdf = h5py.File(buffer, "r") @@ -175,10 +175,7 @@ def test_radar_request_radolan_cdc_most_recent(): # radar locations can change over time -> check if at least 10 radar locations # were found and at least 5 of them match with the provided one assert len(requested_attrs["radarlocations"]) >= 10 - assert ( - len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) - >= 5 - ) + assert len(list(set(requested_attrs["radarlocations"]) & set(attrs["radarlocations"]))) >= 5 skip_attrs = [ "radolanversion", diff --git a/tests/provider/dwd/radar/test_api_recent.py b/tests/provider/dwd/radar/test_api_recent.py index 77eb63fb7..528619e7e 100644 --- a/tests/provider/dwd/radar/test_api_recent.py +++ b/tests/provider/dwd/radar/test_api_recent.py @@ -89,7 +89,7 @@ def test_radar_request_site_recent_sweep_vol_v_hdf5(): assert payload.startswith(b"\x89HDF\r\n") # Verify more details. - # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 # noqa:E501,B950 + # wddump ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092917055800-boo-10132-hd5 hdf = h5py.File(buffer, "r") diff --git a/tests/provider/dwd/radar/test_index.py b/tests/provider/dwd/radar/test_index.py index 0d16cadc2..dda723672 100644 --- a/tests/provider/dwd/radar/test_index.py +++ b/tests/provider/dwd/radar/test_index.py @@ -24,9 +24,7 @@ def test_radar_fileindex_composite_pg_reflectivity_bin(): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match("*/weather/radar/composit/pg/*---bin") for url in urls - ) + assert all(PurePath(url).match("*/weather/radar/composit/pg/*---bin") for url in urls) def test_radar_fileindex_composite_pg_reflectivity_bufr(): @@ -37,9 +35,7 @@ def test_radar_fileindex_composite_pg_reflectivity_bufr(): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match("*/weather/radar/composit/pg/*---bufr") for url in urls - ) + assert all(PurePath(url).match("*/weather/radar/composit/pg/*---bufr") for url in urls) @pytest.mark.xfail(reason="Out of service", strict=True) @@ -50,9 +46,7 @@ def test_radar_fileindex_composite_rx_reflectivity_bin(): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match("*/weather/radar/composit/rx/*---bin") for url in urls - ) + assert all(PurePath(url).match("*/weather/radar/composit/rx/*---bin") for url in urls) @pytest.mark.parametrize( @@ -70,10 +64,7 @@ def test_radar_fileindex_radolan_reflectivity_bin(parameter): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match(f"*/weather/radar/radolan/{parameter.value}/*---bin") - for url in urls - ) + assert all(PurePath(url).match(f"*/weather/radar/radolan/{parameter.value}/*---bin") for url in urls) def test_radar_fileindex_sites_px_reflectivity_bin(): @@ -85,9 +76,7 @@ def test_radar_fileindex_sites_px_reflectivity_bin(): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match("*/weather/radar/sites/px/boo/*---bin") for url in urls - ) + assert all(PurePath(url).match("*/weather/radar/sites/px/boo/*---bin") for url in urls) def test_radar_fileindex_sites_px_reflectivity_bufr(): @@ -99,9 +88,7 @@ def test_radar_fileindex_sites_px_reflectivity_bufr(): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match("*/weather/radar/sites/px/boo/*---buf") for url in urls - ) + assert all(PurePath(url).match("*/weather/radar/sites/px/boo/*---buf") for url in urls) def test_radar_fileindex_sites_px250_reflectivity_bufr(): @@ -124,10 +111,7 @@ def test_radar_fileindex_sites_sweep_bufr(): ) urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - PurePath(url).match("*/weather/radar/sites/sweep_vol_v/boo/*--buf.bz2") - for url in urls - ) + assert all(PurePath(url).match("*/weather/radar/sites/sweep_vol_v/boo/*--buf.bz2") for url in urls) def test_radar_fileindex_sites_sweep_vol_v_hdf5_simple(): @@ -141,9 +125,7 @@ def test_radar_fileindex_sites_sweep_vol_v_hdf5_simple(): urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - "/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_simple" in url for url in urls - ) + assert all("/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_simple" in url for url in urls) def test_radar_fileindex_sites_sweep_vol_v_hdf5_polarimetric(): @@ -157,10 +139,7 @@ def test_radar_fileindex_sites_sweep_vol_v_hdf5_polarimetric(): urls = file_index[DwdColumns.FILENAME.value].tolist() - assert all( - "/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_polarimetric" in url - for url in urls - ) + assert all("/weather/radar/sites/sweep_vol_v/boo/hdf5/filter_polarimetric" in url for url in urls) def test_radar_fileindex_radolan_cdc_daily_recent(): @@ -173,9 +152,7 @@ def test_radar_fileindex_radolan_cdc_daily_recent(): urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( - PurePath(url).match( - "*/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/*---bin.gz" - ) + PurePath(url).match("*/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/*---bin.gz") for url in urls if not url.endswith(".pdf") ) @@ -191,9 +168,7 @@ def test_radar_fileindex_radolan_cdc_daily_historical(): urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( - PurePath(url).match( - "*/climate_environment/CDC/grids_germany/daily/radolan/historical/bin/*/SF*.tar.gz" - ) + PurePath(url).match("*/climate_environment/CDC/grids_germany/daily/radolan/historical/bin/*/SF*.tar.gz") for url in urls if not url.endswith(".pdf") ) @@ -209,9 +184,7 @@ def test_radar_fileindex_radolan_cdc_hourly_recent(): urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( - PurePath(url).match( - "*/climate_environment/CDC/grids_germany/hourly/radolan/recent/bin/*---bin.gz" - ) + PurePath(url).match("*/climate_environment/CDC/grids_germany/hourly/radolan/recent/bin/*---bin.gz") for url in urls if not url.endswith(".pdf") ) @@ -227,9 +200,7 @@ def test_radar_fileindex_radolan_cdc_hourly_historical(): urls = file_index[DwdColumns.FILENAME.value].tolist() assert all( - PurePath(url).match( - "*/climate_environment/CDC/grids_germany/hourly/radolan/historical/bin/*/RW*.tar.gz" - ) + PurePath(url).match("*/climate_environment/CDC/grids_germany/hourly/radolan/historical/bin/*/RW*.tar.gz") for url in urls if not url.endswith(".pdf") ) diff --git a/tests/provider/dwd/radar/test_util.py b/tests/provider/dwd/radar/test_util.py index 5bad74892..eacfd2cf3 100644 --- a/tests/provider/dwd/radar/test_util.py +++ b/tests/provider/dwd/radar/test_util.py @@ -12,14 +12,10 @@ def test_radar_get_date_from_filename(): date = get_date_from_filename("sweep_pcp_v_0-20200926143033_10132--buf.bz2") assert date == datetime.datetime(2020, 9, 26, 14, 30) - date = get_date_from_filename( - "ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092614305700-boo-10132-hd5" - ) + date = get_date_from_filename("ras07-stqual-vol5minng01_sweeph5onem_vradh_00-2020092614305700-boo-10132-hd5") assert date == datetime.datetime(2020, 9, 26, 14, 30) - date = get_date_from_filename( - "ras07-vol5minng01_sweeph5onem_vradh_00-2020092614305700-boo-10132-hd5" - ) + date = get_date_from_filename("ras07-vol5minng01_sweeph5onem_vradh_00-2020092614305700-boo-10132-hd5") assert date == datetime.datetime(2020, 9, 26, 14, 30) date = get_date_from_filename("rab02-tt_10132-20200926161533-boo---buf") diff --git a/tests/provider/dwd/test_date.py b/tests/provider/dwd/test_date.py index c277fdbac..86dfff485 100644 --- a/tests/provider/dwd/test_date.py +++ b/tests/provider/dwd/test_date.py @@ -10,9 +10,7 @@ def test_mktimerange_annual(): - assert mktimerange( - Resolution.ANNUAL, dateutil.parser.isoparse("2019").replace(tzinfo=pytz.UTC) - ) == ( + assert mktimerange(Resolution.ANNUAL, dateutil.parser.isoparse("2019").replace(tzinfo=pytz.UTC)) == ( dateutil.parser.isoparse("2019-01-01 00:00:00Z"), dateutil.parser.isoparse("2019-12-31 00:00:00Z"), ) @@ -27,9 +25,7 @@ def test_mktimerange_annual(): def test_mktimerange_monthly(): - assert mktimerange( - Resolution.MONTHLY, dateutil.parser.isoparse("2020-05").replace(tzinfo=pytz.UTC) - ) == ( + assert mktimerange(Resolution.MONTHLY, dateutil.parser.isoparse("2020-05").replace(tzinfo=pytz.UTC)) == ( dateutil.parser.isoparse("2020-05-01 00:00:00Z"), dateutil.parser.isoparse("2020-05-31 00:00:00Z"), ) diff --git a/tests/provider/dwd/test_index.py b/tests/provider/dwd/test_index.py index 9f3c5a700..cf978f4fe 100644 --- a/tests/provider/dwd/test_index.py +++ b/tests/provider/dwd/test_index.py @@ -29,8 +29,7 @@ def test_build_index_path(): @pytest.mark.remote def test_list_files_of_climate_observations(): files_server = list_remote_files_fsspec( - "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/" - "annual/kl/recent", + "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/" "annual/kl/recent", recursive=False, ) diff --git a/tests/provider/dwd/test_util.py b/tests/provider/dwd/test_util.py index ae6c3c238..cde97dfda 100644 --- a/tests/provider/dwd/test_util.py +++ b/tests/provider/dwd/test_util.py @@ -26,10 +26,7 @@ def test_parse_enumeration_from_template(): parse_enumeration_from_template("CLIMATE_SUMMARY", DwdObservationDataset) == DwdObservationDataset.CLIMATE_SUMMARY ) - assert ( - parse_enumeration_from_template("kl", DwdObservationDataset) - == DwdObservationDataset.CLIMATE_SUMMARY - ) + assert parse_enumeration_from_template("kl", DwdObservationDataset) == DwdObservationDataset.CLIMATE_SUMMARY with pytest.raises(InvalidEnumeration): parse_enumeration_from_template("climate", DwdObservationDataset) diff --git a/tests/provider/eccc/test_api.py b/tests/provider/eccc/test_api.py index cf849f147..1ca8e1afb 100644 --- a/tests/provider/eccc/test_api.py +++ b/tests/provider/eccc/test_api.py @@ -112,6 +112,4 @@ def test_eccc_api_values(): } ) - assert_frame_equal( - values.reset_index(drop=True), expected_df, check_categorical=False - ) + assert_frame_equal(values.reset_index(drop=True), expected_df, check_categorical=False) diff --git a/tests/test_api.py b/tests/test_api.py index 17ec6483a..7bfd3939b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -60,8 +60,6 @@ def test_api(provider, kind, kwargs, si_units): values = next(request.values.query()).df # TODO: DWD Forecast has no quality - assert set(values.columns).issuperset( - {"station_id", "parameter", "date", "value", "quality"} - ) + assert set(values.columns).issuperset({"station_id", "parameter", "date", "value", "quality"}) assert not values.empty diff --git a/tests/ui/explorer/conftest.py b/tests/ui/explorer/conftest.py index 532921e0f..8cb61af28 100644 --- a/tests/ui/explorer/conftest.py +++ b/tests/ui/explorer/conftest.py @@ -38,9 +38,7 @@ def wait_for_element_by_id_clickable(self, element_id, timeout=None): EC.element_to_be_clickable, ((By.ID, element_id),), timeout, - "timeout {}s => waiting for element id {}".format( - timeout if timeout else self._wait_timeout, element_id - ), + "timeout {}s => waiting for element id {}".format(timeout if timeout else self._wait_timeout, element_id), ) diff --git a/tests/ui/test_cli.py b/tests/ui/test_cli.py index 5bb8a3d37..d561bce75 100644 --- a/tests/ui/test_cli.py +++ b/tests/ui/test_cli.py @@ -110,8 +110,7 @@ def invoke_wetterdienst_stations_empty(provider, kind, setting, fmt="json"): result = runner.invoke( cli, - f"stations --provider={provider} --kind={kind} " - f"{setting} --station=123456 --format={fmt}", + f"stations --provider={provider} --kind={kind} " f"{setting} --station=123456 --format={fmt}", ) return result @@ -122,8 +121,7 @@ def invoke_wetterdienst_stations_static(provider, kind, setting, station, fmt="j result = runner.invoke( cli, - f"stations --provider={provider} --kind={kind} " - f"{setting} --station={station} --format={fmt}", + f"stations --provider={provider} --kind={kind} " f"{setting} --station={station} --format={fmt}", ) return result @@ -134,8 +132,7 @@ def invoke_wetterdienst_stations_export(provider, kind, setting, station, target result = runner.invoke( cli, - f"stations --provider={provider} --kind={kind} " - f"{setting} --station={station} --target={target}", + f"stations --provider={provider} --kind={kind} " f"{setting} --station={station} --target={target}", ) return result @@ -159,8 +156,7 @@ def invoke_wetterdienst_values_static(provider, kind, setting, station, fmt="jso result = runner.invoke( cli, - f"values --provider={provider} --kind={kind} " - f"{setting} --station={station} --format={fmt}", + f"values --provider={provider} --kind={kind} " f"{setting} --station={station} --format={fmt}", ) return result @@ -171,22 +167,18 @@ def invoke_wetterdienst_values_export(provider, kind, setting, station, target): result = runner.invoke( cli, - f"values --provider={provider} --kind={kind} " - f"{setting} --station={station} --target={target}", + f"values --provider={provider} --kind={kind} " f"{setting} --station={station} --target={target}", ) return result -def invoke_wetterdienst_values_static_tidy( - provider, kind, setting, station, fmt="json" -): +def invoke_wetterdienst_values_static_tidy(provider, kind, setting, station, fmt="json"): runner = CliRunner() result = runner.invoke( cli, - f"values --provider={provider} --kind={kind} " - f"{setting} --station={station} --format={fmt} --tidy", + f"values --provider={provider} --kind={kind} " f"{setting} --station={station} --format={fmt} --tidy", ) return result @@ -209,9 +201,7 @@ def test_no_provider(): result = runner.invoke(cli, "stations --provider=abc --kind=abc") - assert ( - "Error: Invalid value for '--provider': invalid choice: abc." in result.output - ) + assert "Error: Invalid value for '--provider': invalid choice: abc." in result.output def test_no_kind(): @@ -232,10 +222,7 @@ def test_data_range(capsys): ) assert isinstance(result.exception, TypeError) - assert ( - "Combination of provider ECCC and kind OBSERVATION requires start and end date" - in str(result.exception) - ) + assert "Combination of provider ECCC and kind OBSERVATION requires start and end date" in str(result.exception) @pytest.mark.parametrize( @@ -254,14 +241,10 @@ def test_cli_stations_json(provider, kind, setting, station_id, station_name): assert station_name in station_names -@pytest.mark.parametrize( - "provider,kind,setting,station_id,station_name", SETTINGS_STATIONS -) +@pytest.mark.parametrize("provider,kind,setting,station_id,station_name", SETTINGS_STATIONS) def test_cli_stations_empty(provider, kind, setting, station_id, station_name, caplog): - result = invoke_wetterdienst_stations_empty( - provider=provider, kind=kind, setting=setting, fmt="json" - ) + result = invoke_wetterdienst_stations_empty(provider=provider, kind=kind, setting=setting, fmt="json") assert isinstance(result.exception, SystemExit) assert "ERROR" in caplog.text @@ -304,9 +287,7 @@ def test_cli_stations_csv(provider, kind, setting, station_id, station_name): "provider,kind,setting,station_id,station_name", SETTINGS_STATIONS, ) -def test_cli_stations_excel( - provider, kind, setting, station_id, station_name, tmpdir_factory -): +def test_cli_stations_excel(provider, kind, setting, station_id, station_name, tmpdir_factory): # filename = tmpdir_factory.mktemp("data").join("stations.xlsx") filename = "stations.xlsx" @@ -351,9 +332,7 @@ def test_cli_values_json(setting, expected_columns, capsys, caplog): assert set(first.keys()).issuperset(expected_columns) -@pytest.mark.parametrize( - "provider,kind,setting,station_id,station_name", SETTINGS_VALUES -) +@pytest.mark.parametrize("provider,kind,setting,station_id,station_name", SETTINGS_VALUES) def test_cli_values_json_tidy(provider, kind, setting, station_id, station_name): result = invoke_wetterdienst_values_static_tidy( @@ -388,10 +367,7 @@ def test_cli_values_geojson(provider, kind, setting, station_id, station_name, c provider=provider, kind=kind, setting=setting, station=station_id, fmt="geojson" ) - assert ( - "Error: Invalid value for '--format': invalid choice: " - "geojson. (choose from json, csv)" in result.output - ) + assert "Error: Invalid value for '--format': invalid choice: " "geojson. (choose from json, csv)" in result.output @pytest.mark.parametrize( @@ -410,9 +386,7 @@ def test_cli_values_csv(provider, kind, setting, station_id, station_name): "provider,kind,setting,station_id,station_name", SETTINGS_VALUES, ) -def test_cli_values_excel( - provider, kind, setting, station_id, station_name, tmpdir_factory -): +def test_cli_values_excel(provider, kind, setting, station_id, station_name, tmpdir_factory): # filename = tmpdir_factory.mktemp("data").join("values.xlsx") filename = "values.xlsx" @@ -442,10 +416,7 @@ def test_cli_values_format_unknown(provider, kind, setting, station_id, station_ provider=provider, kind=kind, setting=setting, station=station_id, fmt="foobar" ) - assert ( - "Error: Invalid value for '--format': " - "invalid choice: foobar. (choose from json, csv)" in result.output - ) + assert "Error: Invalid value for '--format': " "invalid choice: foobar. (choose from json, csv)" in result.output @pytest.mark.parametrize( @@ -454,9 +425,7 @@ def test_cli_values_format_unknown(provider, kind, setting, station_id, station_ ) def test_cli_stations_geospatial(provider, kind, setting, station_id, station_name): - result = invoke_wetterdienst_stations_geo( - provider=provider, kind=kind, setting=setting, fmt="json" - ) + result = invoke_wetterdienst_stations_geo(provider=provider, kind=kind, setting=setting, fmt="json") response = json.loads(result.output) @@ -471,9 +440,7 @@ def test_cli_stations_geospatial(provider, kind, setting, station_id, station_na ) def test_cli_values_geospatial(provider, kind, setting, station_id, station_name): - result = invoke_wetterdienst_values_geo( - provider=provider, kind=kind, setting=setting, fmt="json" - ) + result = invoke_wetterdienst_values_geo(provider=provider, kind=kind, setting=setting, fmt="json") response = json.loads(result.output) diff --git a/tests/ui/test_restapi.py b/tests/ui/test_restapi.py index 6dc00b015..aee047cd1 100644 --- a/tests/ui/test_restapi.py +++ b/tests/ui/test_restapi.py @@ -65,10 +65,7 @@ def test_data_range(capsys): }, ) - assert ( - "Combination of provider ECCC and kind OBSERVATION requires start and end date" - in response.text - ) + assert "Combination of provider ECCC and kind OBSERVATION requires start and end date" in response.text def test_dwd_stations_basic(): @@ -194,9 +191,7 @@ def test_dwd_values_no_parameter(): ) assert response.status_code == 400 - assert response.json() == { - "detail": "Query arguments 'parameter', 'resolution' and 'date' are required" - } + assert response.json() == {"detail": "Query arguments 'parameter', 'resolution' and 'date' are required"} def test_dwd_values_no_resolution(): @@ -213,9 +208,7 @@ def test_dwd_values_no_resolution(): ) assert response.status_code == 400 - assert response.json() == { - "detail": "Query arguments 'parameter', 'resolution' and 'date' are required" - } + assert response.json() == {"detail": "Query arguments 'parameter', 'resolution' and 'date' are required"} @pytest.mark.sql @@ -280,8 +273,7 @@ def test_dwd_values_sql_tidy(dicts_are_same): "parameter": "kl", "resolution": "daily", "date": "2019-12-01/2019-12-31", - "sql-values": "SELECT * FROM data " - "WHERE parameter='temperature_air_max_200' AND value < 1.5", + "sql-values": "SELECT * FROM data " "WHERE parameter='temperature_air_max_200' AND value < 1.5", "si-units": False, }, ) diff --git a/tests/util/test_geo.py b/tests/util/test_geo.py index 8970c0c16..70e51e207 100644 --- a/tests/util/test_geo.py +++ b/tests/util/test_geo.py @@ -8,9 +8,7 @@ def test_get_coordinates(): coordinates = Coordinates(np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8])) - np.testing.assert_equal( - coordinates.get_coordinates(), np.array([[1, 5], [2, 6], [3, 7], [4, 8]]) - ) + np.testing.assert_equal(coordinates.get_coordinates(), np.array([[1, 5], [2, 6], [3, 7], [4, 8]])) def test_get_coordinates_in_radians(): diff --git a/wetterdienst/__init__.py b/wetterdienst/__init__.py index bb0bef98d..c8c6f0da8 100644 --- a/wetterdienst/__init__.py +++ b/wetterdienst/__init__.py @@ -29,17 +29,13 @@ def info() -> None: """ Function that prints some basic information about the wetterdienst instance """ wd_info = { "version": __version__, - "authors": "Benjamin Gutzmann , " - "Andreas Motl ", + "authors": "Benjamin Gutzmann , " "Andreas Motl ", "documentation": "https://wetterdienst.readthedocs.io/", "repository": "https://github.com/earthobservations/wetterdienst", "cache_dir": cache_dir, } - text = ( - "Wetterdienst - Open weather data for humans\n" - "-------------------------------------------" - ) + text = "Wetterdienst - Open weather data for humans\n" "-------------------------------------------" for key, value in wd_info.items(): text += f"\n{key}:\t {value}" diff --git a/wetterdienst/api.py b/wetterdienst/api.py index c6844fd08..49ed3acd1 100644 --- a/wetterdienst/api.py +++ b/wetterdienst/api.py @@ -48,9 +48,7 @@ def __new__(cls, provider: Union[Provider, str], kind: Union[Kind, str]): raise KeyError except (InvalidEnumeration, KeyError): - raise ProviderError( - f"No API available for provider {provider} and kind {kind}" - ) + raise ProviderError(f"No API available for provider {provider} and kind {kind}") return api diff --git a/wetterdienst/core/process.py b/wetterdienst/core/process.py index f6a1289c4..3f8471966 100644 --- a/wetterdienst/core/process.py +++ b/wetterdienst/core/process.py @@ -15,9 +15,7 @@ from wetterdienst.util.datetime import mktimerange -def create_date_range( - date: str, resolution: Resolution -) -> Tuple[Optional[datetime], Optional[datetime]]: +def create_date_range(date: str, resolution: Resolution) -> Tuple[Optional[datetime], Optional[datetime]]: date_from, date_to = None, None if "/" in date: @@ -54,9 +52,7 @@ def create_date_range( return date_from, date_to -def filter_by_date_and_resolution( - df: pd.DataFrame, date: str, resolution: Resolution -) -> pd.DataFrame: +def filter_by_date_and_resolution(df: pd.DataFrame, date: str, resolution: Resolution) -> pd.DataFrame: """ Filter Pandas DataFrame by date or date interval. @@ -98,13 +94,9 @@ def filter_by_date_and_resolution( Resolution.MONTHLY, ): date_from, date_to = mktimerange(resolution, date_from, date_to) - expression = (date_from <= df[Columns.FROM_DATE.value]) & ( - df[Columns.TO_DATE.value] <= date_to - ) + expression = (date_from <= df[Columns.FROM_DATE.value]) & (df[Columns.TO_DATE.value] <= date_to) else: - expression = (date_from <= df[Columns.DATE.value]) & ( - df[Columns.DATE.value] <= date_to - ) + expression = (date_from <= df[Columns.DATE.value]) & (df[Columns.DATE.value] <= date_to) df = df[expression] # Filter by specific date. @@ -118,9 +110,7 @@ def filter_by_date_and_resolution( Resolution.MONTHLY, ): date_from, date_to = mktimerange(resolution, date) - expression = (date_from <= df[Columns.FROM_DATE.value]) & ( - df[Columns.TO_DATE.value] <= date_to - ) + expression = (date_from <= df[Columns.FROM_DATE.value]) & (df[Columns.TO_DATE.value] <= date_to) else: expression = date == df[Columns.DATE.value] df = df[expression] diff --git a/wetterdienst/core/scalar/export.py b/wetterdienst/core/scalar/export.py index 62b1a2f5e..533593610 100644 --- a/wetterdienst/core/scalar/export.py +++ b/wetterdienst/core/scalar/export.py @@ -45,9 +45,7 @@ def to_dict(self) -> dict: return df.to_dict(orient="records") def to_json(self, indent: int = 4): - output = self.df.to_json( - orient="records", date_format="iso", indent=indent, force_ascii=False - ) + output = self.df.to_json(orient="records", date_format="iso", indent=indent, force_ascii=False) return output def to_csv(self): @@ -61,9 +59,7 @@ def to_geojson(self, indent: int = 4) -> str: Return: JSON string in GeoJSON FeatureCollection format. """ - return json.dumps( - self.to_ogc_feature_collection(), indent=indent, ensure_ascii=False - ) + return json.dumps(self.to_ogc_feature_collection(), indent=indent, ensure_ascii=False) def to_format(self, fmt: str, **kwargs) -> str: """ @@ -233,9 +229,7 @@ def to_target(self, target: str): dimensions = store.get_dimensions() variables = list(store.get_variables().keys()) - log.info( - f"Wrote Zarr file with dimensions={dimensions} and variables={variables}" - ) + log.info(f"Wrote Zarr file with dimensions={dimensions} and variables={variables}") log.info(f"Zarr Dataset Group info:\n{store.ds.info}") else: @@ -269,9 +263,7 @@ def to_target(self, target: str): connection = duckdb.connect(database=database, read_only=False) connection.register("origin", self.df) connection.execute(f"DROP TABLE IF EXISTS {tablename};") - connection.execute( - f"CREATE TABLE {tablename} AS SELECT * FROM origin;" # noqa:S608 - ) + connection.execute(f"CREATE TABLE {tablename} AS SELECT * FROM origin;") # noqa:S608 weather_table = connection.table(tablename) print(weather_table) # noqa: T001 @@ -338,9 +330,7 @@ def to_target(self, target: str): else: raise KeyError(f"Unknown protocol variant '{protocol}' for InfluxDB") - log.info( - f"Writing to InfluxDB version {version}. database={database}, table={tablename}" - ) + log.info(f"Writing to InfluxDB version {version}. database={database}, table={tablename}") # 1. Mungle the data frame. # Use the "date" column as appropriate timestamp index. @@ -381,9 +371,7 @@ def to_target(self, target: str): ssl = protocol.endswith("s") url = f"http{ssl and 's' or ''}://{connspec.url.hostname}:{connspec.url.port or 8086}" - client = InfluxDBClient( - url=url, org=connspec.url.username, token=connspec.url.password - ) + client = InfluxDBClient(url=url, org=connspec.url.username, token=connspec.url.password) write_api = client.write_api(write_options=SYNCHRONOUS) points = [] @@ -391,9 +379,7 @@ def to_target(self, target: str): for date, record in items.iterrows(): time = date.isoformat() - tags = { - tag: record.pop(tag) for tag in tag_columns if tag in record - } + tags = {tag: record.pop(tag) for tag in tag_columns if tag in record} fields = record.dropna().to_dict() if not fields: diff --git a/wetterdienst/core/scalar/request.py b/wetterdienst/core/scalar/request.py index 10cbcc633..8c4641f6e 100644 --- a/wetterdienst/core/scalar/request.py +++ b/wetterdienst/core/scalar/request.py @@ -107,9 +107,7 @@ def _has_datasets(self) -> bool: def _dataset_base(self) -> Optional[Enum]: """ Dataset base that is used to differ between different datasets """ if self._has_datasets: - raise NotImplementedError( - "implement _dataset_base enumeration that contains available datasets" - ) + raise NotImplementedError("implement _dataset_base enumeration that contains available datasets") return @property @@ -117,8 +115,7 @@ def _dataset_tree(self) -> Optional[object]: """ Detailed dataset tree with all parameters per dataset """ if self._has_datasets: raise NotImplementedError( - "implement _dataset_tree class that contains available datasets " - "and their parameters" + "implement _dataset_tree class that contains available datasets " "and their parameters" ) return None @@ -148,9 +145,7 @@ def _has_tidy_data(self) -> bool: def _parameter_to_dataset_mapping(self) -> dict: """ Mapping to go from a (flat) parameter to dataset """ if not self._unique_dataset: - raise NotImplementedError( - "for non unique datasets implement a mapping from parameter to dataset" - ) + raise NotImplementedError("for non unique datasets implement a mapping from parameter to dataset") return {} @property @@ -209,16 +204,12 @@ def _parse_period(self, period: Period) -> Optional[List[Period]]: else: return ( pd.Series(period) - .apply( - parse_enumeration_from_template, args=(self._period_base, Period) - ) + .apply(parse_enumeration_from_template, args=(self._period_base, Period)) .sort_values() .tolist() ) - def _parse_parameter( - self, parameter: List[Union[str, Enum]] - ) -> List[Tuple[Enum, Enum]]: + def _parse_parameter(self, parameter: List[Union[str, Enum]]) -> List[Tuple[Enum, Enum]]: """ Method to parse parameters, either from string or enum. Case independent for strings. @@ -264,9 +255,7 @@ def _parse_parameter( try: # First parse parameter - parameter_ = parse_enumeration_from_template( - parameter, self._parameter_base[self._dataset_accessor] - ) + parameter_ = parse_enumeration_from_template(parameter, self._parameter_base[self._dataset_accessor]) except (InvalidEnumeration, TypeError): pass else: @@ -277,15 +266,11 @@ def _parse_parameter( elif not dataset_: # If there's multiple datasets the mapping defines which one # is taken for the given parameter - dataset_ = self._parameter_to_dataset_mapping[self.resolution][ - parameter_ - ] + dataset_ = self._parameter_to_dataset_mapping[self.resolution][parameter_] if not self._unique_dataset: # Parameter then has to be taken from the datasets definition - parameter_ = self._dataset_tree[self._dataset_accessor][ - dataset_.name - ][parameter_.name] + parameter_ = self._dataset_tree[self._dataset_accessor][dataset_.name][parameter_.name] parameters.append((parameter_, dataset_)) @@ -342,9 +327,7 @@ def __init__( super().__init__() - self.resolution = parse_enumeration_from_template( - resolution, self._resolution_base, Resolution - ) + self.resolution = parse_enumeration_from_template(resolution, self._resolution_base, Resolution) self.period = self._parse_period(period) self.start_date, self.end_date = self.convert_timestamps(start_date, end_date) @@ -353,12 +336,7 @@ def __init__( tidy = tidy if self._has_datasets: - tidy = tidy or any( - [ - parameter not in self._dataset_base - for parameter, dataset in self.parameter - ] - ) + tidy = tidy or any([parameter not in self._dataset_base for parameter, dataset in self.parameter]) self.tidy = tidy self.si_units = si_units @@ -414,9 +392,7 @@ def convert_timestamps( # TODO: replace this with a response + logging if not start_date <= end_date: - raise StartDateEndDateError( - "Error: 'start_date' must be smaller or equal to 'end_date'." - ) + raise StartDateEndDateError("Error: 'start_date' must be smaller or equal to 'end_date'.") return pd.Timestamp(start_date), pd.Timestamp(end_date) @@ -466,30 +442,20 @@ def discover(cls, filter_=None, dataset=None, flatten: bool = True) -> dict: parameters[f.name.lower()][parameter.name.lower()] = {} if cls._unique_dataset: - origin_unit, si_unit = cls._unit_tree[f.name][ - parameter.name - ].value + origin_unit, si_unit = cls._unit_tree[f.name][parameter.name].value else: dataset = cls._parameter_to_dataset_mapping[f][parameter] - origin_unit, si_unit = cls._unit_tree[f.name][dataset.name][ - parameter.name - ].value + origin_unit, si_unit = cls._unit_tree[f.name][dataset.name][parameter.name].value - parameters[f.name.lower()][parameter.name.lower()][ - "origin" - ] = cls._format_unit(origin_unit) + parameters[f.name.lower()][parameter.name.lower()]["origin"] = cls._format_unit(origin_unit) - parameters[f.name.lower()][parameter.name.lower()][ - "si" - ] = cls._format_unit(si_unit) + parameters[f.name.lower()][parameter.name.lower()]["si"] = cls._format_unit(si_unit) return parameters datasets_filter = ( - pd.Series(dataset, dtype=str) - .apply(parse_enumeration_from_template, args=(cls._dataset_base,)) - .tolist() + pd.Series(dataset, dtype=str).apply(parse_enumeration_from_template, args=(cls._dataset_base,)).tolist() or cls._dataset_base ) @@ -508,21 +474,17 @@ def discover(cls, filter_=None, dataset=None, flatten: bool = True) -> dict: for parameter in cls._dataset_tree[f.name][dataset]: - parameters[f.name.lower()][dataset.lower()][ - parameter.name.lower() - ] = {} + parameters[f.name.lower()][dataset.lower()][parameter.name.lower()] = {} - origin_unit, si_unit = cls._unit_tree[f.name][dataset][ - parameter.name - ].value + origin_unit, si_unit = cls._unit_tree[f.name][dataset][parameter.name].value - parameters[f.name.lower()][dataset.lower()][parameter.name.lower()][ - "origin" - ] = cls._format_unit(origin_unit) + parameters[f.name.lower()][dataset.lower()][parameter.name.lower()]["origin"] = cls._format_unit( + origin_unit + ) - parameters[f.name.lower()][dataset.lower()][parameter.name.lower()][ - "si" - ] = cls._format_unit(si_unit) + parameters[f.name.lower()][dataset.lower()][parameter.name.lower()]["si"] = cls._format_unit( + si_unit + ) return parameters @@ -543,11 +505,7 @@ def _setup_discover_filter(cls, filter_) -> list: filter_ = [*cls._resolution_base] filter_ = ( - pd.Series(filter_) - .apply( - parse_enumeration_from_template, args=(cls._resolution_base, Resolution) - ) - .tolist() + pd.Series(filter_).apply(parse_enumeration_from_template, args=(cls._resolution_base, Resolution)).tolist() ) return filter_ @@ -626,9 +584,7 @@ def filter_by_station_id(self, station_id: Tuple[str, ...]) -> StationsResult: return result - def filter_by_name( - self, name: str, first: bool = True, threshold: int = 90 - ) -> StationsResult: + def filter_by_name(self, name: str, first: bool = True, threshold: int = 90) -> StationsResult: """ Method to filter stations for station name using string comparison. @@ -708,9 +664,7 @@ def filter_by_rank( # If num_stations_nearby is higher then the actual amount of stations # further indices and distances are added which have to be filtered out distances = distances[: min(df.shape[0], rank)] - indices_nearest_neighbours = indices_nearest_neighbours[ - : min(df.shape[0], rank) - ] + indices_nearest_neighbours = indices_nearest_neighbours[: min(df.shape[0], rank)] distances_km = np.array(distances * EARTH_RADIUS_KM) @@ -720,8 +674,7 @@ def filter_by_rank( if df.empty: log.warning( - f"No weather stations were found for coordinate " - f"{latitude}°N and {longitude}°E and number {rank}" + f"No weather stations were found for coordinate " f"{latitude}°N and {longitude}°E and number {rank}" ) result = StationsResult(self, df.reset_index(drop=True)) @@ -752,13 +705,9 @@ def filter_by_distance( distance_in_km = guess(distance, unit, [Distance]).km # TODO: replace the repeating call to self.all() - all_nearby_stations = self.filter_by_rank( - latitude, longitude, self.all().df.shape[0] - ).df + all_nearby_stations = self.filter_by_rank(latitude, longitude, self.all().df.shape[0]).df - df = all_nearby_stations[ - all_nearby_stations[Columns.DISTANCE.value] <= distance_in_km - ] + df = all_nearby_stations[all_nearby_stations[Columns.DISTANCE.value] <= distance_in_km] if df.empty: log.warning( @@ -770,9 +719,7 @@ def filter_by_distance( return result - def filter_by_bbox( - self, left: float, bottom: float, right: float, top: float - ) -> StationsResult: + def filter_by_bbox(self, left: float, bottom: float, right: float, top: float) -> StationsResult: """ Method to filter stations by bounding box. @@ -816,9 +763,7 @@ def filter_by_sql(self, sql: str) -> pd.DataFrame: df = duckdb.query_df(df, "data", sql).df() - df[Columns.FROM_DATE.value] = df[Columns.FROM_DATE.value].dt.tz_localize( - self.tz - ) + df[Columns.FROM_DATE.value] = df[Columns.FROM_DATE.value].dt.tz_localize(self.tz) df[Columns.TO_DATE.value] = df[Columns.TO_DATE.value].dt.tz_localize(self.tz) result = StationsResult(stations=self, df=df.reset_index(drop=True)) diff --git a/wetterdienst/core/scalar/result.py b/wetterdienst/core/scalar/result.py index f2564318b..e4c74ba0a 100644 --- a/wetterdienst/core/scalar/result.py +++ b/wetterdienst/core/scalar/result.py @@ -20,12 +20,7 @@ class StationsResult(ExportMixin): - def __init__( - self, - stations: Union["ScalarRequestCore", "DwdMosmixRequest"], - df: pd.DataFrame, - **kwargs - ) -> None: + def __init__(self, stations: Union["ScalarRequestCore", "DwdMosmixRequest"], df: pd.DataFrame, **kwargs) -> None: # TODO: add more attributes from ScalarStations class self.stations = stations self.df = df @@ -146,7 +141,5 @@ def to_ogc_feature_collection(self): raise NotImplementedError() def filter_by_date(self, date: str) -> pd.DataFrame: - self.df = filter_by_date_and_resolution( - self.df, date=date, resolution=self.stations.resolution - ) + self.df = filter_by_date_and_resolution(self.df, date=date, resolution=self.stations.resolution) return self.df diff --git a/wetterdienst/core/scalar/values.py b/wetterdienst/core/scalar/values.py index 0d43b813b..e602fcbf3 100644 --- a/wetterdienst/core/scalar/values.py +++ b/wetterdienst/core/scalar/values.py @@ -145,9 +145,7 @@ def _get_timezone_from_station(self, station_id: str) -> timezone: station = stations[stations[Columns.STATION_ID.value] == station_id] - longitude, latitude = station.loc[ - :, [Columns.LONGITUDE.value, Columns.LATITUDE.value] - ].T.values + longitude, latitude = station.loc[:, [Columns.LONGITUDE.value, Columns.LATITUDE.value]].T.values tz_string = self._tf.timezone_at(lng=longitude, lat=latitude) @@ -211,9 +209,7 @@ def _convert_values_to_si(series, parameter: Optional[str] = None) -> pd.Series: if self.stations.stations._has_tidy_data: df_si = pd.DataFrame() for par, group in df.groupby(Columns.PARAMETER.value): - group[Columns.VALUE.value] = _convert_values_to_si( - group[Columns.VALUE.value], par - ) + group[Columns.VALUE.value] = _convert_values_to_si(group[Columns.VALUE.value], par) df_si = df_si.append(group) else: @@ -221,9 +217,7 @@ def _convert_values_to_si(series, parameter: Optional[str] = None) -> pd.Series: return df_si - def _create_conversion_factors( - self, dataset - ) -> Dict[str, Tuple[Union[operator.add, operator.mul], float]]: + def _create_conversion_factors(self, dataset) -> Dict[str, Tuple[Union[operator.add, operator.mul], float]]: """ Function to create conversion factors based on a given dataset @@ -249,13 +243,9 @@ def _create_conversion_factors( parameter = parameter.name if self.stations.stations._unique_dataset: - parameter_value = self.stations.stations._dataset_tree[ - dataset_accessor - ][parameter].value + parameter_value = self.stations.stations._dataset_tree[dataset_accessor][parameter].value else: - parameter_value = self.stations.stations._dataset_tree[ - dataset_accessor - ][dataset][parameter].value + parameter_value = self.stations.stations._dataset_tree[dataset_accessor][dataset][parameter].value if si_unit == SIUnit.KILOGRAM_PER_SQUARE_METER.value: # Fixed conversion factors to kg / m², as it only applies @@ -263,9 +253,7 @@ def _create_conversion_factors( if origin_unit == OriginUnit.MILLIMETER.value: conversion_factors[parameter_value] = (operator.mul, 1) else: - raise ValueError( - "manually set conversion factor for precipitation unit" - ) + raise ValueError("manually set conversion factor for precipitation unit") elif si_unit == SIUnit.DEGREE_KELVIN.value: # Apply offset addition to temperature measurements # Take 0 as this is appropriate for adding on other numbers @@ -308,15 +296,10 @@ def __str__(self): """ Str representation of request object """ # TODO: include source # TODO: include data type - station_ids_joined = "& ".join( - [str(station_id) for station_id in self.stations.station_id] - ) + station_ids_joined = "& ".join([str(station_id) for station_id in self.stations.station_id]) parameters_joined = "& ".join( - [ - parameter.value - for parameter, parameter_set in self.stations.stations.parameter - ] + [parameter.value for parameter, parameter_set in self.stations.stations.parameter] ) return ", ".join( @@ -329,9 +312,7 @@ def __str__(self): ) pass - def _create_empty_station_parameter_df( - self, station_id: str, parameter: Enum, dataset: Enum - ) -> pd.DataFrame: + def _create_empty_station_parameter_df(self, station_id: str, parameter: Enum, dataset: Enum) -> pd.DataFrame: """ Function to create an empty DataFrame :param station_id: @@ -388,9 +369,7 @@ def _create_empty_station_parameter_df( return df - def _build_complete_df( - self, df: pd.DataFrame, station_id: str, parameter: Enum, dataset: Enum - ) -> pd.DataFrame: + def _build_complete_df(self, df: pd.DataFrame, station_id: str, parameter: Enum, dataset: Enum) -> pd.DataFrame: # For cases where requests are not defined by start and end date but rather by # periods, use the returned df without modifications # We may put a standard date range here if no data is found @@ -412,15 +391,11 @@ def _build_complete_df( if self.stations.tidy: df[Columns.PARAMETER.value] = parameter.value - df[Columns.PARAMETER.value] = pd.Categorical( - df[Columns.PARAMETER.value] - ) + df[Columns.PARAMETER.value] = pd.Categorical(df[Columns.PARAMETER.value]) if dataset: df[Columns.DATASET.value] = dataset.name.lower() - df[Columns.DATASET.value] = pd.Categorical( - df[Columns.DATASET.value] - ) + df[Columns.DATASET.value] = pd.Categorical(df[Columns.DATASET.value]) return df else: @@ -429,16 +404,12 @@ def _build_complete_df( if self.stations.stations._unique_dataset: parameter_ = parse_enumeration_from_template( parameter, - self.stations.stations._parameter_base[ - self.stations.resolution.name - ], + self.stations.stations._parameter_base[self.stations.resolution.name], ) else: parameter_ = parse_enumeration_from_template( parameter, - self.stations.stations._dataset_tree[ - self.stations.resolution.name - ][dataset.name], + self.stations.stations._dataset_tree[self.stations.resolution.name][dataset.name], ) df = pd.merge( @@ -493,10 +464,7 @@ def query(self) -> Generator[ValuesResult, None, None]: # TODO: For now skip date based parameters as we didn't # yet decide how to convert those values to floats to # make them conform with tidy shape - single_tidy_date_parameters = ( - self.stations.stations.tidy - and parameter.value in self._date_parameters - ) + single_tidy_date_parameters = self.stations.stations.tidy and parameter.value in self._date_parameters if single_tidy_date_parameters: log.warning( f"parameter {parameter.value} is skipped in tidy format " @@ -504,9 +472,7 @@ def query(self) -> Generator[ValuesResult, None, None]: ) continue - parameter_df = self._collect_station_parameter( - station_id, parameter, dataset - ) + parameter_df = self._collect_station_parameter(station_id, parameter, dataset) if parameter_df.empty: continue @@ -525,21 +491,14 @@ def query(self) -> Generator[ValuesResult, None, None]: parameter_df = self.tidy_up_df(parameter_df, dataset) if parameter != dataset: - parameter_df = parameter_df[ - parameter_df[Columns.PARAMETER.value] - == parameter.value.lower() - ] + parameter_df = parameter_df[parameter_df[Columns.PARAMETER.value] == parameter.value.lower()] elif self.stations.stations._has_tidy_data: parameter_df = self.tabulate_df(parameter_df) # Skip date fields in tidy format, no further check required as still # "normal" parameters should be available if self.stations.stations.tidy and self._date_parameters: - parameter_df = parameter_df[ - ~parameter_df[Columns.PARAMETER.value].isin( - self._date_parameters - ) - ] + parameter_df = parameter_df[~parameter_df[Columns.PARAMETER.value].isin(self._date_parameters)] log.warning( f"parameters {self._date_parameters} are skipped in tidy format " f"as the date parameters are currently not converted to floats" @@ -549,9 +508,7 @@ def query(self) -> Generator[ValuesResult, None, None]: # Merge on full date range if values are found to ensure result # even if no actual values exist - parameter_df = self._build_complete_df( - parameter_df, station_id, parameter, dataset - ) + parameter_df = self._build_complete_df(parameter_df, station_id, parameter, dataset) parameter_df[Columns.DATASET.value] = dataset.name.lower() @@ -562,9 +519,7 @@ def query(self) -> Generator[ValuesResult, None, None]: try: station_df = pd.concat(station_data, ignore_index=True) except ValueError: - station_df = self._create_empty_station_parameter_df( - station_id, parameter, dataset - ) + station_df = self._create_empty_station_parameter_df(station_id, parameter, dataset) station_df = self._coerce_meta_fields(station_df) @@ -589,9 +544,7 @@ def query(self) -> Generator[ValuesResult, None, None]: yield ValuesResult(stations=self.stations, df=station_df) @abstractmethod - def _collect_station_parameter( - self, station_id: str, parameter: Enum, dataset: Enum - ) -> pd.DataFrame: + def _collect_station_parameter(self, station_id: str, parameter: Enum, dataset: Enum) -> pd.DataFrame: """ Implementation of data collection for a station id plus parameter from the specified weather service. Takes care of the gathering of the data and putting @@ -621,9 +574,7 @@ def tidy_up_df(self, df: pd.DataFrame, dataset: Enum) -> pd.DataFrame: if Columns.QUALITY.value not in df: df[Columns.QUALITY.value] = np.nan - df[Columns.QUALITY.value] = pd.to_numeric(df[Columns.QUALITY.value]).astype( - float - ) + df[Columns.QUALITY.value] = pd.to_numeric(df[Columns.QUALITY.value]).astype(float) # Set quality of NaN values to NaN as well df.loc[df[Columns.VALUE.value].isna(), Columns.QUALITY.value] = np.NaN @@ -712,31 +663,19 @@ def _coerce_meta_fields(self, df: pd.DataFrame) -> pd.DataFrame: :param df: pandas.DataFrame with the "fresh" data :return: pandas.DataFrame with meta fields being coerced """ - df[Columns.STATION_ID.value] = self._parse_station_id( - df[Columns.STATION_ID.value] - ).astype("category") + df[Columns.STATION_ID.value] = self._parse_station_id(df[Columns.STATION_ID.value]).astype("category") # TODO: why do we need this (again)? - df[Columns.DATE.value] = pd.to_datetime( - df[Columns.DATE.value], infer_datetime_format=True - ) + df[Columns.DATE.value] = pd.to_datetime(df[Columns.DATE.value], infer_datetime_format=True) - df[Columns.DATASET.value] = self._coerce_strings( - df[Columns.DATASET.value] - ).astype("category") + df[Columns.DATASET.value] = self._coerce_strings(df[Columns.DATASET.value]).astype("category") if self.stations.stations.tidy: - df[Columns.PARAMETER.value] = self._coerce_strings( - df[Columns.PARAMETER.value] - ).astype("category") - df[Columns.VALUE.value] = pd.to_numeric(df[Columns.VALUE.value]).astype( - float - ) + df[Columns.PARAMETER.value] = self._coerce_strings(df[Columns.PARAMETER.value]).astype("category") + df[Columns.VALUE.value] = pd.to_numeric(df[Columns.VALUE.value]).astype(float) # TODO: may coerce more carefully quality codes or replace them by numbers - df[Columns.QUALITY.value] = pd.to_numeric( - df[Columns.QUALITY.value], errors="coerce" - ).astype(float) + df[Columns.QUALITY.value] = pd.to_numeric(df[Columns.QUALITY.value], errors="coerce").astype(float) return df @@ -776,11 +715,7 @@ def _coerce_integers(series: pd.Series) -> pd.Series: :param series: :return: """ - return ( - pd.to_numeric(series, errors="coerce") - .astype(pd.Float64Dtype()) - .astype(pd.Int64Dtype()) - ) + return pd.to_numeric(series, errors="coerce").astype(pd.Float64Dtype()).astype(pd.Int64Dtype()) @staticmethod def _coerce_strings(series: pd.Series) -> pd.Series: @@ -812,10 +747,7 @@ def _coerce_irregular_parameter(self, series: pd.Series) -> pd.Series: :return: """ if self._irregular_parameters: - raise NotImplementedError( - "implement _parse_irregular_parameter " - "method to parse irregular parameters" - ) + raise NotImplementedError("implement _parse_irregular_parameter " "method to parse irregular parameters") return pd.Series(series) @@ -831,9 +763,7 @@ def _coerce_parameter_types(self, df: pd.DataFrame) -> pd.DataFrame: continue if column in self._irregular_parameters: df[column] = self._coerce_irregular_parameter(df[column]) - elif column in self._integer_parameters or column.startswith( - Columns.QUALITY_PREFIX.value - ): + elif column in self._integer_parameters or column.startswith(Columns.QUALITY_PREFIX.value): df[column] = self._coerce_integers(df[column]) elif column in self._string_parameters: df[column] = self._coerce_strings(df[column]) @@ -852,9 +782,7 @@ def all(self) -> ValuesResult: tqdm_out = TqdmToLogger(log, level=logging.INFO) - for result in tqdm( - self.query(), total=len(self.stations.station_id), file=tqdm_out - ): + for result in tqdm(self.query(), total=len(self.stations.station_id), file=tqdm_out): data.append(result.df) try: @@ -880,9 +808,7 @@ def _humanize(self, df: pd.DataFrame) -> pd.DataFrame: if not self.stations.tidy: df = df.rename(columns=hcnm) else: - df[Columns.PARAMETER.value] = df[ - Columns.PARAMETER.value - ].cat.rename_categories(hcnm) + df[Columns.PARAMETER.value] = df[Columns.PARAMETER.value].cat.rename_categories(hcnm) return df @@ -896,9 +822,7 @@ def _create_humanized_parameters_mapping(self) -> Dict[str, str]: hcnm = { parameter.value: parameter.name.lower() - for parameter in self.stations.stations._parameter_base[ - self.stations.stations.resolution.name - ] + for parameter in self.stations.stations._parameter_base[self.stations.stations.resolution.name] } return hcnm diff --git a/wetterdienst/metadata/parameter.py b/wetterdienst/metadata/parameter.py index c2aa07a77..054a25c07 100644 --- a/wetterdienst/metadata/parameter.py +++ b/wetterdienst/metadata/parameter.py @@ -47,13 +47,9 @@ class Parameter(Enum): # total CLOUD_COVER_TOTAL = "CLOUD_COVER_TOTAL" CLOUD_COVER_TOTAL_MIDNIGHT_TO_MIDNIGHT = "CLOUD_COVER_TOTAL_MIDNIGHT_TO_MIDNIGHT" - CLOUD_COVER_TOTAL_MIDNIGHT_TO_MIDNIGHT_MANUAL = ( - "CLOUD_COVER_TOTAL_MIDNIGHT_TO_MIDNIGHT_MANUAL" - ) + CLOUD_COVER_TOTAL_MIDNIGHT_TO_MIDNIGHT_MANUAL = "CLOUD_COVER_TOTAL_MIDNIGHT_TO_MIDNIGHT_MANUAL" CLOUD_COVER_TOTAL_SUNRISE_TO_SUNSET = "CLOUD_COVER_TOTAL_SUNRISE_TO_SUNSET" - CLOUD_COVER_TOTAL_SUNRISE_TO_SUNSET_MANUAL = ( - "CLOUD_COVER_TOTAL_SUNRISE_TO_SUNSET_MANUAL" - ) + CLOUD_COVER_TOTAL_SUNRISE_TO_SUNSET_MANUAL = "CLOUD_COVER_TOTAL_SUNRISE_TO_SUNSET_MANUAL" CLOUD_COVER_EFFECTIVE = "CLOUD_COVER_EFFECTIVE" # layers CLOUD_COVER_LAYER1 = "CLOUD_COVER_LAYER1" @@ -127,24 +123,12 @@ class Parameter(Enum): PRECIPITATION_HEIGHT_LAST_24H = "PRECIPITATION_HEIGHT_LAST_24H" PRECIPITATION_HEIGHT_MULTIDAY = "PRECIPITATION_HEIGHT_MULTIDAY" # precipitation height consistent with significant weather - PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_1H = ( - "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_1H" - ) - PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_3H = ( - "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_3H" - ) - PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_6H = ( - "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_6H" - ) - PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_12H = ( - "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_12H" - ) - PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_24H = ( - "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_24H" - ) - PRECIPITATION_HEIGHT_LIQUID_SIGNIFICANT_WEATHER_LAST_1H = ( - "PRECIPITATION_HEIGHT_LIQUID_SIGNIFICANT_WEATHER_LAST_1H" - ) + PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_1H = "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_1H" + PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_3H = "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_3H" + PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_6H = "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_6H" + PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_12H = "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_12H" + PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_24H = "PRECIPITATION_HEIGHT_SIGNIFICANT_WEATHER_LAST_24H" + PRECIPITATION_HEIGHT_LIQUID_SIGNIFICANT_WEATHER_LAST_1H = "PRECIPITATION_HEIGHT_LIQUID_SIGNIFICANT_WEATHER_LAST_1H" # ---- extremes ---- PRECIPITATION_HEIGHT_MAX = "PRECIPITATION_HEIGHT_MAX" PRECIPITATION_HEIGHT_LIQUID_MAX = "PRECIPITATION_HEIGHT_LIQUID_MAX" @@ -156,89 +140,41 @@ class Parameter(Enum): PRECIPITATION_DURATION = "PRECIPITATION_DURATION" # ---- probability ---- # greater 0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_6H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_6H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_12H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_12H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_24H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_24H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_6H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_6H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_12H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_12H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_24H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_0_MM_LAST_24H" # greater 0.1 - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_1_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_1_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_1_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_1_MM_LAST_1H" # greater 0.2 - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_1H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_6H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_6H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_12H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_12H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_24H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_24H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_1H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_6H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_6H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_12H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_12H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_24H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_2_MM_LAST_24H" # greater 0.3 - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_3_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_3_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_3_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_3_MM_LAST_1H" # greater 0.5 - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_5_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_5_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_5_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_5_MM_LAST_1H" # greater 0.7 - PROBABILITY_PRECIPITATION_HEIGHT_GT_0_7_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_7_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_0_7_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_0_7_MM_LAST_1H" # greater 1.0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_1H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_6H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_6H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_12H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_12H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_24H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_24H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_1H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_6H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_6H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_12H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_12H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_24H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_1_0_MM_LAST_24H" # greater 2.0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_2_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_2_0_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_2_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_2_0_MM_LAST_1H" # greater 3.0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_3_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_3_0_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_3_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_3_0_MM_LAST_1H" # greater 5.0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_1H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_6H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_6H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_12H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_12H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_24H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_24H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_1H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_6H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_6H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_12H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_12H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_24H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_5_0_MM_LAST_24H" # greater 10.0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_10_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_10_0_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_10_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_10_0_MM_LAST_1H" # greater 15.0 - PROBABILITY_PRECIPITATION_HEIGHT_GT_15_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_15_0_MM_LAST_1H" - ) - PROBABILITY_PRECIPITATION_HEIGHT_GT_25_0_MM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_HEIGHT_GT_25_0_MM_LAST_1H" - ) + PROBABILITY_PRECIPITATION_HEIGHT_GT_15_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_15_0_MM_LAST_1H" + PROBABILITY_PRECIPITATION_HEIGHT_GT_25_0_MM_LAST_1H = "PROBABILITY_PRECIPITATION_HEIGHT_GT_25_0_MM_LAST_1H" PROBABILITY_PRECIPITATION_LAST_1H = "PROBABILITY_PRECIPITATION_LAST_1H" PROBABILITY_PRECIPITATION_LAST_6H = "PROBABILITY_PRECIPITATION_LAST_6H" @@ -249,60 +185,32 @@ class Parameter(Enum): PROBABILITY_DRIZZLE_LAST_6H = "PROBABILITY_DRIZZLE_LAST_6H" PROBABILITY_DRIZZLE_LAST_12H = "PROBABILITY_DRIZZLE_LAST_12H" - PROBABILITY_PRECIPITATION_STRATIFORM_LAST_1H = ( - "PROBABILITY_PRECIPITATION_STRATIFORM_LAST_1H" - ) - PROBABILITY_PRECIPITATION_STRATIFORM_LAST_6H = ( - "PROBABILITY_PRECIPITATION_STRATIFORM_LAST_6H" - ) - PROBABILITY_PRECIPITATION_STRATIFORM_LAST_12H = ( - "PROBABILITY_PRECIPITATION_STRATIFORM_LAST_12H" - ) + PROBABILITY_PRECIPITATION_STRATIFORM_LAST_1H = "PROBABILITY_PRECIPITATION_STRATIFORM_LAST_1H" + PROBABILITY_PRECIPITATION_STRATIFORM_LAST_6H = "PROBABILITY_PRECIPITATION_STRATIFORM_LAST_6H" + PROBABILITY_PRECIPITATION_STRATIFORM_LAST_12H = "PROBABILITY_PRECIPITATION_STRATIFORM_LAST_12H" - PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_1H = ( - "PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_1H" - ) - PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_6H = ( - "PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_6H" - ) - PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_12H = ( - "PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_12H" - ) + PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_1H = "PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_1H" + PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_6H = "PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_6H" + PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_12H = "PROBABILITY_PRECIPITATION_CONVECTIVE_LAST_12H" - PROBABILITY_PRECIPITATION_LIQUID_LAST_1H = ( - "PROBABILITY_PRECIPITATION_LIQUID_LAST_1H" - ) - PROBABILITY_PRECIPITATION_LIQUID_LAST_6H = ( - "PROBABILITY_PRECIPITATION_LIQUID_LAST_6H" - ) - PROBABILITY_PRECIPITATION_LIQUID_LAST_12H = ( - "PROBABILITY_PRECIPITATION_LIQUID_LAST_12H" - ) + PROBABILITY_PRECIPITATION_LIQUID_LAST_1H = "PROBABILITY_PRECIPITATION_LIQUID_LAST_1H" + PROBABILITY_PRECIPITATION_LIQUID_LAST_6H = "PROBABILITY_PRECIPITATION_LIQUID_LAST_6H" + PROBABILITY_PRECIPITATION_LIQUID_LAST_12H = "PROBABILITY_PRECIPITATION_LIQUID_LAST_12H" PROBABILITY_PRECIPITATION_SOLID_LAST_1H = "PROBABILITY_PRECIPITATION_SOLID_LAST_1H" PROBABILITY_PRECIPITATION_SOLID_LAST_6H = "PROBABILITY_PRECIPITATION_SOLID_LAST_6H" - PROBABILITY_PRECIPITATION_SOLID_LAST_12H = ( - "PROBABILITY_PRECIPITATION_SOLID_LAST_12H" - ) + PROBABILITY_PRECIPITATION_SOLID_LAST_12H = "PROBABILITY_PRECIPITATION_SOLID_LAST_12H" - PROBABILITY_PRECIPITATION_FREEZING_LAST_1H = ( - "PROBABILITY_PRECIPITATION_FREEZING_LAST_1H" - ) - PROBABILITY_PRECIPITATION_FREEZING_LAST_6H = ( - "PROBABILITY_PRECIPITATION_FREEZING_LAST_6H" - ) - PROBABILITY_PRECIPITATION_FREEZING_LAST_12H = ( - "PROBABILITY_PRECIPITATION_FREEZING_LAST_12H" - ) + PROBABILITY_PRECIPITATION_FREEZING_LAST_1H = "PROBABILITY_PRECIPITATION_FREEZING_LAST_1H" + PROBABILITY_PRECIPITATION_FREEZING_LAST_6H = "PROBABILITY_PRECIPITATION_FREEZING_LAST_6H" + PROBABILITY_PRECIPITATION_FREEZING_LAST_12H = "PROBABILITY_PRECIPITATION_FREEZING_LAST_12H" # ---- frequency ---- PRECIPITATION_FREQUENCY = "PRECIPITATION_FREQUENCY" # ---- count ---- # Number of days included in the multiday precipitation total COUNT_DAYS_MULTIDAY_PRECIPITATION = "COUNT_DAYS_MULTIDAY_PRECIPITATION" # Number of days with non-zero precipitation included in multiday precipitation total - COUNT_DAYS_MULTIDAY_PRECIPITATION_HEIGHT_GT_0 = ( - "COUNT_DAYS_MULTIDAY_PRECIPITATION_HEIGHT_GT_0" - ) + COUNT_DAYS_MULTIDAY_PRECIPITATION_HEIGHT_GT_0 = "COUNT_DAYS_MULTIDAY_PRECIPITATION_HEIGHT_GT_0" # PRESSURE # ---- averaged ---- @@ -562,18 +470,12 @@ class Parameter(Enum): COUNT_DAYS_HEATING_DEGREE = "COUNT_DAYS_HEATING_DEGREE" COUNT_DAYS_COOLING_DEGREE = "COUNT_DAYS_COOLING_DEGREE" # Number of days included in the multiday minimum temperature - COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MIN_200 = ( - "COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MIN_200" - ) + COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MIN_200 = "COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MIN_200" # Number of days included in the multiday maximum temperature - COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MAX_200 = ( - "COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MAX_200" - ) + COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MAX_200 = "COUNT_DAYS_MULTIDAY_TEMPERATURE_AIR_MAX_200" # ---- error ---- ERROR_ABSOLUTE_TEMPERATURE_AIR_MEAN_200 = "ERROR_ABSOLUTE_TEMPERATURE_AIR_MEAN_200" - ERROR_ABSOLUTE_TEMPERATURE_DEW_POINT_MEAN_200 = ( - "ERROR_ABSOLUTE_TEMPERATURE_DEW_POINT_MEAN_200" - ) + ERROR_ABSOLUTE_TEMPERATURE_DEW_POINT_MEAN_200 = "ERROR_ABSOLUTE_TEMPERATURE_DEW_POINT_MEAN_200" # VISIBILITY # ---- distance ---- @@ -622,12 +524,8 @@ class Parameter(Enum): WEATHER_TYPE_FREEZING_DRIZZLE = "WEATHER_TYPE_FREEZING_DRIZZLE" WEATHER_TYPE_RAIN = "WEATHER_TYPE_RAIN" WEATHER_TYPE_FREEZING_RAIN = "WEATHER_TYPE_FREEZING_RAIN" - WEATHER_TYPE_SNOW_PELLETS_SNOW_GRAINS_ICE_CRYSTALS = ( - "WEATHER_TYPE_SNOW_PELLETS_SNOW_GRAINS_ICE_CRYSTALS" - ) - WEATHER_TYPE_PRECIPITATION_UNKNOWN_SOURCE = ( - "WEATHER_TYPE_PRECIPITATION_UNKNOWN_SOURCE" - ) + WEATHER_TYPE_SNOW_PELLETS_SNOW_GRAINS_ICE_CRYSTALS = "WEATHER_TYPE_SNOW_PELLETS_SNOW_GRAINS_ICE_CRYSTALS" + WEATHER_TYPE_PRECIPITATION_UNKNOWN_SOURCE = "WEATHER_TYPE_PRECIPITATION_UNKNOWN_SOURCE" WEATHER_TYPE_GROUND_FOG = "WEATHER_TYPE_GROUND_FOG" WEATHER_TYPE_ICE_FOG_FREEZING_FOG = "WEATHER_TYPE_ICE_FOG_FREEZING_FOG" diff --git a/wetterdienst/metadata/period.py b/wetterdienst/metadata/period.py index d927ed0f9..8e1ee4095 100644 --- a/wetterdienst/metadata/period.py +++ b/wetterdienst/metadata/period.py @@ -28,18 +28,12 @@ def _period_type_order_mapping(self): def __lt__(self, other): if self.__class__ is other.__class__: - return ( - self._period_type_order_mapping[self.name] - < self._period_type_order_mapping[other.name] - ) + return self._period_type_order_mapping[self.name] < self._period_type_order_mapping[other.name] return NotImplemented def __gt__(self, other): if self.__class__ is other.__class__: - return ( - self._period_type_order_mapping[self.name] - > self._period_type_order_mapping[other.name] - ) + return self._period_type_order_mapping[self.name] > self._period_type_order_mapping[other.name] return NotImplemented def __ge__(self, other): diff --git a/wetterdienst/metadata/provider.py b/wetterdienst/metadata/provider.py index ed0a664e0..b0bb7159d 100644 --- a/wetterdienst/metadata/provider.py +++ b/wetterdienst/metadata/provider.py @@ -31,7 +31,6 @@ class Provider(Enum): "National Oceanic And Atmospheric Administration", "National Oceanic And Atmospheric Administration", "United States Of America", - "© National Oceanic And Atmospheric Administration (NOAA), " - "Global Historical Climatology Network", + "© National Oceanic And Atmospheric Administration (NOAA), " "Global Historical Climatology Network", "ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily/", ) diff --git a/wetterdienst/metadata/unit.py b/wetterdienst/metadata/unit.py index 566978ce5..20aad7f15 100644 --- a/wetterdienst/metadata/unit.py +++ b/wetterdienst/metadata/unit.py @@ -40,17 +40,13 @@ class OriginUnit(Enum): KILOGRAM_PER_SQUARE_METER = REGISTRY.kilogram / (REGISTRY.meter ** 2) # Temperature - DEGREE_CELSIUS = ( - 1 * REGISTRY.degree_Celsius - ) # without the "1 *" we get an offset error + DEGREE_CELSIUS = 1 * REGISTRY.degree_Celsius # without the "1 *" we get an offset error DEGREE_KELVIN = 1 * REGISTRY.degree_Kelvin # Speed METER_PER_SECOND = REGISTRY.meter / REGISTRY.second KILOMETER_PER_HOUR = REGISTRY.kilometer / REGISTRY.hour - BEAUFORT = ( - REGISTRY.beaufort - ) # beaufort should always stay beaufort! Calculations to m/s are empirical + BEAUFORT = REGISTRY.beaufort # beaufort should always stay beaufort! Calculations to m/s are empirical # Pressure PASCAL = REGISTRY.pascal diff --git a/wetterdienst/provider/dwd/forecast/access.py b/wetterdienst/provider/dwd/forecast/access.py index 3ccdd91b7..4e98dbf41 100644 --- a/wetterdienst/provider/dwd/forecast/access.py +++ b/wetterdienst/provider/dwd/forecast/access.py @@ -39,7 +39,7 @@ def __init__( self.dwdfs = NetworkFilesystemManager.get(ttl=CacheExpiry.FIVE_MINUTES) def download(self, url: str): - # https://stackoverflow.com/questions/37573483/progress-bar-while-download-file-over-http-with-requests # noqa:E501,B950 + # https://stackoverflow.com/questions/37573483/progress-bar-while-download-file-over-http-with-requests # block_size: int or None # Bytes to download in one request; use instance value if None. If @@ -95,14 +95,9 @@ def read(self, url: str): } # Get Basic Metadata - prod_definition = root.findall( - "kml:Document/kml:ExtendedData/dwd:ProductDefinition", root.nsmap - )[0] + prod_definition = root.findall("kml:Document/kml:ExtendedData/dwd:ProductDefinition", root.nsmap)[0] - self.metadata = { - k: prod_definition.find(f"{{{root.nsmap['dwd']}}}{v}").text - for k, v in prod_items.items() - } + self.metadata = {k: prod_definition.find(f"{{{root.nsmap['dwd']}}}{v}").text for k, v in prod_items.items()} self.metadata["issue_time"] = pd.Timestamp(self.metadata["issue_time"]) # Get time steps. @@ -110,9 +105,7 @@ def read(self, url: str): "kml:Document/kml:ExtendedData/dwd:ProductDefinition/dwd:ForecastTimeSteps", root.nsmap, )[0] - self.timesteps = DatetimeIndex( - [pd.Timestamp(i.text) for i in timesteps.getchildren()] - ) + self.timesteps = DatetimeIndex([pd.Timestamp(i.text) for i in timesteps.getchildren()]) # Find all kml:Placemark items. self.items = root.findall("kml:Document/kml:Placemark", root.nsmap) @@ -131,25 +124,19 @@ def get_forecasts(self): for station_forecast in self.iter_items(): station_ids = station_forecast.find("kml:name", self.root.nsmap).text - measurement_list = station_forecast.findall( - "kml:ExtendedData/dwd:Forecast", self.root.nsmap - ) + measurement_list = station_forecast.findall("kml:ExtendedData/dwd:Forecast", self.root.nsmap) data_dict = {"station_id": station_ids, "datetime": self.timesteps} for measurement_item in measurement_list: - measurement_parameter = measurement_item.get( - f"{{{self.root.nsmap['dwd']}}}elementName" - ) + measurement_parameter = measurement_item.get(f"{{{self.root.nsmap['dwd']}}}elementName") if measurement_parameter.lower() in self.parameters: measurement_string = measurement_item.getchildren()[0].text measurement_values = " ".join(measurement_string.split()).split(" ") - measurement_values = [ - np.nan if i == "-" else float(i) for i in measurement_values - ] + measurement_values = [np.nan if i == "-" else float(i) for i in measurement_values] assert len(measurement_values) == len( # noqa:S101 self.timesteps diff --git a/wetterdienst/provider/dwd/forecast/api.py b/wetterdienst/provider/dwd/forecast/api.py index 61b009564..5e00bf38a 100644 --- a/wetterdienst/provider/dwd/forecast/api.py +++ b/wetterdienst/provider/dwd/forecast/api.py @@ -65,7 +65,7 @@ class DwdMosmixValues(ScalarValuesCore): parameter: List - If None, data for all parameters is returned. - If not None, list of parameters, per MOSMIX definition, see - https://www.dwd.de/DE/leistungen/opendata/help/schluessel_datenformate/kml/mosmix_elemente_pdf.pdf?__blob=publicationFile&v=2 # noqa:E501,B950 + https://www.dwd.de/DE/leistungen/opendata/help/schluessel_datenformate/kml/mosmix_elemente_pdf.pdf?__blob=publicationFile&v=2 # noqa:B950 """ _tz = Timezone.GERMANY @@ -86,9 +86,7 @@ def _create_humanized_parameters_mapping(self) -> Dict[str, str]: """ hcnm = { parameter.value: parameter.name.lower() - for parameter in self.stations.stations._parameter_base[ - self.stations.stations.mosmix_type.name - ] + for parameter in self.stations.stations._parameter_base[self.stations.stations.mosmix_type.name] } return hcnm @@ -197,9 +195,7 @@ def _tidy_up_df(self, df: pd.DataFrame, dataset: Enum) -> pd.DataFrame: return df_tidy - def read_mosmix( - self, date: Union[datetime, DwdForecastDate] - ) -> Generator[pd.DataFrame, None, None]: + def read_mosmix(self, date: Union[datetime, DwdForecastDate]) -> Generator[pd.DataFrame, None, None]: """ Manage data acquisition for a given date that is used to filter the found files on the MOSMIX path of the DWD server. @@ -217,9 +213,7 @@ def read_mosmix( yield df_forecast - def _read_mosmix( - self, date: Union[DwdForecastDate, datetime] - ) -> Generator[pd.DataFrame, None, None]: + def _read_mosmix(self, date: Union[DwdForecastDate, datetime]) -> Generator[pd.DataFrame, None, None]: """ Wrapper that either calls read_mosmix_s or read_mosmix_l depending on defined period type @@ -232,9 +226,7 @@ def _read_mosmix( else: yield from self.read_mosmix_large(date) - def read_mosmix_small( - self, date: Union[DwdForecastDate, datetime] - ) -> Generator[pd.DataFrame, None, None]: + def read_mosmix_small(self, date: Union[DwdForecastDate, datetime]) -> Generator[pd.DataFrame, None, None]: """ Reads single MOSMIX-S file with all stations and returns every forecast that matches with one of the defined station ids. @@ -338,10 +330,7 @@ class DwdMosmixRequest(ScalarRequestCore): _dataset_base = DwdMosmixDataset _unit_tree = DwdMosmixUnit - _url = ( - "https://www.dwd.de/DE/leistungen/met_verfahren_mosmix/" - "mosmix_stationskatalog.cfg?view=nasPublication" - ) + _url = "https://www.dwd.de/DE/leistungen/met_verfahren_mosmix/" "mosmix_stationskatalog.cfg?view=nasPublication" _colspecs = [ (0, 5), @@ -416,9 +405,7 @@ def __init__( self, parameter: Optional[Tuple[Union[str, DwdMosmixParameter], ...]], mosmix_type: Union[str, DwdMosmixType], - start_issue: Optional[ - Union[str, datetime, DwdForecastDate] - ] = DwdForecastDate.LATEST, + start_issue: Optional[Union[str, datetime, DwdForecastDate]] = DwdForecastDate.LATEST, end_issue: Optional[Union[str, datetime]] = None, start_date: Optional[Union[str, datetime]] = None, end_date: Optional[Union[str, datetime]] = None, @@ -460,9 +447,7 @@ def __init__( # Parse issue date if not set to fixed "latest" string if start_issue is DwdForecastDate.LATEST and end_issue: - log.info( - "end_issue will be ignored as 'latest' was selected for issue date" - ) + log.info("end_issue will be ignored as 'latest' was selected for issue date") if start_issue is not DwdForecastDate.LATEST: if not start_issue and not end_issue: @@ -472,12 +457,8 @@ def __init__( elif not start_issue: start_issue = end_issue - start_issue = pd.to_datetime(start_issue, infer_datetime_format=True).floor( - "1H" - ) - end_issue = pd.to_datetime(end_issue, infer_datetime_format=True).floor( - "1H" - ) + start_issue = pd.to_datetime(start_issue, infer_datetime_format=True).floor("1H") + end_issue = pd.to_datetime(end_issue, infer_datetime_format=True).floor("1H") # Shift start date and end date to 3, 9, 15, 21 hour format if mosmix_type == DwdMosmixType.LARGE: @@ -525,11 +506,7 @@ def _all(self) -> pd.DataFrame: dtype="str", ) - df = df[ - (df.iloc[:, 0] != "=====") - & (df.iloc[:, 0] != "TABLE") - & (df.iloc[:, 0] != "clu") - ] + df = df[(df.iloc[:, 0] != "=====") & (df.iloc[:, 0] != "TABLE") & (df.iloc[:, 0] != "clu")] df = df.iloc[:, [2, 3, 4, 5, 6, 7]] @@ -543,13 +520,9 @@ def _all(self) -> pd.DataFrame: ] # Convert coordinates from degree minutes to decimal degrees - df[Columns.LATITUDE.value] = ( - df[Columns.LATITUDE.value].astype(float).apply(convert_dm_to_dd) - ) + df[Columns.LATITUDE.value] = df[Columns.LATITUDE.value].astype(float).apply(convert_dm_to_dd) - df[Columns.LONGITUDE.value] = ( - df[Columns.LONGITUDE.value].astype(float).apply(convert_dm_to_dd) - ) + df[Columns.LONGITUDE.value] = df[Columns.LONGITUDE.value].astype(float).apply(convert_dm_to_dd) df = df.reindex(columns=self._base_columns) diff --git a/wetterdienst/provider/dwd/index.py b/wetterdienst/provider/dwd/index.py index 79ea2c2c1..6029e8319 100644 --- a/wetterdienst/provider/dwd/index.py +++ b/wetterdienst/provider/dwd/index.py @@ -46,9 +46,7 @@ def _create_file_index_for_dwd_server( recursive = False files_server = list_remote_files_fsspec(url, recursive=recursive) - files_server = pd.DataFrame( - files_server, columns=[DwdColumns.FILENAME.value], dtype="str" - ) + files_server = pd.DataFrame(files_server, columns=[DwdColumns.FILENAME.value], dtype="str") return files_server diff --git a/wetterdienst/provider/dwd/observation/api.py b/wetterdienst/provider/dwd/observation/api.py index 758b3cf8d..95915d939 100644 --- a/wetterdienst/provider/dwd/observation/api.py +++ b/wetterdienst/provider/dwd/observation/api.py @@ -88,9 +88,7 @@ def _datetime_format(self): :return: """ - return RESOLUTION_TO_DATETIME_FORMAT_MAPPING.get( - self.stations.stations.resolution - ) + return RESOLUTION_TO_DATETIME_FORMAT_MAPPING.get(self.stations.stations.resolution) def __eq__(self, other): """ @@ -99,8 +97,7 @@ def __eq__(self, other): :return: """ return super(DwdObservationValues, self).__eq__(other) and ( - self.stations.resolution == other.stations.resolution - and self.stations.period == other.stations.period + self.stations.resolution == other.stations.resolution and self.stations.period == other.stations.period ) def __str__(self): @@ -108,9 +105,7 @@ def __str__(self): :return: """ - periods_joined = "& ".join( - [period_type.value for period_type in self.stations.period] - ) + periods_joined = "& ".join([period_type.value for period_type in self.stations.period]) return ", ".join( [ @@ -139,10 +134,7 @@ def _collect_station_parameter( periods_and_date_ranges = [] for period in self.stations.period: - if ( - self.stations.resolution in HIGH_RESOLUTIONS - and period == Period.HISTORICAL - ): + if self.stations.resolution in HIGH_RESOLUTIONS and period == Period.HISTORICAL: date_ranges = self._get_historical_date_ranges(station_id, dataset) for date_range in date_ranges: periods_and_date_ranges.append((period, date_range)) @@ -158,12 +150,9 @@ def _collect_station_parameter( log.info(f"Acquiring observations data for {parameter_identifier}.") - if not check_dwd_observations_dataset( - dataset, self.stations.resolution, period - ): + if not check_dwd_observations_dataset(dataset, self.stations.resolution, period): log.info( - f"Invalid combination {dataset.value}/" - f"{self.stations.resolution.value}/{period} is skipped." + f"Invalid combination {dataset.value}/" f"{self.stations.resolution.value}/{period} is skipped." ) continue @@ -180,26 +169,16 @@ def _collect_station_parameter( station_id, date_range, ) - log.info( - f"No files found for {parameter_identifier}. Station will be skipped." - ) + log.info(f"No files found for {parameter_identifier}. Station will be skipped.") continue - filenames_and_files = download_climate_observations_data_parallel( - remote_files - ) + filenames_and_files = download_climate_observations_data_parallel(remote_files) - period_df = parse_climate_observations_data( - filenames_and_files, dataset, self.stations.resolution, period - ) + period_df = parse_climate_observations_data(filenames_and_files, dataset, self.stations.resolution, period) # Filter out values which already are in the DataFrame try: - period_df = period_df[ - ~period_df[DwdColumns.DATE.value].isin( - parameter_df[DwdColumns.DATE.value] - ) - ] + period_df = period_df[~period_df[DwdColumns.DATE.value].isin(parameter_df[DwdColumns.DATE.value])] except KeyError: pass @@ -238,20 +217,20 @@ def _tidy_up_df(self, df: pd.DataFrame, dataset) -> pd.DataFrame: """ Implementation of _tidy_up_df for DWD Observations - :param df: - :param dataset: - :return: + :param df: untidy DataFrame + :param dataset: dataset enumeration + :return: tidied DataFrame """ droppable_columns = [ # Hourly # Cloud type - DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER1_ABBREVIATION.value, # noqa:E501 - DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER2_ABBREVIATION.value, # noqa:E501 - DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER3_ABBREVIATION.value, # noqa:E501 - DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER4_ABBREVIATION.value, # noqa:E501 + DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER1_ABBREVIATION.value, + DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER2_ABBREVIATION.value, + DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER3_ABBREVIATION.value, + DwdObservationDatasetTree.HOURLY.CLOUD_TYPE.CLOUD_TYPE_LAYER4_ABBREVIATION.value, # Cloudiness - DwdObservationDatasetTree.HOURLY.CLOUDINESS.CLOUD_COVER_TOTAL_INDICATOR.value, # noqa:E501 + DwdObservationDatasetTree.HOURLY.CLOUDINESS.CLOUD_COVER_TOTAL_INDICATOR.value, # Solar DwdObservationDatasetTree.HOURLY.SOLAR.END_OF_INTERVAL.value, DwdObservationDatasetTree.HOURLY.SOLAR.TRUE_LOCAL_TIME.value, @@ -267,41 +246,29 @@ def _tidy_up_df(self, df: pd.DataFrame, dataset) -> pd.DataFrame: resolution = self.stations.stations.resolution - if ( - resolution == Resolution.DAILY - and dataset == DwdObservationDataset.CLIMATE_SUMMARY - ): - quality_wind = df.pop( - DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.QUALITY_WIND.value - ) - quality_general = df.pop( - DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.QUALITY_GENERAL.value - ) - - quality = pd.concat( - [ - pd.Series(repeat(quality_wind.tolist(), 2)).explode(), - pd.Series(repeat(quality_general.tolist(), 12)).explode(), - ] - ) + if dataset == DwdObservationDataset.CLIMATE_SUMMARY: + if resolution == Resolution.DAILY: + quality_wind = df.pop(DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.QUALITY_WIND.value) + quality_general = df.pop(DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.QUALITY_GENERAL.value) - elif ( - resolution in (Resolution.MONTHLY, Resolution.ANNUAL) - and dataset == DwdObservationDataset.CLIMATE_SUMMARY - ): - quality_general = df.pop( - DwdObservationDatasetTree.MONTHLY.CLIMATE_SUMMARY.QUALITY_GENERAL.value - ) - quality_precipitation = df.pop( - DwdObservationDatasetTree.MONTHLY.CLIMATE_SUMMARY.QUALITY_PRECIPITATION.value # noqa: E501 - ) - quality = pd.concat( - [ - pd.Series(repeat(quality_general, 9)).explode(), - pd.Series(repeat(quality_precipitation, 2)).explode(), - ] - ) + quality = pd.concat( + [ + pd.Series(repeat(quality_wind.tolist(), 2)).explode(), + pd.Series(repeat(quality_general.tolist(), 12)).explode(), + ] + ) + elif resolution in (Resolution.MONTHLY, Resolution.ANNUAL): + quality_general = df.pop(DwdObservationDatasetTree.MONTHLY.CLIMATE_SUMMARY.QUALITY_GENERAL.value) + quality_precipitation = df.pop( + DwdObservationDatasetTree.MONTHLY.CLIMATE_SUMMARY.QUALITY_PRECIPITATION.value + ) + quality = pd.concat( + [ + pd.Series(repeat(quality_general, 9)).explode(), + pd.Series(repeat(quality_precipitation, 2)).explode(), + ] + ) else: quality = df.pop(df.columns[2]) quality = pd.Series(repeat(quality, df.shape[1])).explode() @@ -333,9 +300,7 @@ def _coerce_dates(self, series: pd.Series, timezone_: timezone) -> pd.Series: :param series: :return: """ - return pd.to_datetime(series, format=self._datetime_format).dt.tz_localize( - self.data_tz - ) + return pd.to_datetime(series, format=self._datetime_format).dt.tz_localize(self.data_tz) def _coerce_irregular_parameter(self, series: pd.Series) -> pd.Series: """ @@ -361,9 +326,7 @@ def _create_humanized_parameters_mapping(self) -> Dict[str, str]: return hcnm - def _get_historical_date_ranges( - self, station_id: str, dataset: DwdObservationDataset - ) -> List[str]: + def _get_historical_date_ranges(self, station_id: str, dataset: DwdObservationDataset) -> List[str]: """ Get particular files for historical data which for high resolution is released in data chunks e.g. decades or monthly chunks @@ -372,9 +335,7 @@ def _get_historical_date_ranges( :param dataset: :return: """ - file_index = create_file_index_for_climate_observations( - dataset, self.stations.resolution, Period.HISTORICAL - ) + file_index = create_file_index_for_climate_observations(dataset, self.stations.resolution, Period.HISTORICAL) file_index = file_index[(file_index[Columns.STATION_ID.value] == station_id)] @@ -456,14 +417,10 @@ def _historical_interval(self) -> pd.Interval: # historical_begin = historical_end + pd.tseries.offsets.DateOffset(years=-1) # a year that is way before any data is collected - historical_begin = pd.Timestamp( - datetime(year=1678, month=1, day=1) - ).tz_localize(historical_end.tz) + historical_begin = pd.Timestamp(datetime(year=1678, month=1, day=1)).tz_localize(historical_end.tz) # TODO: Use date - historical_interval = pd.Interval( - left=historical_begin, right=historical_end, closed="both" - ) + historical_interval = pd.Interval(left=historical_begin, right=historical_end, closed="both") return historical_interval @@ -479,9 +436,7 @@ def _recent_interval(self) -> pd.Interval: recent_begin = recent_end - pd.Timedelta(days=500) # TODO: use date - recent_interval = pd.Interval( - left=recent_begin, right=recent_end, closed="both" - ) + recent_interval = pd.Interval(left=recent_begin, right=recent_end, closed="both") return recent_interval @@ -567,10 +522,7 @@ def __init__( ) if self.start_date and self.period: - log.warning( - f"start_date and end_date filtering limited to defined " - f"periods {self.period}" - ) + log.warning(f"start_date and end_date filtering limited to defined " f"periods {self.period}") # Has to follow the super call as start date and end date are required for getting # automated periods from overlapping intervals @@ -591,9 +543,7 @@ def describe_fields(cls, dataset, resolution, period, language: str = "en") -> d :return: """ dataset = parse_enumeration_from_template(dataset, DwdObservationDataset) - resolution = parse_enumeration_from_template( - resolution, cls._resolution_base, Resolution - ) + resolution = parse_enumeration_from_template(resolution, cls._resolution_base, Resolution) period = parse_enumeration_from_template(period, cls._period_base, Period) file_index = _create_file_index_for_dwd_server( @@ -610,9 +560,7 @@ def describe_fields(cls, dataset, resolution, period, language: str = "en") -> d else: raise ValueError("Only language 'en' or 'de' supported") - file_index = file_index[ - file_index[DwdColumns.FILENAME.value].str.contains(file_prefix) - ] + file_index = file_index[file_index[DwdColumns.FILENAME.value].str.contains(file_prefix)] description_file_url = str(file_index[DwdColumns.FILENAME.value].tolist()[0]) log.info(f"Acquiring field information from {description_file_url}") @@ -638,38 +586,23 @@ def _all(self) -> pd.DataFrame: for period in reversed(self.period): if not check_dwd_observations_dataset(dataset, self.resolution, period): log.warning( - f"The combination of {dataset.value}, " - f"{self.resolution.value}, {period.value} is invalid." + f"The combination of {dataset.value}, " f"{self.resolution.value}, {period.value} is invalid." ) continue - df = create_meta_index_for_climate_observations( - dataset, self.resolution, period - ) + df = create_meta_index_for_climate_observations(dataset, self.resolution, period) - file_index = create_file_index_for_climate_observations( - dataset, self.resolution, period - ) + file_index = create_file_index_for_climate_observations(dataset, self.resolution, period) - df = df[ - df.loc[:, Columns.STATION_ID.value].isin( - file_index[Columns.STATION_ID.value] - ) - ] + df = df[df.loc[:, Columns.STATION_ID.value].isin(file_index[Columns.STATION_ID.value])] if not stations_df.empty: - df = df[ - ~df[Columns.STATION_ID.value].isin( - stations_df[Columns.STATION_ID.value] - ) - ] + df = df[~df[Columns.STATION_ID.value].isin(stations_df[Columns.STATION_ID.value])] stations_df = stations_df.append(df) if not stations_df.empty: - stations_df = stations_df.sort_values( - [Columns.STATION_ID.value], key=lambda x: x.astype(int) - ) + stations_df = stations_df.sort_values([Columns.STATION_ID.value], key=lambda x: x.astype(int)) return stations_df diff --git a/wetterdienst/provider/dwd/observation/download.py b/wetterdienst/provider/dwd/observation/download.py index 644690bdd..5092bf219 100644 --- a/wetterdienst/provider/dwd/observation/download.py +++ b/wetterdienst/provider/dwd/observation/download.py @@ -53,9 +53,7 @@ def __download_climate_observations_data(remote_file: str) -> bytes: try: zip_file = download_file(remote_file, ttl=CacheExpiry.FIVE_MINUTES) except InvalidURL as e: - raise InvalidURL( - f"Error: the station data {remote_file} could not be reached." - ) from e + raise InvalidURL(f"Error: the station data {remote_file} could not be reached.") from e except Exception: raise FailedDownload(f"Download failed for {remote_file}") @@ -75,9 +73,7 @@ def __download_climate_observations_data(remote_file: str) -> bytes: return file_in_bytes # If whatsoever no file was found and returned already throw exception - raise ProductFileNotFound( - f"The archive of {remote_file} does not hold a 'produkt' file." - ) + raise ProductFileNotFound(f"The archive of {remote_file} does not hold a 'produkt' file.") except BadZipFile as e: raise BadZipFile(f"The archive of {remote_file} seems to be corrupted.") from e diff --git a/wetterdienst/provider/dwd/observation/fields.py b/wetterdienst/provider/dwd/observation/fields.py index c42434592..637339082 100644 --- a/wetterdienst/provider/dwd/observation/fields.py +++ b/wetterdienst/provider/dwd/observation/fields.py @@ -53,9 +53,7 @@ def parse_parameters(text): # Remove some anomaly. more = more.replace("0\n1\n", "1\n") # Replace newlines after digits with "-". - more = re.sub( - r"^(\d+)\n(.*)", r"\g<1>- \g<2>", more, flags=re.MULTILINE - ) + more = re.sub(r"^(\d+)\n(.*)", r"\g<1>- \g<2>", more, flags=re.MULTILINE) # Remove all newlines _within_ text descriptions, per item. more = re.sub(r"\n(?!\d+)", " ", more, flags=re.DOTALL) else: @@ -98,9 +96,7 @@ def read_description(url, language: str = "en") -> dict: data["parameters"] = parse_parameters(parameters_text) # Read "Quality information" section. - data["quality_information"] = parse_section( - document, sections["quality_information"] - ) + data["quality_information"] = parse_section(document, sections["quality_information"]) return data @@ -117,9 +113,18 @@ def process(url) -> None: # pragma: no cover if __name__ == "__main__": # pragma: no cover - ten_minutes_air = "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/10_minutes/air_temperature/recent/DESCRIPTION_obsgermany_climate_10min_tu_recent_en.pdf" # noqa:E501,B950 - hourly_solar = "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/solar/DESCRIPTION_obsgermany_climate_hourly_solar_en.pdf" # noqa:E501,B950 - daily_kl = "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/kl/recent/DESCRIPTION_obsgermany_climate_daily_kl_recent_en.pdf" # noqa:E501,B950 + ten_minutes_air = ( + "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/10_minutes/" + "air_temperature/recent/DESCRIPTION_obsgermany_climate_10min_tu_recent_en.pdf" + ) + hourly_solar = ( + "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/" + "solar/DESCRIPTION_obsgermany_climate_hourly_solar_en.pdf" + ) + daily_kl = ( + "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/daily/" + "kl/recent/DESCRIPTION_obsgermany_climate_daily_kl_recent_en.pdf" + ) for item in ten_minutes_air, hourly_solar, daily_kl: print(item) # noqa: T001 diff --git a/wetterdienst/provider/dwd/observation/fileindex.py b/wetterdienst/provider/dwd/observation/fileindex.py index d6a019cdc..bc39b2843 100644 --- a/wetterdienst/provider/dwd/observation/fileindex.py +++ b/wetterdienst/provider/dwd/observation/fileindex.py @@ -68,13 +68,9 @@ def create_file_index_for_climate_observations( Returns: file index in a pandas.DataFrame with sets of parameters and station id """ - file_index = _create_file_index_for_dwd_server( - parameter_set, resolution, period, DWDCDCBase.CLIMATE_OBSERVATIONS - ) + file_index = _create_file_index_for_dwd_server(parameter_set, resolution, period, DWDCDCBase.CLIMATE_OBSERVATIONS) - file_index = file_index.loc[ - file_index[DwdColumns.FILENAME.value].str.endswith(Extension.ZIP.value), : - ] + file_index = file_index.loc[file_index[DwdColumns.FILENAME.value].str.endswith(Extension.ZIP.value), :] file_index.loc[:, DwdColumns.STATION_ID.value] = ( file_index[DwdColumns.FILENAME.value].str.findall(STATION_ID_REGEX).str[0] @@ -82,9 +78,7 @@ def create_file_index_for_climate_observations( file_index = file_index.dropna().reset_index(drop=True) - file_index.loc[:, DwdColumns.STATION_ID.value] = file_index[ - DwdColumns.STATION_ID.value - ].astype(str) + file_index.loc[:, DwdColumns.STATION_ID.value] = file_index[DwdColumns.STATION_ID.value].astype(str) if resolution in HIGH_RESOLUTIONS and period == Period.HISTORICAL: # Date range string for additional filtering of historical files @@ -111,17 +105,12 @@ def create_file_index_for_climate_observations( # Temporary fix for filenames with wrong ordered/faulty dates # Fill those cases with minimum/maximum date to ensure that they are loaded as # we don't know what exact date range the included data has - wrong_date_order_index = ( - file_index[DwdColumns.FROM_DATE.value] - > file_index[DwdColumns.TO_DATE.value] - ) + wrong_date_order_index = file_index[DwdColumns.FROM_DATE.value] > file_index[DwdColumns.TO_DATE.value] file_index.loc[wrong_date_order_index, DwdColumns.FROM_DATE.value] = file_index[ DwdColumns.FROM_DATE.value ].min() - file_index.loc[wrong_date_order_index, DwdColumns.TO_DATE.value] = file_index[ - DwdColumns.TO_DATE.value - ].max() + file_index.loc[wrong_date_order_index, DwdColumns.TO_DATE.value] = file_index[DwdColumns.TO_DATE.value].max() file_index.loc[:, DwdColumns.INTERVAL.value] = file_index.apply( lambda x: pd.Interval( @@ -132,8 +121,6 @@ def create_file_index_for_climate_observations( axis=1, ) - file_index = file_index.sort_values( - by=[DwdColumns.STATION_ID.value, DwdColumns.FILENAME.value] - ) + file_index = file_index.sort_values(by=[DwdColumns.STATION_ID.value, DwdColumns.FILENAME.value]) return file_index diff --git a/wetterdienst/provider/dwd/observation/metadata/dataset.py b/wetterdienst/provider/dwd/observation/metadata/dataset.py index 9b4dd9663..06f691ed5 100644 --- a/wetterdienst/provider/dwd/observation/metadata/dataset.py +++ b/wetterdienst/provider/dwd/observation/metadata/dataset.py @@ -42,9 +42,7 @@ class DwdObservationDataset(Enum): WEATHER_PHENOMENA = "weather_phenomena" -RESOLUTION_DATASET_MAPPING: Dict[ - Resolution, Dict[DwdObservationDataset, List[Period]] -] = { +RESOLUTION_DATASET_MAPPING: Dict[Resolution, Dict[DwdObservationDataset, List[Period]]] = { Resolution.MINUTE_1: { DwdObservationDataset.PRECIPITATION: [ Period.HISTORICAL, diff --git a/wetterdienst/provider/dwd/observation/metadata/parameter.py b/wetterdienst/provider/dwd/observation/metadata/parameter.py index b344842ec..ad5722998 100644 --- a/wetterdienst/provider/dwd/observation/metadata/parameter.py +++ b/wetterdienst/provider/dwd/observation/metadata/parameter.py @@ -420,9 +420,7 @@ class SOLAR(Enum): RADIATION_GLOBAL = "fg_lberg" SUNSHINE_DURATION = "sd_lberg" SUN_ZENITH_ANGLE = "zenit" - TRUE_LOCAL_TIME = ( - "true_local_time" # original name was adjusted to this one - ) + TRUE_LOCAL_TIME = "true_local_time" # original name was adjusted to this one # sun class SUNSHINE_DURATION(Enum): diff --git a/wetterdienst/provider/dwd/observation/metaindex.py b/wetterdienst/provider/dwd/observation/metaindex.py index 750251a33..a385930c6 100644 --- a/wetterdienst/provider/dwd/observation/metaindex.py +++ b/wetterdienst/provider/dwd/observation/metaindex.py @@ -91,9 +91,7 @@ def create_meta_index_for_climate_observations( if cond: meta_index = _create_meta_index_for_1minute_historical_precipitation() else: - meta_index = _create_meta_index_for_climate_observations( - parameter_set, resolution, period - ) + meta_index = _create_meta_index_for_climate_observations(parameter_set, resolution, period) # If no state column available, take state information from daily historical # precipitation @@ -162,9 +160,9 @@ def _create_meta_index_for_climate_observations( ) # Fix column names, as header is not aligned to fixed column widths - meta_index.columns = "".join( - [column for column in meta_index.columns if "unnamed" not in column.lower()] - ).split(" ") + meta_index.columns = "".join([column for column in meta_index.columns if "unnamed" not in column.lower()]).split( + " " + ) meta_index = meta_index.rename(columns=str.lower) @@ -205,9 +203,7 @@ def _create_meta_index_for_1minute_historical_precipitation() -> pd.DataFrame: """ - parameter_path = ( - f"{Resolution.MINUTE_1.value}/" f"{DwdObservationDataset.PRECIPITATION.value}/" - ) + parameter_path = f"{Resolution.MINUTE_1.value}/" f"{DwdObservationDataset.PRECIPITATION.value}/" url = reduce( urljoin, @@ -222,21 +218,15 @@ def _create_meta_index_for_1minute_historical_precipitation() -> pd.DataFrame: metadata_file_paths = list_remote_files_fsspec(url, recursive=False) - station_ids = [ - re.findall(STATION_ID_REGEX, file).pop(0) for file in metadata_file_paths - ] + station_ids = [re.findall(STATION_ID_REGEX, file).pop(0) for file in metadata_file_paths] meta_index_df = pd.DataFrame(columns=METADATA_COLUMNS) with ThreadPoolExecutor() as executor: - metadata_files = executor.map( - _download_metadata_file_for_1minute_precipitation, metadata_file_paths - ) + metadata_files = executor.map(_download_metadata_file_for_1minute_precipitation, metadata_file_paths) with ThreadPoolExecutor() as executor: - metadata_dfs = executor.map( - _parse_geo_metadata, zip(metadata_files, station_ids) - ) + metadata_dfs = executor.map(_parse_geo_metadata, zip(metadata_files, station_ids)) meta_index_df = meta_index_df.append(other=list(metadata_dfs), ignore_index=True) @@ -250,9 +240,7 @@ def _create_meta_index_for_1minute_historical_precipitation() -> pd.DataFrame: meta_index_df = meta_index_df.drop(labels=Columns.STATE.value, axis=1) # Make station id str - meta_index_df[Columns.STATION_ID.value] = meta_index_df[ - Columns.STATION_ID.value - ].str.pad(5, "left", "0") + meta_index_df[Columns.STATION_ID.value] = meta_index_df[Columns.STATION_ID.value].str.pad(5, "left", "0") return meta_index_df @@ -282,9 +270,7 @@ def _download_metadata_file_for_1minute_precipitation(metadata_file: str) -> Byt return file -def _parse_geo_metadata( - metadata_file_and_station_id: Tuple[BytesIO, str] -) -> pd.DataFrame: +def _parse_geo_metadata(metadata_file_and_station_id: Tuple[BytesIO, str]) -> pd.DataFrame: """A function that analysis the given file (bytes) and extracts geography of 1minute metadata zip and catches the relevant information and create a similar file to those that can usually be found already prepared for other @@ -311,9 +297,7 @@ def _parse_geo_metadata( metadata_geo_df = metadata_geo_df.rename(columns=GERMAN_TO_ENGLISH_COLUMNS_MAPPING) - metadata_geo_df[Columns.FROM_DATE.value] = metadata_geo_df.loc[ - 0, Columns.FROM_DATE.value - ] + metadata_geo_df[Columns.FROM_DATE.value] = metadata_geo_df.loc[0, Columns.FROM_DATE.value] metadata_geo_df = metadata_geo_df.iloc[[-1], :] diff --git a/wetterdienst/provider/dwd/observation/parser.py b/wetterdienst/provider/dwd/observation/parser.py index 90c08f92d..2001a5fca 100644 --- a/wetterdienst/provider/dwd/observation/parser.py +++ b/wetterdienst/provider/dwd/observation/parser.py @@ -25,13 +25,11 @@ # Parameter names used to create full 1 minute precipitation dataset wherever those # columns are missing (which is the case for non historical data) PRECIPITATION_PARAMETERS = ( - DwdObservationDatasetTree.MINUTE_1.PRECIPITATION.PRECIPITATION_HEIGHT_DROPLET.value, # Noqa: E501, B950 - DwdObservationDatasetTree.MINUTE_1.PRECIPITATION.PRECIPITATION_HEIGHT_ROCKER.value, # Noqa: E501, B950 + DwdObservationDatasetTree.MINUTE_1.PRECIPITATION.PRECIPITATION_HEIGHT_DROPLET.value, + DwdObservationDatasetTree.MINUTE_1.PRECIPITATION.PRECIPITATION_HEIGHT_ROCKER.value, ) -PRECIPITATION_MINUTE_1_QUALITY = ( - DwdObservationDatasetTree.MINUTE_1.PRECIPITATION.QUALITY -) +PRECIPITATION_MINUTE_1_QUALITY = DwdObservationDatasetTree.MINUTE_1.PRECIPITATION.QUALITY def parse_climate_observations_data( @@ -87,17 +85,13 @@ def _parse_climate_observations_data( try: df = pd.read_csv( - filepath_or_buffer=BytesIO( - file.read().replace(b" ", b"") - ), # prevent leading/trailing whitespace + filepath_or_buffer=BytesIO(file.read().replace(b" ", b"")), # prevent leading/trailing whitespace sep=STATION_DATA_SEP, dtype="str", na_values=NA_STRING, ) except pd.errors.ParserError: - log.warning( - f"The file representing {filename} could not be parsed and is skipped." - ) + log.warning(f"The file representing {filename} could not be parsed and is skipped.") return pd.DataFrame() except ValueError: log.warning(f"The file representing {filename} is None and is skipped.") @@ -118,24 +112,17 @@ def _parse_climate_observations_data( # information. Also rename column with true local time to english one df = df.rename( columns={ - "mess_datum_woz": ( - DwdObservationDatasetTree.HOURLY.SOLAR.TRUE_LOCAL_TIME.value # Noqa: E501, B950 - ), + "mess_datum_woz": DwdObservationDatasetTree.HOURLY.SOLAR.TRUE_LOCAL_TIME.value, } ) # Duplicate the date column to end of interval column - df[DwdObservationDatasetTree.HOURLY.SOLAR.END_OF_INTERVAL.value] = df[ - DwdOrigColumns.DATE.value - ] + df[DwdObservationDatasetTree.HOURLY.SOLAR.END_OF_INTERVAL.value] = df[DwdOrigColumns.DATE.value] # Fix real date column by cutting of minutes df[DwdOrigColumns.DATE.value] = df[DwdOrigColumns.DATE.value].str[:-3] - if ( - resolution == Resolution.MINUTE_1 - and dataset == DwdObservationDataset.PRECIPITATION - ): + if resolution == Resolution.MINUTE_1 and dataset == DwdObservationDataset.PRECIPITATION: # Need to unfold historical data, as it is encoded in its run length e.g. # from time X to time Y precipitation is 0 if period == Period.HISTORICAL: diff --git a/wetterdienst/provider/dwd/observation/util/parameter.py b/wetterdienst/provider/dwd/observation/util/parameter.py index 39c94fb69..b79168949 100644 --- a/wetterdienst/provider/dwd/observation/util/parameter.py +++ b/wetterdienst/provider/dwd/observation/util/parameter.py @@ -22,34 +22,22 @@ def create_parameter_to_dataset_combination( parameter: Union[DwdObservationParameter, DwdObservationDataset], resolution: Resolution, -) -> Tuple[ - Union[DwdObservationParameter, DwdObservationDataset], - DwdObservationDataset, -]: +) -> Tuple[Union[DwdObservationParameter, DwdObservationDataset], DwdObservationDataset,]: """Function to create a mapping from a requested parameter to a provided parameter set which has to be downloaded first to extract the parameter from it""" try: - parameter_ = parse_enumeration_from_template( - parameter, DwdObservationParameter[resolution.name] - ) + parameter_ = parse_enumeration_from_template(parameter, DwdObservationParameter[resolution.name]) parameter = PARAMETER_TO_DATASET_MAPPING[resolution][parameter_] - return parameter, parse_enumeration_from_template( - parameter.__class__.__name__, DwdObservationDataset - ) + return parameter, parse_enumeration_from_template(parameter.__class__.__name__, DwdObservationDataset) except (KeyError, InvalidEnumeration): try: - parameter_set = parse_enumeration_from_template( - parameter, DwdObservationDataset - ) + parameter_set = parse_enumeration_from_template(parameter, DwdObservationDataset) return parameter_set, parameter_set except InvalidEnumeration: - raise InvalidParameter( - f"parameter {parameter} could not be parsed for " - f"time resolution {resolution}" - ) + raise InvalidParameter(f"parameter {parameter} could not be parsed for " f"time resolution {resolution}") def check_dwd_observations_dataset( diff --git a/wetterdienst/provider/dwd/radar/access.py b/wetterdienst/provider/dwd/radar/access.py index 04036e8c9..3f1425ea2 100644 --- a/wetterdienst/provider/dwd/radar/access.py +++ b/wetterdienst/provider/dwd/radar/access.py @@ -129,9 +129,7 @@ def collect_radar_data( results = [] for period in period_types: - file_index = create_fileindex_radolan_cdc( - resolution=resolution, period=period - ) + file_index = create_fileindex_radolan_cdc(resolution=resolution, period=period) # Filter for dates range if start_date and end_date are defined. if period == Period.RECENT: @@ -143,14 +141,8 @@ def collect_radar_data( # This is for matching historical data, e.g. "RW-200509.tar.gz". else: file_index = file_index[ - ( - file_index[DwdColumns.DATETIME.value].dt.year - == start_date.year - ) - & ( - file_index[DwdColumns.DATETIME.value].dt.month - == start_date.month - ) + (file_index[DwdColumns.DATETIME.value].dt.year == start_date.year) + & (file_index[DwdColumns.DATETIME.value].dt.month == start_date.month) ] results.append(file_index) @@ -321,9 +313,7 @@ def _download_radolan_data(remote_radolan_filepath: str) -> BytesIO: return download_file(remote_radolan_filepath, ttl=CacheExpiry.TWELVE_HOURS) -def _extract_radolan_data( - date_time: datetime, archive_in_bytes: BytesIO -) -> RadarResult: +def _extract_radolan_data(date_time: datetime, archive_in_bytes: BytesIO) -> RadarResult: """ Function used to extract RADOLAN_CDC file for the requested datetime from the downloaded archive. @@ -356,14 +346,10 @@ def _extract_radolan_data( filename=file.name, ) - raise FileNotFoundError( - f"RADOLAN file for {date_time_string} not found." - ) # pragma: no cover + raise FileNotFoundError(f"RADOLAN file for {date_time_string} not found.") # pragma: no cover except EOFError as ex: - raise FailedDownload( - f"RADOLAN file for {date_time_string} is invalid: {ex}" - ) # pragma: no cover + raise FailedDownload(f"RADOLAN file for {date_time_string} is invalid: {ex}") # pragma: no cover # Otherwise if there's an error the data is from recent time period and only has to # be unpacked once @@ -372,6 +358,4 @@ def _extract_radolan_data( archive_in_bytes.seek(0) with gzip.GzipFile(fileobj=archive_in_bytes, mode="rb") as gz_file: - return RadarResult( - data=BytesIO(gz_file.read()), timestamp=date_time, filename=gz_file.name - ) + return RadarResult(data=BytesIO(gz_file.read()), timestamp=date_time, filename=gz_file.name) diff --git a/wetterdienst/provider/dwd/radar/api.py b/wetterdienst/provider/dwd/radar/api.py index 2d05ae277..445c20f00 100644 --- a/wetterdienst/provider/dwd/radar/api.py +++ b/wetterdienst/provider/dwd/radar/api.py @@ -75,12 +75,8 @@ def __init__( self.format = parse_enumeration_from_template(fmt, DwdRadarDataFormat) self.subset = parse_enumeration_from_template(subset, DwdRadarDataSubset) self.elevation = elevation and int(elevation) - self.resolution: Resolution = parse_enumeration_from_template( - resolution, DwdRadarResolution, Resolution - ) - self.period: Period = parse_enumeration_from_template( - period, DwdRadarPeriod, Period - ) + self.resolution: Resolution = parse_enumeration_from_template(resolution, DwdRadarResolution, Resolution) + self.period: Period = parse_enumeration_from_template(period, DwdRadarPeriod, Period) # Sanity checks. if self.parameter == DwdRadarParameter.RADOLAN_CDC: @@ -89,18 +85,14 @@ def __init__( Resolution.HOURLY, Resolution.DAILY, ): - raise ValueError( - "RADOLAN_CDC only supports daily and hourly resolutions" - ) + raise ValueError("RADOLAN_CDC only supports daily and hourly resolutions") elevation_parameters = [ DwdRadarParameter.SWEEP_VOL_VELOCITY_H, DwdRadarParameter.SWEEP_VOL_REFLECTIVITY_H, ] if self.elevation is not None and self.parameter not in elevation_parameters: - raise ValueError( - f"Argument 'elevation' only valid for parameter={elevation_parameters}" - ) + raise ValueError(f"Argument 'elevation' only valid for parameter={elevation_parameters}") if start_date == DwdRadarDate.LATEST: @@ -134,10 +126,7 @@ def __init__( start_date = datetime.utcnow() - timedelta(minutes=5) end_date = None - if ( - start_date == DwdRadarDate.MOST_RECENT - and parameter == DwdRadarParameter.RADOLAN_CDC - ): + if start_date == DwdRadarDate.MOST_RECENT and parameter == DwdRadarParameter.RADOLAN_CDC: start_date = datetime.utcnow() - timedelta(minutes=50) end_date = None @@ -169,7 +158,7 @@ def adjust_datetimes(self): minute marks for respective RadarParameter. - RADOLAN_CDC is always published at HH:50. - https://opendata.dwd.de/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/ # noqa:E501,B950 + https://opendata.dwd.de/climate_environment/CDC/grids_germany/daily/radolan/recent/bin/ - RQ_REFLECTIVITY is published each 15 minutes. https://opendata.dwd.de/weather/radar/radvor/rq/ @@ -180,10 +169,7 @@ def adjust_datetimes(self): """ - if ( - self.parameter == DwdRadarParameter.RADOLAN_CDC - or self.parameter in RADAR_PARAMETERS_RADOLAN - ): + if self.parameter == DwdRadarParameter.RADOLAN_CDC or self.parameter in RADAR_PARAMETERS_RADOLAN: # Align "start_date" to the most recent 50 minute mark available. self.start_date = raster_minutes(self.start_date, 50) diff --git a/wetterdienst/provider/dwd/radar/index.py b/wetterdienst/provider/dwd/radar/index.py index 6d692d902..1a000cabc 100644 --- a/wetterdienst/provider/dwd/radar/index.py +++ b/wetterdienst/provider/dwd/radar/index.py @@ -91,29 +91,21 @@ def create_fileindex_radar( files_server = list_remote_files_fsspec(url, recursive=True) - files_server = pd.DataFrame( - files_server, columns=[DwdColumns.FILENAME.value], dtype="str" - ) + files_server = pd.DataFrame(files_server, columns=[DwdColumns.FILENAME.value], dtype="str") # Some directories have both "---bin" and "---bufr" files within the same directory, # so we need to filter here by designated RadarDataFormat. Example: # https://opendata.dwd.de/weather/radar/sites/px/boo/ if fmt is not None: if fmt == DwdRadarDataFormat.BINARY: - files_server = files_server[ - files_server[DwdColumns.FILENAME.value].str.contains("--bin") - ] + files_server = files_server[files_server[DwdColumns.FILENAME.value].str.contains("--bin")] elif fmt == DwdRadarDataFormat.BUFR: - files_server = files_server[ - files_server[DwdColumns.FILENAME.value].str.contains("--buf") - ] + files_server = files_server[files_server[DwdColumns.FILENAME.value].str.contains("--buf")] # Decode datetime of file for filtering. if parse_datetime: - files_server[DwdColumns.DATETIME.value] = files_server[ - DwdColumns.FILENAME.value - ].apply(get_date_from_filename) + files_server[DwdColumns.DATETIME.value] = files_server[DwdColumns.FILENAME.value].apply(get_date_from_filename) files_server = files_server.dropna() @@ -121,9 +113,7 @@ def create_fileindex_radar( @fileindex_cache_five_minutes.cache_on_arguments() -def create_fileindex_radolan_cdc( - resolution: Resolution, period: Period -) -> pd.DataFrame: +def create_fileindex_radolan_cdc(resolution: Resolution, period: Period) -> pd.DataFrame: """ Function used to create a file index for the RADOLAN_CDC product. The file index will include both recent as well as historical files. A datetime column is created @@ -144,9 +134,7 @@ def create_fileindex_radolan_cdc( file_index = file_index[ file_index[DwdColumns.FILENAME.value].str.contains("/bin/") - & file_index[DwdColumns.FILENAME.value].str.endswith( - (Extension.GZ.value, Extension.TAR_GZ.value) - ) + & file_index[DwdColumns.FILENAME.value].str.endswith((Extension.GZ.value, Extension.TAR_GZ.value)) ].copy() # Decode datetime of file for filtering. @@ -200,10 +188,11 @@ def build_path_to_parameter( if parameter == DwdRadarParameter.RADOLAN_CDC: if resolution == Resolution.MINUTE_5: # See also page 4 on - # https://opendata.dwd.de/climate_environment/CDC/help/RADOLAN/Unterstuetzungsdokumente/Unterstuetzungsdokumente-Verwendung_von_RADOLAN-Produkten_im_ASCII-GIS-Rasterformat_in_GIS.pdf # noqa:E501,B950 - parameter_path = f"{DWD_CDC_PATH}/grids_germany/{resolution.value}/radolan/reproc/2017_002/bin" # noqa:E501,B950 + # https://opendata.dwd.de/climate_environment/CDC/help/RADOLAN/Unterstuetzungsdokumente/ + # Unterstuetzungsdokumente-Verwendung_von_RADOLAN-Produkten_im_ASCII-GIS-Rasterformat_in_GIS.pdf + parameter_path = f"{DWD_CDC_PATH}/grids_germany/{resolution.value}/radolan/reproc/2017_002/bin" else: - parameter_path = f"{DWD_CDC_PATH}/grids_germany/{resolution.value}/radolan/{period.value}/bin" # noqa:E501,B950 + parameter_path = f"{DWD_CDC_PATH}/grids_germany/{resolution.value}/radolan/{period.value}/bin" elif parameter in RADAR_PARAMETERS_COMPOSITES: parameter_path = f"weather/radar/composit/{parameter.value}" @@ -237,9 +226,7 @@ def build_path_to_parameter( candidates = [DwdRadarDataFormat.BUFR, DwdRadarDataFormat.HDF5] if candidates: - raise ValueError( - f"Argument 'format' is missing, use one of {candidates}" - ) + raise ValueError(f"Argument 'format' is missing, use one of {candidates}") # Compute path to BINARY/BUFR vs. HDF5. parameter_path = f"weather/radar/sites/{parameter.value}/{site.value}" @@ -249,9 +236,7 @@ def build_path_to_parameter( DwdRadarDataSubset.SIMPLE, DwdRadarDataSubset.POLARIMETRIC, ] - raise ValueError( - f"Argument 'subset' is missing, use one of {candidates}" - ) + raise ValueError(f"Argument 'subset' is missing, use one of {candidates}") parameter_path = f"{parameter_path}/{fmt.value}/filter_{subset.value}/" else: # pragma: no cover diff --git a/wetterdienst/provider/dwd/radar/metadata/parameter.py b/wetterdienst/provider/dwd/radar/metadata/parameter.py index 9dcc8946c..eaa2d7a69 100644 --- a/wetterdienst/provider/dwd/radar/metadata/parameter.py +++ b/wetterdienst/provider/dwd/radar/metadata/parameter.py @@ -34,7 +34,7 @@ class DwdRadarParameter(Enum): # /sites # https://opendata.dwd.de/weather/radar/sites/ - # https://docs.wradlib.org/en/stable/notebooks/fileio/wradlib_radar_formats.html#German-Weather-Service:-DX-format # noqa:E501,B950 + # https://docs.wradlib.org/en/stable/notebooks/fileio/wradlib_radar_formats.html#German-Weather-Service:-DX-format DX_REFLECTIVITY = "dx" LMAX_VOLUME_SCAN = "lmax" PE_ECHO_TOP = "pe" diff --git a/wetterdienst/provider/dwd/radar/sites.py b/wetterdienst/provider/dwd/radar/sites.py index 99d834521..23404ef1d 100644 --- a/wetterdienst/provider/dwd/radar/sites.py +++ b/wetterdienst/provider/dwd/radar/sites.py @@ -8,8 +8,8 @@ Sources ======= -- April, 2018: https://www.dwd.de/DE/derdwd/messnetz/atmosphaerenbeobachtung/_functions/HaeufigGesucht/koordinaten-radarverbund.pdf?__blob=publicationFile # noqa:E501,B950 -- October, 2020: https://www.dwd.de/DE/leistungen/radolan/radolan_info/radolan_radvor_op_komposit_format_pdf.pdf?__blob=publicationFile # noqa:E501,B950 +- April, 2018: https://www.dwd.de/DE/derdwd/messnetz/atmosphaerenbeobachtung/_functions/HaeufigGesucht/koordinaten-radarverbund.pdf?__blob=publicationFile # noqa:B950 +- October, 2020: https://www.dwd.de/DE/leistungen/radolan/radolan_info/radolan_radvor_op_komposit_format_pdf.pdf?__blob=publicationFile # noqa:B950 References ========== @@ -51,14 +51,14 @@ class DwdRadarSitesGenerator: # pragma: no cover """ Parse list of sites from PDF documents [1,2] and output as Python dictionary. - [1] https://www.dwd.de/DE/derdwd/messnetz/atmosphaerenbeobachtung/_functions/HaeufigGesucht/koordinaten-radarverbund.pdf?__blob=publicationFile # noqa:E501,B950 - [2] https://www.dwd.de/DE/leistungen/radolan/radolan_info/radolan_radvor_op_komposit_format_pdf.pdf?__blob=publicationFile # noqa:E501,B950 + [1] https://www.dwd.de/DE/derdwd/messnetz/atmosphaerenbeobachtung/_functions/HaeufigGesucht/koordinaten-radarverbund.pdf?__blob=publicationFile # noqa:B950 + [2] https://www.dwd.de/DE/leistungen/radolan/radolan_info/radolan_radvor_op_komposit_format_pdf.pdf?__blob=publicationFile # noqa:B950 """ url = ( "https://www.dwd.de/DE/derdwd/messnetz/atmosphaerenbeobachtung/_functions" "/HaeufigGesucht/koordinaten-radarverbund.pdf?__blob=publicationFile" - ) # noqa:E501,B950 + ) def all(self) -> Dict: # pragma: no cover """ @@ -111,19 +111,13 @@ def read_pdf(self) -> pd.DataFrame: # Mungle into one coherent data frame. data = firsts - data = data.drop( - labels=["coordinates_wgs84_text", "coordinates_gauss"], axis="columns" - ) + data = data.drop(labels=["coordinates_wgs84_text", "coordinates_gauss"], axis="columns") data = data.rename(columns={"coordinates_wgs84": "latitude"}) data.insert(4, "longitude", seconds["coordinates_wgs84"].values) data = data.reset_index(drop=True) for column in ["latitude", "longitude"]: - data[column] = ( - data[column] - .apply(lambda x: x.strip("NE").replace(",", ".")) - .apply(float) # noqa: E501 - ) + data[column] = data[column].apply(lambda x: x.strip("NE").replace(",", ".")).apply(float) for column in ["wmo_id", "altitude"]: data[column] = data[column].apply(int) diff --git a/wetterdienst/provider/eccc/observation/api.py b/wetterdienst/provider/eccc/observation/api.py index 52454a581..0953aecbf 100644 --- a/wetterdienst/provider/eccc/observation/api.py +++ b/wetterdienst/provider/eccc/observation/api.py @@ -49,9 +49,7 @@ class EcccObservationValues(ScalarValuesCore): _has_quality = True _session = requests.Session() - _session.mount( - "https://", HTTPAdapter(max_retries=Retry(total=10, connect=5, read=5)) - ) + _session.mount("https://", HTTPAdapter(max_retries=Retry(total=10, connect=5, read=5))) _base_url = ( "https://climate.weather.gc.ca/climate_data/bulk_data_e.html?" @@ -87,9 +85,7 @@ def _create_humanized_parameters_mapping(self): # TODO: change to something general, depending on ._has_datasets hcnm = { parameter.value: parameter.name.lower() - for parameter in self.stations.stations._parameter_base[ - self.stations.stations.resolution.name - ] + for parameter in self.stations.stations._parameter_base[self.stations.stations.resolution.name] } return hcnm @@ -133,9 +129,7 @@ def _collect_station_parameter( :param dataset: dataset of query, can be skipped as ECCC has unique dataset :return: pandas.DataFrame with data """ - meta = self.stations.df[ - self.stations.df[Columns.STATION_ID.value] == station_id - ] + meta = self.stations.df[self.stations.df[Columns.STATION_ID.value] == station_id] name, from_date, to_date = ( meta[ @@ -157,9 +151,7 @@ def _collect_station_parameter( start_date = self.stations.stations.start_date end_date = self.stations.stations.end_date - start_year = start_year and max( - start_year, start_date and start_date.year or start_year - ) + start_year = start_year and max(start_year, start_date and start_date.year or start_year) end_year = end_year and min(end_year, end_date and end_date.year or end_year) # Following lines may partially be based on @Zeitsperre's canada-climate-python @@ -210,9 +202,7 @@ def _collect_station_parameter( return df - def _create_file_urls( - self, station_id: str, start_year: int, end_year: int - ) -> Generator[str, None, None]: + def _create_file_urls(self, station_id: str, start_year: int, end_year: int) -> Generator[str, None, None]: """ :param station_id: @@ -228,9 +218,7 @@ def _create_file_urls( # For hourly data request only necessary data to reduce amount of data being # downloaded and parsed - for date in pd.date_range( - f"{start_year}-01-01", f"{end_year + 1}-01-01", freq=freq, closed=None - ): + for date in pd.date_range(f"{start_year}-01-01", f"{end_year + 1}-01-01", freq=freq, closed=None): url = self._base_url.format(int(station_id), self._timeframe) url += f"&Year={date.year}" @@ -362,8 +350,7 @@ def _download_stations() -> bytes: """ ftp_url = ( - "ftp://client_climate:foobar@ftp.tor.ec.gc.ca" - "/Pub/Get_More_Data_Plus_de_donnees/Station Inventory EN.csv" + "ftp://client_climate:foobar@ftp.tor.ec.gc.ca" "/Pub/Get_More_Data_Plus_de_donnees/Station Inventory EN.csv" ) http_url = ( diff --git a/wetterdienst/provider/eumetnet/opera/sites.py b/wetterdienst/provider/eumetnet/opera/sites.py index 89fef06d2..a31978de9 100644 --- a/wetterdienst/provider/eumetnet/opera/sites.py +++ b/wetterdienst/provider/eumetnet/opera/sites.py @@ -80,8 +80,7 @@ def by_countryname(self, name: str) -> List[Dict]: """ sites = list( filter( - lambda site: site["country"] - and site["country"].lower() == name.lower(), + lambda site: site["country"] and site["country"].lower() == name.lower(), self.sites, ) ) @@ -98,9 +97,9 @@ class OperaRadarSitesGenerator: """ url = ( - "https://www.eumetnet.eu/wp-content/themes/aeron-child/observations-programme" - "/current-activities/opera/database/OPERA_Database/OPERA_RADARS_DB.json" - ) # noqa: E501 + "https://www.eumetnet.eu/wp-content/themes/aeron-child/observations-programme/" + "current-activities/opera/database/OPERA_Database/OPERA_RADARS_DB.json" + ) def get_opera_radar_sites(self) -> List[Dict]: # pragma: no cover @@ -151,9 +150,7 @@ def convert_types(element): def filter_and_convert(elements): for element in elements: - if ( - element["location"] and element["latitude"] and element["longitude"] - ): # noqa: E501 + if element["location"] and element["latitude"] and element["longitude"]: yield convert_types(element) return list(filter_and_convert(data)) @@ -170,9 +167,7 @@ def export(self): Generate "sites.json.gz". """ sites = self.get_opera_radar_sites() - with gzip.open( - OperaRadarSites.data_file, mode="wt", compresslevel=9, encoding="utf-8" - ) as fp: + with gzip.open(OperaRadarSites.data_file, mode="wt", compresslevel=9, encoding="utf-8") as fp: json.dump(sites, fp, indent=4) diff --git a/wetterdienst/provider/noaa/ghcn/api.py b/wetterdienst/provider/noaa/ghcn/api.py index 0f50ecfc3..1a1a11171 100644 --- a/wetterdienst/provider/noaa/ghcn/api.py +++ b/wetterdienst/provider/noaa/ghcn/api.py @@ -46,9 +46,7 @@ class NoaaGhcnValues(ScalarValuesCore): _data_tz = Timezone.DYNAMIC - _base_url = ( - "https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/by_station/{station_id}.csv.gz" - ) + _base_url = "https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/by_station/{station_id}.csv.gz" # use to get timezones from stations _tf = TimezoneFinder() @@ -56,9 +54,7 @@ class NoaaGhcnValues(ScalarValuesCore): # multiplication factors _mp_factors = PARAMETER_MULTIPLICATION_FACTORS - def _collect_station_parameter( - self, station_id: str, parameter, dataset - ) -> pd.DataFrame: + def _collect_station_parameter(self, station_id: str, parameter, dataset) -> pd.DataFrame: """ Collection method for NOAA GHCN data. Parameter and dataset can be ignored as data is provided as a whole. @@ -88,9 +84,7 @@ def _collect_station_parameter( Columns.QUALITY.value, ) - df.loc[:, Columns.PARAMETER.value] = df.loc[ - :, Columns.PARAMETER.value - ].str.lower() + df.loc[:, Columns.PARAMETER.value] = df.loc[:, Columns.PARAMETER.value].str.lower() # get timezone from station timezone_ = self._get_timezone_from_station(station_id) @@ -132,9 +126,7 @@ def _apply_factors(self, df: pd.DataFrame) -> pd.DataFrame: return df_factors - def _add_derived_parameters( - self, df: pd.DataFrame, station_id: str - ) -> pd.DataFrame: + def _add_derived_parameters(self, df: pd.DataFrame, station_id: str) -> pd.DataFrame: """ Method to add derived parameters to DataFrame, specifically temperature_air_mean_200 from maximum and minimum daily temperature @@ -167,9 +159,7 @@ def _add_derived_parameters( .reset_index() ) - df_temperatures = df_temperatures.reindex( - columns=(Columns.DATE.value, tmax_key, tmin_key) - ) + df_temperatures = df_temperatures.reindex(columns=(Columns.DATE.value, tmax_key, tmin_key)) start_date = self.stations.start_date end_date = self.stations.end_date @@ -202,8 +192,7 @@ def _add_derived_parameters( df_tmean = df_tmean.merge(df_temperatures, how="left", on=Columns.DATE.value) df_tmean[Columns.VALUE.value] = ( - df_tmean[tmax_key].astype(float, errors="ignore") - + df_tmean[tmin_key].astype(float, errors="ignore") + df_tmean[tmax_key].astype(float, errors="ignore") + df_tmean[tmin_key].astype(float, errors="ignore") ) / 2 df_tmean = df_tmean.drop(columns=[tmax_key, tmin_key]) diff --git a/wetterdienst/ui/cli.py b/wetterdienst/ui/cli.py index b19829bbb..5edf73bee 100644 --- a/wetterdienst/ui/cli.py +++ b/wetterdienst/ui/cli.py @@ -86,9 +86,7 @@ def station_options(command): ), cloup.option_group( "Latitude-Longitude rank/distance filtering", - cloup.option( - "--coordinates", metavar="LATITUDE,LONGITUDE", type=click.STRING - ), + cloup.option("--coordinates", metavar="LATITUDE,LONGITUDE", type=click.STRING), cloup.option("--rank", type=click.INT), cloup.option("--distance", type=click.FLOAT), help="Provide --coordinates plus either --rank or --distance.", @@ -449,9 +447,7 @@ def coverage(provider, kind, filter_, debug): def fields(provider, kind, dataset, resolution, period, language, **kwargs): api = get_api(provider, kind) - if not ( - api.provider == Provider.DWD and api.kind == Kind.OBSERVATION - ) and kwargs.get("fields"): + if not (api.provider == Provider.DWD and api.kind == Kind.OBSERVATION) and kwargs.get("fields"): raise click.BadParameter("'fields' command only available for provider 'DWD'") metadata = api.describe_fields( diff --git a/wetterdienst/ui/core.py b/wetterdienst/ui/core.py index a2e10bb4f..5ec622009 100644 --- a/wetterdienst/ui/core.py +++ b/wetterdienst/ui/core.py @@ -103,17 +103,14 @@ def get_stations( else: res = Resolution.HOUR_6 else: - res = parse_enumeration_from_template( - resolution, api._resolution_base, Resolution - ) + res = parse_enumeration_from_template(resolution, api._resolution_base, Resolution) # Split date string into start and end date string start_date, end_date = create_date_range(date=date, resolution=res) if api._data_range == DataRange.LOOSELY and not start_date and not end_date: raise TypeError( - f"Combination of provider {api.provider.name} and kind {api.kind.name} " - f"requires start and end date" + f"Combination of provider {api.provider.name} and kind {api.kind.name} " f"requires start and end date" ) # Todo: We may have to apply other measures to allow for diff --git a/wetterdienst/ui/explorer/app.py b/wetterdienst/ui/explorer/app.py index 74751008c..57c055347 100644 --- a/wetterdienst/ui/explorer/app.py +++ b/wetterdienst/ui/explorer/app.py @@ -70,12 +70,7 @@ def fetch_stations(parameter: str, resolution: str, period: str): The data will be stored on a hidden within the browser DOM. """ - log.info( - f"Requesting stations for " - f"parameter={parameter}, " - f"resolution={resolution}, " - f"period={period}" - ) + log.info(f"Requesting stations for " f"parameter={parameter}, " f"resolution={resolution}, " f"period={period}") try: stations = DwdObservationRequest( parameter=DwdObservationDataset(parameter), @@ -204,9 +199,7 @@ def render_navigation_variables(payload): Input("dataframe-stations", "children"), ], ) -def render_status_response_stations( - parameter: str, resolution: str, period: str, payload: str -): +def render_status_response_stations(parameter: str, resolution: str, period: str, payload: str): """ Report about the status of the query. """ @@ -278,9 +271,7 @@ def render_status_response_values( missing.append(candidate) if missing: - empty_message.append( - html.Span(f"Please select all of the missing options {missing}.") - ) + empty_message.append(html.Span(f"Please select all of the missing options {missing}.")) messages += [html.Div(empty_message), html.Br()] @@ -374,9 +365,7 @@ def render_graph(variable, payload): except ValueError: climate_data = pd.DataFrame() - log.info( - f"Rendering graph for variable={variable} from {frame_summary(climate_data)}" - ) + log.info(f"Rendering graph for variable={variable} from {frame_summary(climate_data)}") fig = default_figure(climate_data, variable) @@ -392,9 +381,7 @@ def render_graph(variable, payload): return fig -def start_service( - listen_address: Optional[str] = None, reload: Optional[bool] = False -): # pragma: no cover +def start_service(listen_address: Optional[str] = None, reload: Optional[bool] = False): # pragma: no cover """ This entrypoint will be used by `wetterdienst.cli`. """ diff --git a/wetterdienst/ui/explorer/layout/observations_germany.py b/wetterdienst/ui/explorer/layout/observations_germany.py index 40162be09..c305f57c6 100644 --- a/wetterdienst/ui/explorer/layout/observations_germany.py +++ b/wetterdienst/ui/explorer/layout/observations_germany.py @@ -15,25 +15,17 @@ def get_parameters(): return sorted( - [ - {"label": param.value, "value": param.value} - for param in DwdObservationDataset - ], + [{"label": param.value, "value": param.value} for param in DwdObservationDataset], key=operator.itemgetter("label"), ) def get_resolutions(): - return [ - {"label": param.value, "value": param.value} - for param in DwdObservationResolution - ] + return [{"label": param.value, "value": param.value} for param in DwdObservationResolution] def get_periods(): - return [ - {"label": param.value, "value": param.value} for param in DwdObservationPeriod - ] + return [{"label": param.value, "value": param.value} for param in DwdObservationPeriod] def dashboard_layout() -> html: diff --git a/wetterdienst/ui/explorer/library.py b/wetterdienst/ui/explorer/library.py index 9ed04f2a8..54d92c6b1 100644 --- a/wetterdienst/ui/explorer/library.py +++ b/wetterdienst/ui/explorer/library.py @@ -39,13 +39,7 @@ def default_figure(climate_data: pd.DataFrame, column: str) -> go.Figure: add_annotation_no_data(fig) return fig - fig = go.Figure( - data=[ - go.Scatter( - x=climate_data.date, y=climate_data.loc[:, column], hoverinfo="x+y" - ) - ] - ) + fig = go.Figure(data=[go.Scatter(x=climate_data.date, y=climate_data.loc[:, column], hoverinfo="x+y")]) fig.update_layout(yaxis={"title": f"{column}"}, showlegend=False) return fig diff --git a/wetterdienst/ui/restapi.py b/wetterdienst/ui/restapi.py index 3c888ff3a..8fa8e1f9a 100644 --- a/wetterdienst/ui/restapi.py +++ b/wetterdienst/ui/restapi.py @@ -36,9 +36,7 @@ def index(): for provider in Provider: shortname = provider.name _, name, country, copyright_, url = provider.value - sources += ( - f"
  • {shortname} ({name}, {country}) - {copyright_}
    • " - ) + sources += f"
  • {shortname} ({name}, {country}) - {copyright_}
    • " return f""" @@ -67,7 +65,7 @@ def index(): - """ # noqa:E501,B950 + """ # noqa:B950 @app.get("/robots.txt", response_class=PlainTextResponse) @@ -90,9 +88,7 @@ def coverage( if not provider or not kind: cov = Wetterdienst.discover() - return Response( - content=json.dumps(cov, indent=4), media_type="application/json" - ) + return Response(content=json.dumps(cov, indent=4), media_type="application/json") api = get_api(provider=provider, kind=kind) @@ -137,8 +133,7 @@ def stations( if parameter is None or resolution is None: raise HTTPException( status_code=400, - detail="Query arguments 'parameter', 'resolution' " - "and 'period' are required", + detail="Query arguments 'parameter', 'resolution' " "and 'period' are required", ) if fmt not in ("json", "geojson"): @@ -279,8 +274,7 @@ def values( if parameter is None or resolution is None: raise HTTPException( status_code=400, - detail="Query arguments 'parameter', 'resolution' " - "and 'date' are required", + detail="Query arguments 'parameter', 'resolution' " "and 'date' are required", ) if fmt not in ("json", "geojson"): @@ -376,9 +370,7 @@ def make_json_response(data, provider): return response -def start_service( - listen_address: Optional[str] = None, reload: Optional[bool] = False -): # pragma: no cover +def start_service(listen_address: Optional[str] = None, reload: Optional[bool] = False): # pragma: no cover setup_logging() diff --git a/wetterdienst/util/cache.py b/wetterdienst/util/cache.py index f45929c37..45746672e 100644 --- a/wetterdienst/util/cache.py +++ b/wetterdienst/util/cache.py @@ -73,33 +73,25 @@ class CacheExpiry(Enum): # Define cache regions. -metaindex_cache = make_region( - function_key_generator=kwarg_function_key_generator -).configure( +metaindex_cache = make_region(function_key_generator=kwarg_function_key_generator).configure( backend, expiration_time=60 * 60 * 12, arguments={"filename": os.path.join(cache_dir, "dogpile", "metaindex.dbm")}, ) -fileindex_cache_five_minutes = make_region( - function_key_generator=kwarg_function_key_generator -).configure( +fileindex_cache_five_minutes = make_region(function_key_generator=kwarg_function_key_generator).configure( backend, expiration_time=60 * 5, arguments={"filename": os.path.join(cache_dir, "dogpile", "fileindex_5m.dbm")}, ) -fileindex_cache_twelve_hours = make_region( - function_key_generator=kwarg_function_key_generator -).configure( +fileindex_cache_twelve_hours = make_region(function_key_generator=kwarg_function_key_generator).configure( backend, expiration_time=60 * 60 * 12, arguments={"filename": os.path.join(cache_dir, "dogpile", "fileindex_12h.dbm")}, ) -payload_cache_twelve_hours = make_region( - function_key_generator=kwarg_function_key_generator -).configure( +payload_cache_twelve_hours = make_region(function_key_generator=kwarg_function_key_generator).configure( backend, expiration_time=60 * 60 * 12, arguments={"filename": os.path.join(cache_dir, "dogpile", "payload_12h.dbm")}, diff --git a/wetterdienst/util/datetime.py b/wetterdienst/util/datetime.py index 42dc1e9c9..aafe7d70e 100644 --- a/wetterdienst/util/datetime.py +++ b/wetterdienst/util/datetime.py @@ -77,8 +77,6 @@ def mktimerange( date_to = date_to + relativedelta(day=31) else: - raise NotImplementedError( - "mktimerange only implemented for annual and monthly time ranges" - ) + raise NotImplementedError("mktimerange only implemented for annual and monthly time ranges") return date_from, date_to diff --git a/wetterdienst/util/enumeration.py b/wetterdienst/util/enumeration.py index 7a18d5f45..627f53438 100644 --- a/wetterdienst/util/enumeration.py +++ b/wetterdienst/util/enumeration.py @@ -56,9 +56,7 @@ def parse_enumeration_from_template( else: enum_parsed = intermediate(enum_) except ValueError: - raise InvalidEnumeration( - f"{enum_} could not be parsed from {intermediate.__name__}." - ) + raise InvalidEnumeration(f"{enum_} could not be parsed from {intermediate.__name__}.") if base: try: @@ -67,9 +65,7 @@ def parse_enumeration_from_template( try: enum_parsed = base(enum_parsed) except ValueError: - raise InvalidEnumeration( - f"{enum_parsed} could not be parsed from {base.__name__}." - ) + raise InvalidEnumeration(f"{enum_parsed} could not be parsed from {base.__name__}.") return enum_parsed diff --git a/wetterdienst/util/geo.py b/wetterdienst/util/geo.py index 9f8932cbd..e003cacab 100644 --- a/wetterdienst/util/geo.py +++ b/wetterdienst/util/geo.py @@ -38,9 +38,7 @@ def get_coordinates_in_radians(self): return np.radians(self.get_coordinates()) def __eq__(self, other): - return np.array_equal(self.latitudes, other.latitudes) and np.array_equal( - self.longitudes, other.longitudes - ) + return np.array_equal(self.latitudes, other.latitudes) and np.array_equal(self.longitudes, other.longitudes) def derive_nearest_neighbours( @@ -67,9 +65,7 @@ def derive_nearest_neighbours( """ points = np.c_[np.radians(latitudes), np.radians(longitudes)] distance_tree = cKDTree(points) - return distance_tree.query( - coordinates.get_coordinates_in_radians(), k=number_nearby - ) + return distance_tree.query(coordinates.get_coordinates_in_radians(), k=number_nearby) def convert_dm_to_dd(dms: float) -> float: diff --git a/wetterdienst/util/io.py b/wetterdienst/util/io.py index ffeff6e18..88dd02f10 100644 --- a/wetterdienst/util/io.py +++ b/wetterdienst/util/io.py @@ -3,7 +3,7 @@ def read_in_chunks(file_object, chunk_size=1024): Lazy function (generator) to read a file piece by piece. Default chunk size: 1k. - -- https://stackoverflow.com/questions/519633/lazy-method-for-reading-big-file-in-python/519653#519653 # Noqa: E501, B950 + -- https://stackoverflow.com/questions/519633/lazy-method-for-reading-big-file-in-python/519653#519653 """ while True: data = file_object.read(chunk_size) diff --git a/wetterdienst/util/logging.py b/wetterdienst/util/logging.py index 69bd92244..cbac43f85 100644 --- a/wetterdienst/util/logging.py +++ b/wetterdienst/util/logging.py @@ -10,7 +10,7 @@ class TqdmToLogger(io.StringIO): Output stream for TQDM which will output to logger module instead of the StdOut. - Source: https://stackoverflow.com/questions/14897756/python-progress-bar-through-logging-module # noqa: E501 + Source: https://stackoverflow.com/questions/14897756/python-progress-bar-through-logging-module """ logger = None diff --git a/wetterdienst/util/network.py b/wetterdienst/util/network.py index dd02a5786..0a3e32fe6 100644 --- a/wetterdienst/util/network.py +++ b/wetterdienst/util/network.py @@ -47,9 +47,7 @@ def register(cls, ttl=CacheExpiry.NO_CACHE): ttl_name, ttl_value = cls.resolve_ttl(ttl) key = f"ttl-{ttl_name}" real_cache_dir = os.path.join(cache_dir, "fsspec", key) - filesystem_real = HTTPFileSystem( - use_listings_cache=True, client_kwargs=FSSPEC_CLIENT_KWARGS - ) + filesystem_real = HTTPFileSystem(use_listings_cache=True, client_kwargs=FSSPEC_CLIENT_KWARGS) if WD_CACHE_DISABLE or ttl is CacheExpiry.NO_CACHE: filesystem_effective = filesystem_real else: @@ -91,9 +89,7 @@ def list_remote_files_legacy(url: str, recursive: bool) -> List[str]: soup = BeautifulSoup(r.text, "lxml") - files_and_folders = [ - link.get("href") for link in soup.find_all("a") if link.get("href") != "../" - ] + files_and_folders = [link.get("href") for link in soup.find_all("a") if link.get("href") != "../"] files = [] folders = [] @@ -105,9 +101,7 @@ def list_remote_files_legacy(url: str, recursive: bool) -> List[str]: folders.append(urljoin(url, f)) if recursive: - files_in_folders = [ - list_remote_files_legacy(folder, recursive) for folder in folders - ] + files_in_folders = [list_remote_files_legacy(folder, recursive) for folder in folders] for files_in_folder in files_in_folders: files.extend(files_in_folder) @@ -116,9 +110,7 @@ def list_remote_files_legacy(url: str, recursive: bool) -> List[str]: # v2: "Remote directory index" implementation based on FSSPEC. -def list_remote_files_fsspec( - url: str, recursive: bool = False, ttl: CacheExpiry = CacheExpiry.FILEINDEX -) -> List[str]: +def list_remote_files_fsspec(url: str, recursive: bool = False, ttl: CacheExpiry = CacheExpiry.FILEINDEX) -> List[str]: """ A function used to create a listing of all files of a given path on the server.