From b274e1e71714e085144a779444c477d837cfa528 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 13:14:05 -0400 Subject: [PATCH 01/12] function, docs, tests --- docs/sphinx/source/reference/iotools.rst | 11 ++ docs/sphinx/source/whatsnew/v0.13.2.rst | 3 +- pvlib/iotools/__init__.py | 1 + pvlib/iotools/ecmwf.py | 187 +++++++++++++++++++++++ tests/conftest.py | 13 ++ tests/iotools/test_ecmwf.py | 78 ++++++++++ 6 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 pvlib/iotools/ecmwf.py create mode 100644 tests/iotools/test_ecmwf.py diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 12db7d6818..f54ba59928 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -237,6 +237,17 @@ lower quality. iotools.read_crn +ECMWF ERA5 +^^^^^^^^^^ + +A global reanalysis dataset providing weather and solar resource data. + +.. autosummary:: + :toctree: generated/ + + iotools.get_era5 + + Generic data file readers ------------------------- diff --git a/docs/sphinx/source/whatsnew/v0.13.2.rst b/docs/sphinx/source/whatsnew/v0.13.2.rst index e12f7277e5..e2134feac3 100644 --- a/docs/sphinx/source/whatsnew/v0.13.2.rst +++ b/docs/sphinx/source/whatsnew/v0.13.2.rst @@ -27,7 +27,8 @@ Enhancements :py:func:`~pvlib.singlediode.bishop88_mpp`, :py:func:`~pvlib.singlediode.bishop88_v_from_i`, and :py:func:`~pvlib.singlediode.bishop88_i_from_v`. (:issue:`2497`, :pull:`2498`) - +* Add :py:func:`~pvlib.iotools.get_era5`, a function for accessing + ERA-5 reanalysis data. (:pull:`2573`) Documentation diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 75663507f3..693ac66d9c 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -45,3 +45,4 @@ from pvlib.iotools.meteonorm import get_meteonorm_observation_training # noqa: F401, E501 from pvlib.iotools.meteonorm import get_meteonorm_tmy # noqa: F401 from pvlib.iotools.nasa_power import get_nasa_power # noqa: F401 +from pvlib.iotools.ecmwf import get_era5 # noqa: F401 diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py new file mode 100644 index 0000000000..a586efd702 --- /dev/null +++ b/pvlib/iotools/ecmwf.py @@ -0,0 +1,187 @@ +import requests +import pandas as pd +from io import BytesIO, StringIO +import zipfile +import time + + +VARIABLE_MAP = { + # short names + 'd2m': 'temp_dew', + 't2m': 'temp_air', + 'sp': 'pressure', + 'ssrd': 'ghi', + 'tp': 'precipitation', + + # long names + '2m_dewpoint_temperature': 'temp_dew', + '2m_temperature': 'temp_air', + 'surface_pressure': 'pressure', + 'surface_solar_radiation_downwards': 'ghi', + 'total_precipitation': 'precipitation', +} + + +def same(x): + return x + + +def k_to_c(temp_k): + return temp_k - 273.15 + + +def j_to_w(j): + return j / 3600 + + +def m_to_cm(m): + return m / 100 + +UNITS = { + 'u100': same, + 'v100': same, + 'u10': same, + 'v10': same, + 'd2m': k_to_c, + 't2m': k_to_c, + 'msl': same, + 'sst': k_to_c, + 'skt': k_to_c, + 'sp': same, + 'ssrd': j_to_w, + 'strd': j_to_w, + 'tp': m_to_cm, +} + + +def get_era5(latitude, longitude, start, end, variables, api_key, + map_variables=True, timeout=60, + url='https://cds.climate.copernicus.eu/api/retrieve/v1/'): + """ + Retrieve ERA5 reanalysis data from the ECMWF's Copernicus Data Store. + + This API [1]_ provides a subset of the full ERA5 dataset. See [2]_ for + the available variables. Data are available on a 0.25° x 0.25° grid. + + Parameters + ---------- + latitude : float + In decimal degrees, north is positive (ISO 19115). + longitude: float + In decimal degrees, east is positive (ISO 19115). + start : datetime like or str + First day of the requested period. + end : datetime like or str + Last day of the requested period. + variables : list of str + List of variable names to retrieve. See [1]_ for options. + api_key : str + ECMWF API key. + map_variables : bool, default True + When true, renames columns of the DataFrame to pvlib variable names + where applicable. See variable :const:`VARIABLE_MAP`. + timeout : int, default 60 + Number of seconds to wait for the requested data to become available + before timeout. + url : str, optional + API endpoint URL. + + Raises + ------ + Exception + If ``timeout`` is reached without the job finishing. + + Returns + ------- + data : pd.DataFrame + Time series data. The index corresponds to the start of the interval. + meta : dict + Metadata. + + References + ---------- + .. [1] https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-timeseries?tab=overview + .. [2] https://confluence.ecmwf.int/pages/viewpage.action?pageId=505390919 + """ # noqa: E501 + start = pd.to_datetime(start).strftime("%Y-%m-%d") + end = pd.to_datetime(end).strftime("%Y-%m-%d") + + headers = {'PRIVATE-TOKEN': api_key} + + # allow variables to be specified with pvlib names + reverse_map = {v: k for k, v in VARIABLE_MAP.items()} + variables = [reverse_map.get(k, k) for k in variables] + + # Step 1: submit data request (add it to the queue) + params = { + "inputs": { + "variable": variables, + "location": {"longitude": longitude, "latitude": latitude}, + "date": [f"{start}/{end}"], + "data_format": "csv" + } + } + slug = "processes/reanalysis-era5-single-levels-timeseries/execution" + response = requests.post(url + slug, json=params, headers=headers) + submission_response = response.json() + job_id = submission_response['jobID'] + + # Step 2: poll until the data request is ready + slug = "jobs/" + job_id + poll_interval = 1 + num_polls = 0 + while True: + response = requests.get(url + slug, headers=headers) + poll_response = response.json() + job_status = poll_response['status'] + + if job_status == 'successful': + break # ready to proceed to next step + elif job_status == 'failed': + msg = ( + 'Request failed. Please check the ECMWF website for details: ' + 'https://cds.climate.copernicus.eu/requests?tab=all' + ) + raise Exception(msg) + + num_polls += 1 + if num_polls * poll_interval > timeout: + raise Exception( + 'Request timed out. Try increasing the timeout parameter or ' + 'reducing the request size.' + ) + + time.sleep(1) + + # Step 3: get the download link for our requested dataset + slug = "jobs/" + job_id + "/results" + response = requests.get(url + slug, headers=headers) + results_response = response.json() + download_url = results_response['asset']['value']['href'] + + # Step 4: finally, download our dataset. it's a zipfile of one CSV + response = requests.get(download_url) + zipbuffer = BytesIO(response.content) + archive = zipfile.ZipFile(zipbuffer) + filename = archive.filelist[0].filename + csvbuffer = StringIO(archive.read(filename).decode('utf-8')) + df = pd.read_csv(csvbuffer) + + # and parse into the usual formats + metadata = submission_response['metadata'] # include messages from ECMWF + metadata['jobID'] = job_id + if not df.empty: + metadata['latitude'] = df['latitude'].values[0] + metadata['longitude'] = df['longitude'].values[0] + + df.index = pd.to_datetime(df['valid_time']).dt.tz_localize('UTC') + df = df.drop(columns=['valid_time', 'latitude', 'longitude']) + + if map_variables: + # convert units and rename + for shortname in df.columns: + converter = UNITS[shortname] + df[shortname] = converter(df[shortname]) + df = df.rename(columns=VARIABLE_MAP) + + return df, metadata diff --git a/tests/conftest.py b/tests/conftest.py index 0dc957751b..220e44fa91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -130,6 +130,19 @@ def nrel_api_key(): reason='requires solaranywhere credentials') +try: + # Attempt to load ECMWF API key used for testing + # pvlib.iotools.get_era5 + ecwmf_api_key = os.environ["ECMWF_API_KEY"] + has_ecmwf_credentials = True +except KeyError: + has_ecmwf_credentials = False + +requires_ecmwf_credentials = pytest.mark.skipif( + not has_solaranywhere_credentials, + reason='requires ECMWF credentials') + + try: import statsmodels # noqa: F401 has_statsmodels = True diff --git a/tests/iotools/test_ecmwf.py b/tests/iotools/test_ecmwf.py new file mode 100644 index 0000000000..3b984f7c84 --- /dev/null +++ b/tests/iotools/test_ecmwf.py @@ -0,0 +1,78 @@ +""" +tests for pvlib/iotools/ecmwf.py +""" + +import pandas as pd +import pytest +import pvlib +import os +from tests.conftest import RERUNS, RERUNS_DELAY, requires_ecmwf_credentials + + +@pytest.fixture +def params(): + api_key = os.environ["ECMWF_API_KEY"] + + return { + 'latitude': 40.01, 'longitude': -80.01, + 'start': '2020-06-01', 'end': '2020-06-02', + 'variables': ['ghi', 'temp_air'], + 'api_key': api_key, + } + + +@pytest.fixture +def expected(): + index = pd.date_range("2020-06-01 00:00", "2020-06-01 23:59", freq="h", + tz="UTC") + index.name = 'valid_time' + temp_air = [16.6, 15.2, 13.5, 11.2, 10.8, 9.1, 7.3, 6.8, 7.6, 7.4, 8.5, + 8.1, 9.8, 11.5, 14.1, 17.4, 18.3, 20., 20.7, 20.9, 21.5, + 21.6, 21., 20.7] + ghi = [153., 18.4, 0., 0., 0., 0., 0., 0., 0., 0., 0., 60., 229.5, + 427.8, 620.1, 785.5, 910.1, 984.2, 1005.9, 962.4, 844.1, 685.2, + 526.9, 331.4] + df = pd.DataFrame({'temp_air': temp_air, 'ghi': ghi}, index=index) + return df + + +@requires_ecmwf_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_era5(params, expected): + df, meta = pvlib.iotools.get_era5(**params) + pd.testing.assert_frame_equal(df, expected, check_freq=False) + assert meta['longitude'] == -80.0 + assert meta['latitude'] == 40.0 + assert isinstance(meta['jobID'], str) + + +@requires_ecmwf_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_era5_map_variables(params, expected): + df, meta = pvlib.iotools.get_era5(**params, map_variables=False) + expected = expected.rename(columns={'temp_air': 't2m', 'ghi': 'ssrd'}) + pd.testing.assert_frame_equal(df, expected, check_freq=False) + assert meta['longitude'] == -80.0 + assert meta['latitude'] == 40.0 + assert isinstance(meta['jobID'], str) + + +@requires_ecmwf_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_era5_error(params): + params['variables'] = ['nonexistent'] + match = 'Request failed. Please check the ECMWF website' + with pytest.raises(Exception, match=match): + df, meta = pvlib.iotools.get_era5(**params) + + +@requires_ecmwf_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_era5_timeout(params): + match = 'Request failed. Please check the ECMWF website' + with pytest.raises(Exception, match=match): + df, meta = pvlib.iotools.get_era5(**params, timeout=1) From d3bd4262de5e314ff5937792d45cec343d0a031c Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 13:14:23 -0400 Subject: [PATCH 02/12] make api key secret accessible to tests --- .github/workflows/pytest-remote-data.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest-remote-data.yml b/.github/workflows/pytest-remote-data.yml index 31145bd091..d7bb5ceb52 100644 --- a/.github/workflows/pytest-remote-data.yml +++ b/.github/workflows/pytest-remote-data.yml @@ -100,6 +100,7 @@ jobs: SOLARANYWHERE_API_KEY: ${{ secrets.SOLARANYWHERE_API_KEY }} BSRN_FTP_USERNAME: ${{ secrets.BSRN_FTP_USERNAME }} BSRN_FTP_PASSWORD: ${{ secrets.BSRN_FTP_PASSWORD }} + ECMWF_API_KEY: ${{ secrets.ECMWF_API_KEY }} run: pytest tests/iotools --cov=./ --cov-report=xml --remote-data - name: Upload coverage to Codecov From 0b5ba09a504675b64e25570911c5fcc68bb94dd3 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 13:23:45 -0400 Subject: [PATCH 03/12] bit more docs --- pvlib/iotools/ecmwf.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py index a586efd702..a527c70e5b 100644 --- a/pvlib/iotools/ecmwf.py +++ b/pvlib/iotools/ecmwf.py @@ -60,7 +60,9 @@ def get_era5(latitude, longitude, start, end, variables, api_key, """ Retrieve ERA5 reanalysis data from the ECMWF's Copernicus Data Store. - This API [1]_ provides a subset of the full ERA5 dataset. See [2]_ for + A CDS API key is needed to access this API. Register for one at [1]_. + + This API [2]_ provides a subset of the full ERA5 dataset. See [3]_ for the available variables. Data are available on a 0.25° x 0.25° grid. Parameters @@ -76,7 +78,7 @@ def get_era5(latitude, longitude, start, end, variables, api_key, variables : list of str List of variable names to retrieve. See [1]_ for options. api_key : str - ECMWF API key. + ECMWF CDS API key. map_variables : bool, default True When true, renames columns of the DataFrame to pvlib variable names where applicable. See variable :const:`VARIABLE_MAP`. @@ -100,8 +102,9 @@ def get_era5(latitude, longitude, start, end, variables, api_key, References ---------- - .. [1] https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-timeseries?tab=overview - .. [2] https://confluence.ecmwf.int/pages/viewpage.action?pageId=505390919 + .. [1] https://cds.climate.copernicus.eu/ + .. [2] https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-timeseries?tab=overview + .. [3] https://confluence.ecmwf.int/pages/viewpage.action?pageId=505390919 """ # noqa: E501 start = pd.to_datetime(start).strftime("%Y-%m-%d") end = pd.to_datetime(end).strftime("%Y-%m-%d") From 396833bb9a5556a4869bdfb6378de9dc7994ff1e Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 13:34:15 -0400 Subject: [PATCH 04/12] handle another API error --- pvlib/iotools/ecmwf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py index a527c70e5b..a5e95f6039 100644 --- a/pvlib/iotools/ecmwf.py +++ b/pvlib/iotools/ecmwf.py @@ -127,6 +127,8 @@ def get_era5(latitude, longitude, start, end, variables, api_key, slug = "processes/reanalysis-era5-single-levels-timeseries/execution" response = requests.post(url + slug, json=params, headers=headers) submission_response = response.json() + if not response.ok: + raise Exception(response.json()) # likely need to accept license job_id = submission_response['jobID'] # Step 2: poll until the data request is ready From 749fd5e5afd434ad6430185dac3841d8a444aef0 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 13:39:37 -0400 Subject: [PATCH 05/12] lint --- pvlib/iotools/ecmwf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py index a5e95f6039..89c7030140 100644 --- a/pvlib/iotools/ecmwf.py +++ b/pvlib/iotools/ecmwf.py @@ -37,6 +37,7 @@ def j_to_w(j): def m_to_cm(m): return m / 100 + UNITS = { 'u100': same, 'v100': same, @@ -128,9 +129,10 @@ def get_era5(latitude, longitude, start, end, variables, api_key, response = requests.post(url + slug, json=params, headers=headers) submission_response = response.json() if not response.ok: - raise Exception(response.json()) # likely need to accept license + raise Exception(submission_response) # likely need to accept license + job_id = submission_response['jobID'] - + # Step 2: poll until the data request is ready slug = "jobs/" + job_id poll_interval = 1 From 8c1ab6dbd1358702e0080cdc924749afb932ccc8 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 16:25:16 -0400 Subject: [PATCH 06/12] fix test --- tests/iotools/test_ecmwf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/iotools/test_ecmwf.py b/tests/iotools/test_ecmwf.py index 3b984f7c84..73dd1baa46 100644 --- a/tests/iotools/test_ecmwf.py +++ b/tests/iotools/test_ecmwf.py @@ -73,6 +73,6 @@ def test_get_era5_error(params): @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_era5_timeout(params): - match = 'Request failed. Please check the ECMWF website' + match = 'Request timed out. Try increasing' with pytest.raises(Exception, match=match): df, meta = pvlib.iotools.get_era5(**params, timeout=1) From 8f4da8896a972a47979e6dca095c4cc19a0f31fb Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 16:43:47 -0400 Subject: [PATCH 07/12] fix tests, again --- pvlib/iotools/ecmwf.py | 9 +++++---- tests/conftest.py | 2 +- tests/iotools/test_ecmwf.py | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py index 89c7030140..f42f874d36 100644 --- a/pvlib/iotools/ecmwf.py +++ b/pvlib/iotools/ecmwf.py @@ -126,7 +126,8 @@ def get_era5(latitude, longitude, start, end, variables, api_key, } } slug = "processes/reanalysis-era5-single-levels-timeseries/execution" - response = requests.post(url + slug, json=params, headers=headers) + response = requests.post(url + slug, json=params, headers=headers, + timeout=timeout) submission_response = response.json() if not response.ok: raise Exception(submission_response) # likely need to accept license @@ -138,7 +139,7 @@ def get_era5(latitude, longitude, start, end, variables, api_key, poll_interval = 1 num_polls = 0 while True: - response = requests.get(url + slug, headers=headers) + response = requests.get(url + slug, headers=headers, timeout=timeout) poll_response = response.json() job_status = poll_response['status'] @@ -162,12 +163,12 @@ def get_era5(latitude, longitude, start, end, variables, api_key, # Step 3: get the download link for our requested dataset slug = "jobs/" + job_id + "/results" - response = requests.get(url + slug, headers=headers) + response = requests.get(url + slug, headers=headers, timeout=timeout) results_response = response.json() download_url = results_response['asset']['value']['href'] # Step 4: finally, download our dataset. it's a zipfile of one CSV - response = requests.get(download_url) + response = requests.get(download_url, timeout=timeout) zipbuffer = BytesIO(response.content) archive = zipfile.ZipFile(zipbuffer) filename = archive.filelist[0].filename diff --git a/tests/conftest.py b/tests/conftest.py index 220e44fa91..6207001c0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -139,7 +139,7 @@ def nrel_api_key(): has_ecmwf_credentials = False requires_ecmwf_credentials = pytest.mark.skipif( - not has_solaranywhere_credentials, + not has_ecmwf_credentials, reason='requires ECMWF credentials') diff --git a/tests/iotools/test_ecmwf.py b/tests/iotools/test_ecmwf.py index 73dd1baa46..719ed19f7d 100644 --- a/tests/iotools/test_ecmwf.py +++ b/tests/iotools/test_ecmwf.py @@ -15,7 +15,7 @@ def params(): return { 'latitude': 40.01, 'longitude': -80.01, - 'start': '2020-06-01', 'end': '2020-06-02', + 'start': '2020-06-01', 'end': '2020-06-01', 'variables': ['ghi', 'temp_air'], 'api_key': api_key, } @@ -41,7 +41,7 @@ def expected(): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_era5(params, expected): df, meta = pvlib.iotools.get_era5(**params) - pd.testing.assert_frame_equal(df, expected, check_freq=False) + pd.testing.assert_frame_equal(df, expected, check_freq=False, atol=0.1) assert meta['longitude'] == -80.0 assert meta['latitude'] == 40.0 assert isinstance(meta['jobID'], str) @@ -53,7 +53,9 @@ def test_get_era5(params, expected): def test_get_era5_map_variables(params, expected): df, meta = pvlib.iotools.get_era5(**params, map_variables=False) expected = expected.rename(columns={'temp_air': 't2m', 'ghi': 'ssrd'}) - pd.testing.assert_frame_equal(df, expected, check_freq=False) + expected['t2m'] += 273.15 # undo unit conversions + expected['ssrd'] *= 3600 + pd.testing.assert_frame_equal(df, expected, check_freq=False, atol=0.1) assert meta['longitude'] == -80.0 assert meta['latitude'] == 40.0 assert isinstance(meta['jobID'], str) From 9906815c660fdbde79ae6b1da0e440bc3701d194 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Fri, 17 Oct 2025 17:55:05 -0400 Subject: [PATCH 08/12] one more --- tests/iotools/test_ecmwf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/iotools/test_ecmwf.py b/tests/iotools/test_ecmwf.py index 719ed19f7d..6fdbc0be57 100644 --- a/tests/iotools/test_ecmwf.py +++ b/tests/iotools/test_ecmwf.py @@ -53,8 +53,8 @@ def test_get_era5(params, expected): def test_get_era5_map_variables(params, expected): df, meta = pvlib.iotools.get_era5(**params, map_variables=False) expected = expected.rename(columns={'temp_air': 't2m', 'ghi': 'ssrd'}) - expected['t2m'] += 273.15 # undo unit conversions - expected['ssrd'] *= 3600 + expected['t2m'] -= 273.15 # apply unit conversions manually + expected['ssrd'] /= 3600 pd.testing.assert_frame_equal(df, expected, check_freq=False, atol=0.1) assert meta['longitude'] == -80.0 assert meta['latitude'] == 40.0 From ee7474d48b749c8a2663e340425d9c2b0549c708 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 20 Oct 2025 09:11:50 -0400 Subject: [PATCH 09/12] use Timeout instead of Exception --- pvlib/iotools/ecmwf.py | 2 +- tests/iotools/test_ecmwf.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py index f42f874d36..a171de7970 100644 --- a/pvlib/iotools/ecmwf.py +++ b/pvlib/iotools/ecmwf.py @@ -154,7 +154,7 @@ def get_era5(latitude, longitude, start, end, variables, api_key, num_polls += 1 if num_polls * poll_interval > timeout: - raise Exception( + raise requests.exceptions.Timeout( 'Request timed out. Try increasing the timeout parameter or ' 'reducing the request size.' ) diff --git a/tests/iotools/test_ecmwf.py b/tests/iotools/test_ecmwf.py index 6fdbc0be57..feacca0aa7 100644 --- a/tests/iotools/test_ecmwf.py +++ b/tests/iotools/test_ecmwf.py @@ -5,6 +5,7 @@ import pandas as pd import pytest import pvlib +import requests import os from tests.conftest import RERUNS, RERUNS_DELAY, requires_ecmwf_credentials @@ -76,5 +77,5 @@ def test_get_era5_error(params): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_era5_timeout(params): match = 'Request timed out. Try increasing' - with pytest.raises(Exception, match=match): + with pytest.raises(requests.exceptions.Timeout, match=match): df, meta = pvlib.iotools.get_era5(**params, timeout=1) From f34309ed87331d8fe2b2006a03102b0da70405a1 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 20 Oct 2025 09:24:31 -0400 Subject: [PATCH 10/12] Apply suggestions from code review Co-authored-by: Adam R. Jensen <39184289+AdamRJensen@users.noreply.github.com> --- docs/sphinx/source/whatsnew/v0.13.2.rst | 2 +- pvlib/iotools/ecmwf.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.13.2.rst b/docs/sphinx/source/whatsnew/v0.13.2.rst index e2134feac3..8df594b643 100644 --- a/docs/sphinx/source/whatsnew/v0.13.2.rst +++ b/docs/sphinx/source/whatsnew/v0.13.2.rst @@ -28,7 +28,7 @@ Enhancements :py:func:`~pvlib.singlediode.bishop88_v_from_i`, and :py:func:`~pvlib.singlediode.bishop88_i_from_v`. (:issue:`2497`, :pull:`2498`) * Add :py:func:`~pvlib.iotools.get_era5`, a function for accessing - ERA-5 reanalysis data. (:pull:`2573`) + ERA5 reanalysis data. (:pull:`2573`) Documentation diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/ecmwf.py index f42f874d36..a29a3852fd 100644 --- a/pvlib/iotools/ecmwf.py +++ b/pvlib/iotools/ecmwf.py @@ -82,7 +82,8 @@ def get_era5(latitude, longitude, start, end, variables, api_key, ECMWF CDS API key. map_variables : bool, default True When true, renames columns of the DataFrame to pvlib variable names - where applicable. See variable :const:`VARIABLE_MAP`. + where applicable. Also converts units of some variables. See variable + :const:`VARIABLE_MAP` and :const:`UNITS`. timeout : int, default 60 Number of seconds to wait for the requested data to become available before timeout. @@ -188,7 +189,7 @@ def get_era5(latitude, longitude, start, end, variables, api_key, if map_variables: # convert units and rename for shortname in df.columns: - converter = UNITS[shortname] + converter = UNITS.get(shortname, same) df[shortname] = converter(df[shortname]) df = df.rename(columns=VARIABLE_MAP) From 3c8f2f2d5b3df791a91ac827e5f54960ed30feaf Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 20 Oct 2025 09:48:56 -0400 Subject: [PATCH 11/12] rename from ECMWF to ERA5 --- pvlib/iotools/__init__.py | 2 +- pvlib/iotools/{ecmwf.py => era5.py} | 0 tests/iotools/{test_ecmwf.py => test_era5.py} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename pvlib/iotools/{ecmwf.py => era5.py} (100%) rename tests/iotools/{test_ecmwf.py => test_era5.py} (98%) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 693ac66d9c..0f54bce232 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -45,4 +45,4 @@ from pvlib.iotools.meteonorm import get_meteonorm_observation_training # noqa: F401, E501 from pvlib.iotools.meteonorm import get_meteonorm_tmy # noqa: F401 from pvlib.iotools.nasa_power import get_nasa_power # noqa: F401 -from pvlib.iotools.ecmwf import get_era5 # noqa: F401 +from pvlib.iotools.era5 import get_era5 # noqa: F401 diff --git a/pvlib/iotools/ecmwf.py b/pvlib/iotools/era5.py similarity index 100% rename from pvlib/iotools/ecmwf.py rename to pvlib/iotools/era5.py diff --git a/tests/iotools/test_ecmwf.py b/tests/iotools/test_era5.py similarity index 98% rename from tests/iotools/test_ecmwf.py rename to tests/iotools/test_era5.py index feacca0aa7..067650c30d 100644 --- a/tests/iotools/test_ecmwf.py +++ b/tests/iotools/test_era5.py @@ -1,5 +1,5 @@ """ -tests for pvlib/iotools/ecmwf.py +tests for pvlib/iotools/era5.py """ import pandas as pd From ac6fe82cf14664ff0f5d14306cedc6b271b9c981 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Mon, 20 Oct 2025 10:07:56 -0400 Subject: [PATCH 12/12] and fix tests --- tests/iotools/test_era5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/iotools/test_era5.py b/tests/iotools/test_era5.py index 067650c30d..c9e1fee39a 100644 --- a/tests/iotools/test_era5.py +++ b/tests/iotools/test_era5.py @@ -54,8 +54,8 @@ def test_get_era5(params, expected): def test_get_era5_map_variables(params, expected): df, meta = pvlib.iotools.get_era5(**params, map_variables=False) expected = expected.rename(columns={'temp_air': 't2m', 'ghi': 'ssrd'}) - expected['t2m'] -= 273.15 # apply unit conversions manually - expected['ssrd'] /= 3600 + df['t2m'] -= 273.15 # apply unit conversions manually + df['ssrd'] /= 3600 pd.testing.assert_frame_equal(df, expected, check_freq=False, atol=0.1) assert meta['longitude'] == -80.0 assert meta['latitude'] == 40.0