diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index cf6fbf4ac..39081220f 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -26,6 +26,7 @@ of sources and file formats relevant to solar energy modeling. iotools.read_midc_raw_data_from_nrel iotools.read_crn iotools.read_solrad + iotools.get_solrad iotools.get_psm3 iotools.read_psm3 iotools.parse_psm3 diff --git a/docs/sphinx/source/whatsnew/v0.10.4.rst b/docs/sphinx/source/whatsnew/v0.10.4.rst index 3f31123c0..c1c977b71 100644 --- a/docs/sphinx/source/whatsnew/v0.10.4.rst +++ b/docs/sphinx/source/whatsnew/v0.10.4.rst @@ -8,6 +8,8 @@ v0.10.4 (Anticipated March, 2024) Enhancements ~~~~~~~~~~~~ * Added the Huld PV model used by PVGIS (:pull:`1940`) +* Added :py:func:`~pvlib.iotools.get_solrad` for fetching irradiance data from + the SOLRAD ground station network. (:pull:`1967`) * Added metadata parsing to :py:func:`~pvlib.iotools.read_solrad` to follow the standard iotools convention of returning a tuple of (data, meta). Previously the function only returned a dataframe. (:pull:`1968`) @@ -52,4 +54,5 @@ Contributors * Cliff Hansen (:ghuser:`cwhanse`) * :ghuser:`matsuobasho` * Adam R. Jensen (:ghuser:`AdamRJensen`) +* Kevin Anderson (:ghuser:`kandersolar`) * Peter Dudfield (:ghuser:`peterdudfield`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 2ec1753eb..0fbec16c1 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -8,6 +8,7 @@ from pvlib.iotools.midc import read_midc_raw_data_from_nrel # noqa: F401 from pvlib.iotools.crn import read_crn # noqa: F401 from pvlib.iotools.solrad import read_solrad # noqa: F401 +from pvlib.iotools.solrad import get_solrad # noqa: F401 from pvlib.iotools.psm3 import get_psm3 # noqa: F401 from pvlib.iotools.psm3 import read_psm3 # noqa: F401 from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 diff --git a/pvlib/iotools/solrad.py b/pvlib/iotools/solrad.py index 6d9dde743..90bf5b666 100644 --- a/pvlib/iotools/solrad.py +++ b/pvlib/iotools/solrad.py @@ -1,6 +1,7 @@ """Functions to read data from the NOAA SOLRAD network.""" import pandas as pd +import warnings import requests import io @@ -72,6 +73,10 @@ def read_solrad(filename): metadata : dict Metadata. + See Also + -------- + get_solrad + Notes ----- SOLRAD data resolution is described by the README_SOLRAD.txt: @@ -104,6 +109,7 @@ def read_solrad(filename): if str(filename).startswith('ftp') or str(filename).startswith('http'): response = requests.get(filename) + response.raise_for_status() file_buffer = io.StringIO(response.content.decode()) else: with open(str(filename), 'r') as file_buffer: @@ -135,3 +141,81 @@ def read_solrad(filename): data = data.set_index(dtindex) return data, meta + + +def get_solrad(station, start, end, + url="https://gml.noaa.gov/aftp/data/radiation/solrad/"): + """Request data from NOAA SOLRAD and read it into a Dataframe. + + A list of stations and their descriptions can be found in [1]_, + The data files are described in [2]_. + + Data is returned for complete days, including ``start`` and ``end``. + + Parameters + ---------- + station : str + Three letter station abbreviation. + start : datetime-like + First day of the requested period + end : datetime-like + Last day of the requested period + url : str, default: 'https://gml.noaa.gov/aftp/data/radiation/solrad/' + API endpoint URL + + Returns + ------- + data : pd.DataFrame + Dataframe with data from SOLRAD. + meta : dict + Metadata. + + See Also + -------- + read_solrad + + Notes + ----- + Recent SOLRAD data is 1-minute averages. Prior to 2015-01-01, it was + 3-minute averages. + + References + ---------- + .. [1] https://gml.noaa.gov/grad/solrad/index.html + .. [2] https://gml.noaa.gov/aftp/data/radiation/solrad/README_SOLRAD.txt + + Examples + -------- + >>> # Retrieve one month of irradiance data from the ABQ SOLRAD station + >>> data, metadata = pvlib.iotools.get_solrad( + >>> station='abq', start="2020-01-01", end="2020-01-31") + """ + # Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted + start = pd.to_datetime(start) + end = pd.to_datetime(end) + + # Generate list of filenames + dates = pd.date_range(start.floor('d'), end, freq='d') + station = station.lower() + filenames = [ + f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat" + for d in dates + ] + + dfs = [] # Initialize list of monthly dataframes + for f in filenames: + try: + dfi, file_metadata = read_solrad(url + f) + dfs.append(dfi) + except requests.exceptions.HTTPError: + warnings.warn(f"The following file was not found: {f}") + + data = pd.concat(dfs, axis='rows') + + meta = {'station': station, + 'filenames': filenames, + # all file should have the same metadata, so just merge in the + # metadata from the last file + **file_metadata} + + return data, meta diff --git a/pvlib/tests/iotools/test_solrad.py b/pvlib/tests/iotools/test_solrad.py index abfa5d6e3..963014302 100644 --- a/pvlib/tests/iotools/test_solrad.py +++ b/pvlib/tests/iotools/test_solrad.py @@ -117,3 +117,36 @@ def test_read_solrad_https(): remote_data, _ = solrad.read_solrad(https_testfile) # local file only contains four rows to save space assert_frame_equal(local_data, remote_data.iloc[:4]) + + +@pytest.mark.remote_data +@pytest.mark.parametrize('testfile, station', [ + (testfile, 'abq'), + (testfile_mad, 'msn'), +]) +def test_get_solrad(testfile, station): + df, meta = solrad.get_solrad(station, "2019-02-25", "2019-02-25") + + assert meta['station'] == station + assert isinstance(meta['filenames'], list) + + assert len(df) == 1440 + assert df.index[0] == pd.to_datetime('2019-02-25 00:00+00:00') + assert df.index[-1] == pd.to_datetime('2019-02-25 23:59+00:00') + + expected, _ = solrad.read_solrad(testfile) + actual = df.reindex(expected.index) + # ABQ test file has an unexplained NaN in row 4; just verify first 3 rows + assert_frame_equal(actual.iloc[:3], expected.iloc[:3]) + + +@pytest.mark.remote_data +def test_get_solrad_missing_day(): + # data availability begins for ABQ on 2002-02-01 (DOY 32), so requesting + # data before that will raise a warning + message = 'The following file was not found: abq/2002/abq02031.dat' + with pytest.warns(UserWarning, match=message): + df, meta = solrad.get_solrad('abq', '2002-01-31', '2002-02-01') + + # but the data for 2022-02-01 is still returned + assert not df.empty