Add BSRN reference data (#604)

* start adding template for bsrn sites

* fix ref site csv

* add to api

* add bsrn to ref obs page

* update docs

* mock out bsrn test

* flake8

docs/source/api.rst

@@ -347,6 +347,7 @@ files with site and observation metadata.
    io.reference_observations.arm
    io.reference_observations.pvdaq
    io.reference_observations.eia
+   io.reference_observations.bsrn
 
 SFA API
 =======

docs/source/reference-observations.rst

@@ -47,12 +47,12 @@ Data Files
 Modules
 -------
 * :py:mod:`solarforecastarbiter.io.reference_observations.reference_data`
-   This module coordinates the initialization and update process. It also 
+   This module coordinates the initialization and update process. It also
    contains the `NETWORKHANDLER_MAP` dictionary, which maps network names to
    the correct `Network Handlers`_. The functions in the module are utilized by
    the CLI `referencedata` command.
 
-* :py:mod:`solarforecastarbiter.io.reference_observations.common` 
+* :py:mod:`solarforecastarbiter.io.reference_observations.common`
    The `common` module contains utility functions for use throughout the
    `reference_data` subpackage. It has useful functions for converting
    external data into Solar Forecast Arbiter Datamodel objects and
@@ -72,7 +72,7 @@ The required network handler functions are:
 * `initialize_site_observations(api, site)`
    Create an observation at the site for each variable available from the
    network.
-      
+
       * api: :py:class:`solarforecastarbiter.io.api.APISession`
       * site: :py:class:`solarforecastarbiter.datamodel.Site`
 
@@ -146,9 +146,15 @@ Available Network Handlers
    :py:mod:`solarforecastarbiter.io.reference_observations.pvdaq`
 
 * EIA: U.S. Energy Information Adminstration Open Data\*
-  https://www.eia.gov/opendata/
+   https://www.eia.gov/opendata/
+
+   :py:mod:`solarforecastarbiter.io.reference_observations.eia`
+
+* WRMC BSRN: World Radiation Monitoring Center - Baseline Surface Radiation Network\*
+   https://bsrn.awi.de
+
+   :py:mod:`solarforecastarbiter.io.reference_observations.bsrn`
 
-  :py:mod:`solarforecastarbiter.io.reference_observations.eia`
 
-\* Requesting data from these networks requires a valid api key for their
-associated api.
+\* Requesting data from these networks requires a valid api key or other
+credentials for the associated API.

docs/source/whatsnew/1.0.0rc4.rst

@@ -32,7 +32,8 @@ API Changes
   (:issue:`556`, :pull:`580`)
 * Added `new_index` argument to :py:func:`utils.compute_aggregate` to define
   an index for the computed aggregate. (:issue:`587`)(:pull:`590`)
-
+* Added :py:mod:`io.reference_observations.bsrn` to initialize reference data
+  from the BSRN network (:issue:`541`) (:pull:`604`)
 
 
 Enhancements

solarforecastarbiter/io/reference_observations/bsrn.py

@@ -0,0 +1,81 @@
+import logging
+
+
+from solarforecastarbiter.io.reference_observations import (
+    common, default_forecasts)
+
+
+logger = logging.getLogger('reference_data')
+bsrn_variables = ('ghi', 'dni', 'dhi', 'air_temperature', 'relative_humidity')
+
+
+def initialize_site_observations(api, site):
+    """Creates an observation at the site for each variable in bsrn_variables.
+
+    Parameters
+    ----------
+    site : datamodel.Site
+        The site object for which to create Observations.
+    """
+    for variable in bsrn_variables:
+        common.create_observation(api, site, variable)
+
+
+def initialize_site_forecasts(api, site):
+    """
+    Create forecasts for each variable in bsrn_variables at the site
+
+    Parameters
+    ----------
+    api : solarforecastarbiter.io.api.APISession
+        An active Reference user session.
+    site : datamodel.Site
+        The site object for which to create Forecasts.
+    """
+    common.create_forecasts(api, site, bsrn_variables,
+                            default_forecasts.TEMPLATE_FORECASTS)
+
+
+def fetch(api, site, start, end):
+    """Retrieve observation data for a BSRN site between start and end.
+
+    Parameters
+    ----------
+    api : io.APISession
+        Unused but conforms to common.update_site_observations call
+    site : datamodel.Site
+        Site object with the appropriate metadata.
+    start : datetime
+        The beginning of the period to request data for.
+    end : datetime
+        The end of the period to request data for.
+
+    Returns
+    -------
+    data : pandas.DataFrame
+        All of the requested data concatenated into a single DataFrame.
+    """
+    raise NotImplementedError('Fetching BSRN data not yet implemented')
+
+
+def update_observation_data(api, sites, observations, start, end):
+    """Post new observation data to all BSRN observations from
+    start to end.
+
+    Parameters
+    ----------
+    api : solarforecastarbiter.io.api.APISession
+        An active Reference user session.
+    sites: list
+        List of all reference sites as Objects
+    observations: list of solarforecastarbiter.datamodel.Observation
+        List of all reference observations.
+    start : datetime
+        The beginning of the period to request data for.
+    end : datetime
+        The end of the period to request data for.
+    """
+    bsrn_sites = common.filter_by_networks(sites, ['WRMC BSRN'])
+    for site in bsrn_sites:
+        common.update_site_observations(
+            api, fetch, site, observations, start, end)

solarforecastarbiter/io/reference_observations/reference_data.py

@@ -39,7 +39,8 @@
     common,
     arm,
     pvdaq,
-    eia
+    eia,
+    bsrn
 )
 
 
@@ -54,11 +55,13 @@
     'DOE ARM': arm,
     'NREL PVDAQ': pvdaq,
     'EIA': eia,
+    'WRMC BSRN': bsrn,
 }
 
 # list of options for the 'network' argument
 NETWORK_OPTIONS = ['NOAA SURFRAD', 'NOAA SOLRAD', 'NOAA USCRN', 'NREL MIDC',
-                   'UO SRML', 'DOE RTC', 'DOE ARM', 'NREL PVDAQ', 'EIA']
+                   'UO SRML', 'DOE RTC', 'DOE ARM', 'NREL PVDAQ', 'EIA',
+                   'WRMC BSRN']
 
 DEFAULT_SITEFILE = resource_filename(
     Requirement.parse('solarforecastarbiter'),
@@ -99,6 +102,9 @@
 
 EIA: U.S. Energy Information Administration Open Data
 https://www.eia.gov/opendata/
+
+WRMC BSRN: World Radiation Monitoring Center - Baseline Surface Radiation Network
+https://bsrn.awi.de
 """  # noqa: E501
 
 

solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv

@@ -249,3 +249,5 @@ interval_length,name,latitude,longitude,elevation,network_api_id,network_api_abb
 60,NYISO,42,-75,0,NYIS-ALL,,America/New_York,https://www.eia.gov/opendata/,EIA
 60,PJM,39,-79,0,PJM-ALL,,America/New_York,https://www.eia.gov/opendata/,EIA
 60,Southwest Power Pool,40,-99,0,SWPP-ALL,,America/Chicago,https://www.eia.gov/opendata/,EIA
+1,NASA Langley Research Center,37.1038,-76.3872,3.0,LRC,,Etc/GMT+5,"Driemel, A., Augustine, J., Behrens, K., Colle, S., Cox, C., Cuevas-Agulló, E., Denn, F. M., Duprat, T., Fukuda, M., Grobe, H., Haeffelin, M., Hodges, G., Hyett, N., Ijima, O., Kallis, A., Knap, W., Kustov, V., Long, C. N., Longenecker, D., Lupi, A., Maturilli, M., Mimouni, M., Ntsangwane, L., Ogihara, H., Olano, X., Olefs, M., Omori, M., Passamani, L., Pereira, E. B., Schmithüsen, H., Schumacher, S., Sieger, R., Tamlyn, J., Vogt, R., Vuilleumier, L., Xia, X., Ohmura, A., and König-Langlo, G.: Baseline Surface Radiation Network (BSRN): structure and data description (1992–2017), Earth Syst. Sci. Data, 10, 1491-1501, doi:10.5194/essd-10-1491-2018, 2018.",WRMC BSRN
+1,Granite Island MI,46.721,-87.411,208.0,GIM,,Etc/GMT+6,"Driemel, A., Augustine, J., Behrens, K., Colle, S., Cox, C., Cuevas-Agulló, E., Denn, F. M., Duprat, T., Fukuda, M., Grobe, H., Haeffelin, M., Hodges, G., Hyett, N., Ijima, O., Kallis, A., Knap, W., Kustov, V., Long, C. N., Longenecker, D., Lupi, A., Maturilli, M., Mimouni, M., Ntsangwane, L., Ogihara, H., Olano, X., Olefs, M., Omori, M., Passamani, L., Pereira, E. B., Schmithüsen, H., Schumacher, S., Sieger, R., Tamlyn, J., Vogt, R., Vuilleumier, L., Xia, X., Ohmura, A., and König-Langlo, G.: Baseline Surface Radiation Network (BSRN): structure and data description (1992–2017), Earth Syst. Sci. Data, 10, 1491-1501, doi:10.5194/essd-10-1491-2018, 2018.",WRMC BSRN

solarforecastarbiter/io/reference_observations/tests/test_ref_bsrn.py

@@ -0,0 +1,80 @@
+import pytest
+import re
+import pandas as pd
+
+from solarforecastarbiter.datamodel import Site
+from solarforecastarbiter.io import api
+from solarforecastarbiter.io.reference_observations import bsrn
+
+
+@pytest.fixture
+def session(requests_mock):
+    return api.APISession('')
+
+
+@pytest.fixture
+def site():
+    return Site(
+        name='WRMC BSRN NASA Langley Research Center',
+        latitude=37.1048,
+        longitude=-76.3872,
+        elevation=3.0,
+        timezone='Etc/GMT+5',
+        site_id='',
+        provider='',
+        extra_parameters='{"network_api_id": "LRC", "attribution": "Driemel, A., Augustine, J., Behrens, K., Colle, S., Cox, C., Cuevas-Agull\\u00f3, E., Denn, F. M., Duprat, T., Fukuda, M., Grobe, H., Haeffelin, M., Hodges, G., Hyett, N., Ijima, O., Kallis, A., Knap, W., Kustov, V., Long, C. N., Longenecker, D., Lupi, A., Maturilli, M., Mimouni, M., Ntsangwane, L., Ogihara, H., Olano, X., Olefs, M., Omori, M., Passamani, L., Pereira, E. B., Schmith\\u00fcsen, H., Schumacher, S., Sieger, R., Tamlyn, J., Vogt, R., Vuilleumier, L., Xia, X., Ohmura, A., and K\\u00f6nig-Langlo, G.: Baseline Surface Radiation Network (BSRN): structure and data description (1992\\u20132017), Earth Syst. Sci. Data, 10, 1491-1501, doi:10.5194/essd-10-1491-2018, 2018.", "network": "WRMC BSRN", "network_api_abbreviation": "", "observation_interval_length": 1}',  # noqa: E501
+    )
+
+
+@pytest.fixture()
+def mock_list_sites(mocker, many_sites):
+    mocker.patch('solarforecastarbiter.io.api.APISession.list_sites',
+                 return_value=many_sites)
+
+
+def test_initialize_site_observations(
+        requests_mock, mocker, session, site, single_observation,
+        single_observation_text, mock_list_sites):
+    matcher = re.compile(f'{session.base_url}/observations/.*')
+    requests_mock.register_uri('POST', matcher,
+                               text=single_observation.observation_id)
+    requests_mock.register_uri('GET', matcher, content=single_observation_text)
+    status = mocker.patch(
+        'solarforecastarbiter.io.api.APISession.create_observation')
+    bsrn.initialize_site_observations(session, site)
+    assert status.called
+
+
+def test_initialize_site_obs(mock_api, mocker, site):
+    mocked = mocker.patch(
+        'solarforecastarbiter.io.reference_observations.common.'
+        'create_observation')
+
+    bsrn.initialize_site_observations(mock_api, site)
+    mocked.assert_called()
+
+
+def test_fetch(mocker, session, site):
+    start = pd.Timestamp('2020-01-01T0000Z')
+    end = pd.Timestamp('2020-01-02T0000Z')
+    with pytest.raises(NotImplementedError):
+        bsrn.fetch(session, site, start, end)
+
+
+def test_initialize_site_forecasts(mocker, session, site):
+    mocked = mocker.patch(
+        'solarforecastarbiter.io.reference_observations.common.'
+        'create_forecasts')
+    bsrn.initialize_site_forecasts(session, site)
+    mocked.assert_called()
+
+
+def test_update_observation_data(mocker, session, site):
+    obs_update = mocker.patch(
+        'solarforecastarbiter.io.reference_observations.common.'
+        'update_site_observations')
+    start = pd.Timestamp('20200101T0000Z')
+    end = pd.Timestamp('20200102T0000Z')
+    bsrn.update_observation_data(session, [site], [], start, end)
+    obs_update.assert_called()
+    assert obs_update.call_count == 1