diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..32ce6b9a --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,42 @@ +.. currentmodule:: pvanalytics + +############# +API Reference +############# + +Quality +======= + +Irradiance +---------- + +The ``check_*_limits_qcrad`` functions use the QCRad algorithm [1]_ to +identify irradiance measurements that are beyond physical limits. + +.. autosummary:: + :toctree: generated/ + + quality.irradiance.check_ghi_limits_qcrad + quality.irradiance.check_dhi_limits_qcrad + quality.irradiance.check_dni_limits_qcrad + +All three checks can be combined into a single function call. + +.. autosummary:: + :toctree: generated/ + + quality.irradiance.check_irradiance_limits_qcrad + +Irradiance measurements can also be checked for consistency. + +.. autosummary:: + :toctree: generated/ + + quality.irradiance.check_irradiance_consistency_qcrad + +.. rubric:: References + +.. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control + Algorithm for Surface Radiation Measurements, The Open Atmospheric + Science Journal 2, pp. 23-37, 2008. + diff --git a/docs/conf.py b/docs/conf.py index 7597fdf5..b39f9b61 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,9 +10,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('..')) # -- Project information ----------------------------------------------------- @@ -28,8 +28,13 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.napoleon' ] +autosummary_generate = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/index.rst b/docs/index.rst index 720a4fdc..95683a9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,6 +21,9 @@ Contents :maxdepth: 2 :caption: Contents: + api + licenses + Indices and tables diff --git a/docs/licenses.rst b/docs/licenses.rst new file mode 100644 index 00000000..ef3df0b9 --- /dev/null +++ b/docs/licenses.rst @@ -0,0 +1,27 @@ +Licenses +======== + +* The implementation of the QCRad algorithm in + :py:mod:`pvanalytics.quality.irradiance` is derived from `solarforecastarbiter + `_ under the + terms of the MIT License + + Copyright (c) 2019 SolarArbiter + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index fef153ad..67186484 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -1 +1,329 @@ """Quality control functions for irradiance data.""" + +import numpy as np +from pvlib.tools import cosd + + +QCRAD_LIMITS = {'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100}, + 'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50}, + 'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0}, + 'ghi_lb': -4, 'dhi_lb': -4, 'dni_lb': -4} + +QCRAD_CONSISTENCY = { + 'ghi_ratio': { + 'low_zenith': { + 'zenith_bounds': [0.0, 75], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.92, 1.08]}, + 'high_zenith': { + 'zenith_bounds': [75, 93], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.85, 1.15]}}, + 'dhi_ratio': { + 'low_zenith': { + 'zenith_bounds': [0.0, 75], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.0, 1.05]}, + 'high_zenith': { + 'zenith_bounds': [75, 93], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.0, 1.10]}}} + + +def _check_limits(val, lb=None, ub=None, lb_ge=False, ub_le=False): + """Return True where lb < (or <=) val < (or <=) ub.""" + if lb_ge: + lb_op = np.greater_equal + else: + lb_op = np.greater + if ub_le: + ub_op = np.less_equal + else: + ub_op = np.less + + if (lb is not None) & (ub is not None): + return lb_op(val, lb) & ub_op(val, ub) + elif lb is not None: + return lb_op(val, lb) + elif ub is not None: + return ub_op(val, ub) + else: + raise ValueError('must provide either upper or lower bound') + + +def _qcrad_ub(dni_extra, sza, lim): + cosd_sza = cosd(sza) + cosd_sza[cosd_sza < 0] = 0 + return lim['mult'] * dni_extra * cosd_sza**lim['exp'] + lim['min'] + + +def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): + r"""Test for physical limits on GHI using the QCRad criteria. + + Test is applied to each GHI value. A GHI value passes if value > + lower bound and value < upper bound. Lower bounds are constant for + all tests. Upper bounds are calculated as + + .. math:: + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} + + Parameters + ---------- + ghi : Series + Global horizontal irradiance in :math:`W/m^2` + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in :math:`W/m^2` + limits : dict, default QCRAD_LIMITS + Must have keys 'ghi_ub' and 'ghi_lb'. For 'ghi_ub' value is a + dict with keys {'mult', 'exp', 'min'} and float values. For + 'ghi_lb' value is a float. + + Returns + ------- + Series + True where value passes limits test. + + """ + if not limits: + limits = QCRAD_LIMITS + ghi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['ghi_ub']) + + ghi_limit_flag = _check_limits(ghi, limits['ghi_lb'], ghi_ub) + + return ghi_limit_flag + + +def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): + r"""Test for physical limits on DHI using the QCRad criteria. + + Test is applied to each DHI value. A DHI value passes if value > + lower bound and value < upper bound. Lower bounds are constant for + all tests. Upper bounds are calculated as + + .. math:: + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} + + Parameters + ---------- + dhi : Series + Diffuse horizontal irradiance in :math:`W/m^2` + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in :math:`W/m^2` + limits : dict, default QCRAD_LIMITS + Must have keys 'dhi_ub' and 'dhi_lb'. For 'dhi_ub' value is a + dict with keys {'mult', 'exp', 'min'} and float values. For + 'dhi_lb' value is a float. + + Returns + ------- + Series + True where value passes limit test. + + """ + if not limits: + limits = QCRAD_LIMITS + + dhi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dhi_ub']) + + dhi_limit_flag = _check_limits(dhi, limits['dhi_lb'], dhi_ub) + + return dhi_limit_flag + + +def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): + r"""Test for physical limits on DNI using the QCRad criteria. + + Test is applied to each DNI value. A DNI value passes if value > + lower bound and value < upper bound. Lower bounds are constant for + all tests. Upper bounds are calculated as + + .. math:: + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} + + Parameters + ---------- + dni : Series + Direct normal irradiance in :math:`W/m^2` + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in :math:`W/m^2` + limits : dict, default QCRAD_LIMITS + Must have keys 'dni_ub' and 'dni_lb'. For 'dni_ub' value is a + dict with keys {'mult', 'exp', 'min'} and float values. For + 'dni_lb' value is a float. + + Returns + ------- + Series + True where value passes limit test. + + """ + if not limits: + limits = QCRAD_LIMITS + + dni_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dni_ub']) + + dni_limit_flag = _check_limits(dni, limits['dni_lb'], dni_ub) + + return dni_limit_flag + + +def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, + dni=None, limits=None): + r"""Test for physical limits on GHI, DHI or DNI using the QCRad criteria. + + Criteria from [1]_ are used to determine physically plausible + lower and upper bounds. Each value is tested and a value passes if + value > lower bound and value < upper bound. Lower bounds are + constant for all tests. Upper bounds are calculated as + + .. math:: + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} + + .. note:: If any of `ghi`, `dhi`, or `dni` are None, the + corresponding element of the returned tuple will also be None. + + Parameters + ---------- + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in :math:`W/m^2` + ghi : Series or None, default None + Global horizontal irradiance in :math:`W/m^2` + dhi : Series or None, default None + Diffuse horizontal irradiance in :math:`W/m^2` + dni : Series or None, default None + Direct normal irradiance in :math:`W/m^2` + limits : dict, default QCRAD_LIMITS + for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with + keys {'mult', 'exp', 'min'} and float values. For keys + 'ghi_lb', 'dhi_lb', 'dni_lb', value is a float. + + Returns + ------- + ghi_limit_flag : Series + True for each value that is physically possible. None if `ghi` is None. + dhi_limit_flag : Series + True for each value that is physically possible. None if `dni` is None. + dhi_limit_flag : Series + True for each value that is physically possible. None if `dhi` is None. + + References + ---------- + .. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control + Algorithm for Surface Radiation Measurements, The Open Atmospheric + Science Journal 2, pp. 23-37, 2008. + + """ + if not limits: + limits = QCRAD_LIMITS + + if ghi is not None: + ghi_limit_flag = check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, + limits=limits) + else: + ghi_limit_flag = None + + if dhi is not None: + dhi_limit_flag = check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, + limits=limits) + else: + dhi_limit_flag = None + + if dni is not None: + dni_limit_flag = check_dni_limits_qcrad(dni, solar_zenith, dni_extra, + limits=limits) + else: + dni_limit_flag = None + + return ghi_limit_flag, dhi_limit_flag, dni_limit_flag + + +def _get_bounds(bounds): + return (bounds['ghi_bounds'][0], bounds['ghi_bounds'][1], + bounds['zenith_bounds'][0], bounds['zenith_bounds'][1], + bounds['ratio_bounds'][0], bounds['ratio_bounds'][1]) + + +def _check_irrad_ratio(ratio, ghi, sza, bounds): + # unpack bounds dict + ghi_lb, ghi_ub, sza_lb, sza_ub, ratio_lb, ratio_ub = _get_bounds(bounds) + # for zenith set lb_ge to handle edge cases, e.g., zenith=0 + return ((_check_limits(sza, lb=sza_lb, ub=sza_ub, lb_ge=True)) + & (_check_limits(ghi, lb=ghi_lb, ub=ghi_ub)) + & (_check_limits(ratio, lb=ratio_lb, ub=ratio_ub))) + + +def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, + param=None): + """Check consistency of GHI, DHI and DNI using QCRad criteria. + + Uses criteria given in [1]_ to validate the ratio of irradiance + components. + + .. warning:: Not valid for night time. While you can pass data + from night time to this function, be aware that the truth + values returned for that data will not be valid. + + Parameters + ---------- + ghi : Series + Global horizontal irradiance in :math:`W/m^2` + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in :math:`W/m^2` + dhi : Series + Diffuse horizontal irradiance in :math:`W/m^2` + dni : Series + Direct normal irradiance in :math:`W/m^2` + param : dict + keys are 'ghi_ratio' and 'dhi_ratio'. For each key, value is a dict + with keys 'high_zenith' and 'low_zenith'; for each of these keys, + value is a dict with keys 'zenith_bounds', 'ghi_bounds', and + 'ratio_bounds' and value is an ordered pair [lower, upper] + of float. + + Returns + ------- + consistent_components : Series + True where `ghi`, `dhi` and `dni` components are consistent. + diffuse_ratio_limit : Series + True where diffuse to GHI ratio passes limit test. + + References + ---------- + .. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control + Algorithm for Surface Radiation Measurements, The Open Atmospheric + Science Journal 2, pp. 23-37, 2008. + + """ + if not param: + param = QCRAD_CONSISTENCY + + # sum of components + component_sum = dni * cosd(solar_zenith) + dhi + ghi_ratio = ghi / component_sum + dhi_ratio = dhi / ghi + + bounds = param['ghi_ratio'] + consistent_components = ( + _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, + sza=solar_zenith, bounds=bounds['high_zenith']) + | _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, + sza=solar_zenith, bounds=bounds['low_zenith'])) + + bounds = param['dhi_ratio'] + diffuse_ratio_limit = ( + _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, + bounds=bounds['high_zenith']) + | _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, + bounds=bounds['low_zenith'])) + + return consistent_components, diffuse_ratio_limit diff --git a/pvanalytics/tests/__init__.py b/pvanalytics/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pvanalytics/tests/quality/__init__.py b/pvanalytics/tests/quality/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py new file mode 100644 index 00000000..31e800c9 --- /dev/null +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -0,0 +1,126 @@ +"""Tests for irradiance quality control functions.""" +import pandas as pd +import numpy as np + +import pytest +from pandas.util.testing import assert_series_equal + +from pvanalytics.quality import irradiance + + +@pytest.fixture +def irradiance_qcrad(): + """Synthetic irradiance data and its expected quality flags.""" + output = pd.DataFrame( + columns=['ghi', 'dhi', 'dni', 'solar_zenith', 'dni_extra', + 'ghi_limit_flag', 'dhi_limit_flag', 'dni_limit_flag', + 'consistent_components', 'diffuse_ratio_limit'], + data=np.array([[-100, 100, 100, 30, 1370, 0, 1, 1, 0, 0], + [100, -100, 100, 30, 1370, 1, 0, 1, 0, 0], + [100, 100, -100, 30, 1370, 1, 1, 0, 0, 1], + [1000, 100, 900, 0, 1370, 1, 1, 1, 1, 1], + [1000, 200, 800, 15, 1370, 1, 1, 1, 1, 1], + [1000, 200, 800, 60, 1370, 0, 1, 1, 0, 1], + [1000, 300, 850, 80, 1370, 0, 0, 1, 0, 1], + [1000, 500, 800, 90, 1370, 0, 0, 1, 0, 1], + [500, 100, 1100, 0, 1370, 1, 1, 1, 0, 1], + [1000, 300, 1200, 0, 1370, 1, 1, 1, 0, 1], + [500, 600, 100, 60, 1370, 1, 1, 1, 0, 0], + [500, 600, 400, 80, 1370, 0, 0, 1, 0, 0], + [500, 500, 300, 80, 1370, 0, 0, 1, 1, 1], + [0, 0, 0, 93, 1370, 1, 1, 1, 0, 0]])) + dtypes = ['float64', 'float64', 'float64', 'float64', 'float64', + 'bool', 'bool', 'bool', 'bool', 'bool'] + for (col, typ) in zip(output.columns, dtypes): + output[col] = output[col].astype(typ) + return output + + +def test_check_ghi_limits_qcrad(irradiance_qcrad): + """Test that QCRad identifies out of bounds GHI values.""" + expected = irradiance_qcrad + ghi_out_expected = expected['ghi_limit_flag'] + ghi_out = irradiance.check_ghi_limits_qcrad(expected['ghi'], + expected['solar_zenith'], + expected['dni_extra']) + assert_series_equal(ghi_out, ghi_out_expected, check_names=False) + + +def test_check_dhi_limits_qcrad(irradiance_qcrad): + """Test that QCRad identifies out of bounds DHI values.""" + expected = irradiance_qcrad + dhi_out_expected = expected['dhi_limit_flag'] + dhi_out = irradiance.check_dhi_limits_qcrad(expected['dhi'], + expected['solar_zenith'], + expected['dni_extra']) + assert_series_equal(dhi_out, dhi_out_expected, check_names=False) + + +def test_check_dni_limits_qcrad(irradiance_qcrad): + """Test that QCRad identifies out of bounds DNI values.""" + expected = irradiance_qcrad + dni_out_expected = expected['dni_limit_flag'] + dni_out = irradiance.check_dni_limits_qcrad(expected['dni'], + expected['solar_zenith'], + expected['dni_extra']) + assert_series_equal(dni_out, dni_out_expected, check_names=False) + + +def test_check_irradiance_limits_qcrad(irradiance_qcrad): + """Test different input combinations to check_irradiance_limits_qcrad.""" + expected = irradiance_qcrad + ghi_out_expected = expected['ghi_limit_flag'] + ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( + expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi']) + assert_series_equal(ghi_out, ghi_out_expected, check_names=False) + assert dhi_out is None + assert dni_out is None + + dhi_out_expected = expected['dhi_limit_flag'] + ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( + expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi'], + dhi=expected['dhi']) + assert_series_equal(dhi_out, dhi_out_expected, check_names=False) + + dni_out_expected = expected['dni_limit_flag'] + ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( + expected['solar_zenith'], expected['dni_extra'], + dni=expected['dni']) + assert_series_equal(dni_out, dni_out_expected, check_names=False) + + +def test_check_irradiance_consistency_qcrad(irradiance_qcrad): + """Test that QCRad identifies consistent irradiance measurements.""" + expected = irradiance_qcrad + cons_comp, diffuse = irradiance.check_irradiance_consistency_qcrad( + expected['ghi'], expected['solar_zenith'], expected['dni_extra'], + expected['dhi'], expected['dni']) + assert_series_equal(cons_comp, expected['consistent_components'], + check_names=False) + assert_series_equal(diffuse, expected['diffuse_ratio_limit'], + check_names=False) + + +def test_check_limits(): + """Test the private check limits function.""" + expected = pd.Series(data=[True, False]) + data = pd.Series(data=[3, 2]) + result = irradiance._check_limits(val=data, lb=2.5) + assert_series_equal(expected, result, check_names=False) + result = irradiance._check_limits(val=data, lb=3, lb_ge=True) + assert_series_equal(expected, result, check_names=False) + + data = pd.Series(data=[3, 4]) + result = irradiance._check_limits(val=data, ub=3.5) + assert_series_equal(expected, result, check_names=False) + result = irradiance._check_limits(val=data, ub=3, ub_le=True) + assert_series_equal(expected, result, check_names=False) + + result = irradiance._check_limits(val=data, lb=3, ub=4, lb_ge=True, + ub_le=True) + assert all(result) + result = irradiance._check_limits(val=data, lb=3, ub=4) + assert not any(result) + + with pytest.raises(ValueError): + irradiance._check_limits(val=data) diff --git a/requirements.txt b/requirements.txt index 8b137891..8738885c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ - +numpy>=1.16.0 +pandas>=0.25.0 +pvlib>=0.7.0