From 7808fc7e020124d80089bb1583d5298856f7629d Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 18 Feb 2020 15:47:20 -0700 Subject: [PATCH 01/21] Add implementation of QCRad quality tests First cut porting the implementation from solarforeacstarbiter. * Adds qcrad filtering functions to quality.irradiance * Adds tests directory and test module to cover the new irradiance checks. * Adds a licenses.rst file to the documentation to keep track of opensource liscences and copyright notices. --- docs/index.rst | 2 + docs/licenses.rst | 27 ++ pvanalytics/quality/irradiance.py | 322 +++++++++++++++++++ pvanalytics/tests/quality/test_irradiance.py | 93 ++++++ 4 files changed, 444 insertions(+) create mode 100644 docs/licenses.rst create mode 100644 pvanalytics/tests/quality/test_irradiance.py diff --git a/docs/index.rst b/docs/index.rst index 720a4fdc..1daac17a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,6 +21,8 @@ Contents :maxdepth: 2 :caption: Contents: + licenses + Indices and tables diff --git a/docs/licenses.rst b/docs/licenses.rst new file mode 100644 index 00000000..ef3df0b9 --- /dev/null +++ b/docs/licenses.rst @@ -0,0 +1,27 @@ +Licenses +======== + +* The implementation of the QCRad algorithm in + :py:mod:`pvanalytics.quality.irradiance` is derived from `solarforecastarbiter + `_ under the + terms of the MIT License + + Copyright (c) 2019 SolarArbiter + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index fef153ad..93a7e966 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -1 +1,323 @@ """Quality control functions for irradiance data.""" + +import numpy as np +import pandas as pd +from pvlib.tools import cosd +from pvlib.irradiance import clearsky_index +from pvlib.clearsky import detect_clearsky as _detect_clearsky + +QCRAD_LIMITS = {'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100}, + 'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50}, + 'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0}, + 'ghi_lb': -4, 'dhi_lb': -4, 'dni_lb': -4} + +QCRAD_CONSISTENCY = { + 'ghi_ratio': { + 'low_zenith': { + 'zenith_bounds': [0.0, 75], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.92, 1.08]}, + 'high_zenith': { + 'zenith_bounds': [75, 93], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.85, 1.15]}}, + 'dhi_ratio': { + 'low_zenith': { + 'zenith_bounds': [0.0, 75], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.0, 1.05]}, + 'high_zenith': { + 'zenith_bounds': [75, 93], + 'ghi_bounds': [50, np.Inf], + 'ratio_bounds': [0.0, 1.10]}}} + +def _check_limits(val, lb=None, ub=None, lb_ge=False, ub_le=False): + """ Returns True where lb < (or <=) val < (or <=) ub + """ + if lb_ge: + lb_op = np.greater_equal + else: + lb_op = np.greater + if ub_le: + ub_op = np.less_equal + else: + ub_op = np.less + + if (lb is not None) & (ub is not None): + return lb_op(val, lb) & ub_op(val, ub) + elif lb is not None: + return lb_op(val, lb) + elif ub is not None: + return ub_op(val, ub) + else: + raise ValueError('must provide either upper or lower bound') + + +def _qcrad_ub(dni_extra, sza, lim): + cosd_sza = cosd(sza) + cosd_sza[cosd_sza < 0] = 0 + return lim['mult'] * dni_extra * cosd_sza**lim['exp'] + lim['min'] + + +def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): + """Tests for physical limits on GHI using the QCRad criteria. + + Test passes if a value > lower bound and value < upper bound. Lower bounds + are constant for all tests. Upper bounds are calculated as + + .. math:: + ub = min + mult * dni_extra * cos( solar_zenith)^exp + + Parameters + ---------- + ghi : Series + Global horizontal irradiance in W/m^2 + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in W/m^2 + limits : dict, default QCRAD_LIMITS + for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with + keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', + value is a float. + + Returns + ------- + ghi_limit_flag : Series + True if value passes physically-possible test + + """ + if not limits: + limits = QCRAD_LIMITS + ghi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['ghi_ub']) + + ghi_limit_flag = _check_limits(ghi, limits['ghi_lb'], ghi_ub) + ghi_limit_flag.name = 'ghi_limit_flag' + + return ghi_limit_flag + + +def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): + """Tests for physical limits on DHI using the QCRad criteria. + + Test passes if a value > lower bound and value < upper + bound. Lower bounds are constant for all tests. Upper bounds are + calculated as + + .. math:: + ub = min + mult * dni_extra * cos( solar_zenith)^exp + + Parameters + ---------- + dhi : Series + Diffuse horizontal irradiance in W/m^2 + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in W/m^2 + limits : dict, default QCRAD_LIMITS + for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with + keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', + value is a float. + + Returns + ------- + dhi_limit_flag : Series + True if value passes physically-possible test + + """ + if not limits: + limits = QCRAD_LIMITS + + dhi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dhi_ub']) + + dhi_limit_flag = _check_limits(dhi, limits['dhi_lb'], dhi_ub) + dhi_limit_flag.name = 'dhi_limit_flag' + + return dhi_limit_flag + + +def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): + """Tests for physical limits on DNI using the QCRad criteria. + + Test passes if a value > lower bound and value < upper + bound. Lower bounds are constant for all tests. Upper bounds are + calculated as + + .. math:: + ub = min + mult * dni_extra * cos( solar_zenith)^exp + + Parameters + ---------- + dni : Series + Direct normal irradiance in W/m^2 + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in W/m^2 + limits : dict, default QCRAD_LIMITS + for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with + keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', + value is a float. + + Returns + ------- + dni_limit_flag : Series + True if value passes physically-possible test + + """ + if not limits: + limits = QCRAD_LIMITS + + dni_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dni_ub']) + + dni_limit_flag = _check_limits(dni, limits['dni_lb'], dni_ub) + dni_limit_flag.name = 'dni_limit_flag' + + return dni_limit_flag + + +def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, + dni=None, limits=None): + """Tests for physical limits on GHI, DHI or DNI using the QCRad + criteria. + + Test passes if a value > lower bound and value < upper bound. Lower bounds + are constant for all tests. Upper bounds are calculated as + + .. math:: + ub = min + mult * dni_extra * cos( solar_zenith)^exp + + Parameters + ---------- + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in W/m^2 + ghi : Series or None, default None + Global horizontal irradiance in W/m^2 + dhi : Series or None, default None + Diffuse horizontal irradiance in W/m^2 + dni : Series or None, default None + Direct normal irradiance in W/m^2 + limits : dict, default QCRAD_LIMITS + for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with + keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', + value is a float. + + Returns + ------- + ghi_limit_flag : Series or None, default None + True if value passes physically-possible test + dhi_limit_flag : Series or None, default None + dhi_limit_flag : Series or None, default None + + References + ---------- + .. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control + Algorithm for Surface Radiation Measurements, The Open Atmospheric + Science Journal 2, pp. 23-37, 2008. + + """ + if not limits: + limits = QCRAD_LIMITS + + if ghi is not None: + ghi_limit_flag = check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, + limits=limits) + else: + ghi_limit_flag = None + + if dhi is not None: + dhi_limit_flag = check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, + limits=limits) + else: + dhi_limit_flag = None + + if dni is not None: + dni_limit_flag = check_dni_limits_qcrad(dni, solar_zenith, dni_extra, + limits=limits) + else: + dni_limit_flag = None + + return ghi_limit_flag, dhi_limit_flag, dni_limit_flag + + +def _get_bounds(bounds): + return (bounds['ghi_bounds'][0], bounds['ghi_bounds'][1], + bounds['zenith_bounds'][0], bounds['zenith_bounds'][1], + bounds['ratio_bounds'][0], bounds['ratio_bounds'][1]) + + +def _check_irrad_ratio(ratio, ghi, sza, bounds): + # unpack bounds dict + ghi_lb, ghi_ub, sza_lb, sza_ub, ratio_lb, ratio_ub = _get_bounds(bounds) + # for zenith set lb_ge to handle edge cases, e.g., zenith=0 + return ((_check_limits(sza, lb=sza_lb, ub=sza_ub, lb_ge=True)) & + (_check_limits(ghi, lb=ghi_lb, ub=ghi_ub)) & + (_check_limits(ratio, lb=ratio_lb, ub=ratio_ub))) + + +def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, + param=None): + """Checks consistency of GHI, DHI and DNI. Not valid for night time. + + Parameters + ---------- + ghi : Series + Global horizontal irradiance in W/m^2 + solar_zenith : Series + Solar zenith angle in degrees + dni_extra : Series + Extraterrestrial normal irradiance in W/m^2 + dhi : Series + Diffuse horizontal irradiance in W/m^2 + dni : Series + Direct normal irradiance in W/m^2 + param : dict + keys are 'ghi_ratio' and 'dhi_ratio'. For each key, value is a dict + with keys 'high_zenith' and 'low_zenith'; for each of these keys, + value is a dict with keys 'zenith_bounds', 'ghi_bounds', and + 'ratio_bounds' and value is an ordered pair [lower, upper] + of float. + + Returns + ------- + consistent_components : Series + True if ghi, dhi and dni components are consistent. + diffuse_ratio_limit : Series + True if diffuse to ghi ratio passes limit test. + + References + ---------- + .. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control + Algorithm for Surface Radiation Measurements, The Open Atmospheric + Science Journal 2, pp. 23-37, 2008. + + """ + + if not param: + param = QCRAD_CONSISTENCY + + # sum of components + component_sum = dni * cosd(solar_zenith) + dhi + ghi_ratio = ghi / component_sum + dhi_ratio = dhi / ghi + + bounds = param['ghi_ratio'] + consistent_components = ( + _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, + sza=solar_zenith, bounds=bounds['high_zenith']) | + _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, + sza=solar_zenith, bounds=bounds['low_zenith'])) + consistent_components.name = 'consistent_components' + + bounds = param['dhi_ratio'] + diffuse_ratio_limit = ( + _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, + bounds=bounds['high_zenith']) | + _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, + bounds=bounds['low_zenith'])) + diffuse_ratio_limit.name = 'diffuse_ratio_limit' + + return consistent_components, diffuse_ratio_limit diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py new file mode 100644 index 00000000..8180ad19 --- /dev/null +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -0,0 +1,93 @@ +import pandas as pd +import numpy as np + +import pytest +from pandas.util.testing import assert_series_equal + +from pvanalytics.quality import irradiance + + +@pytest.fixture +def irradiance_qcrad(): + output = pd.DataFrame( + columns=['ghi', 'dhi', 'dni', 'solar_zenith', 'dni_extra', + 'ghi_limit_flag', 'dhi_limit_flag', 'dni_limit_flag', + 'consistent_components', 'diffuse_ratio_limit'], + data=np.array([[-100, 100, 100, 30, 1370, 0, 1, 1, 0, 0], + [100, -100, 100, 30, 1370, 1, 0, 1, 0, 0], + [100, 100, -100, 30, 1370, 1, 1, 0, 0, 1], + [1000, 100, 900, 0, 1370, 1, 1, 1, 1, 1], + [1000, 200, 800, 15, 1370, 1, 1, 1, 1, 1], + [1000, 200, 800, 60, 1370, 0, 1, 1, 0, 1], + [1000, 300, 850, 80, 1370, 0, 0, 1, 0, 1], + [1000, 500, 800, 90, 1370, 0, 0, 1, 0, 1], + [500, 100, 1100, 0, 1370, 1, 1, 1, 0, 1], + [1000, 300, 1200, 0, 1370, 1, 1, 1, 0, 1], + [500, 600, 100, 60, 1370, 1, 1, 1, 0, 0], + [500, 600, 400, 80, 1370, 0, 0, 1, 0, 0], + [500, 500, 300, 80, 1370, 0, 0, 1, 1, 1], + [0, 0, 0, 93, 1370, 1, 1, 1, 0, 0]])) + dtypes = ['float64', 'float64', 'float64', 'float64', 'float64', + 'bool', 'bool', 'bool', 'bool', 'bool'] + for (col, typ) in zip(output.columns, dtypes): + output[col] = output[col].astype(typ) + return output + + +def test_check_ghi_limits_qcrad(irradiance_qcrad): + expected = irradiance_qcrad + ghi_out_expected = expected['ghi_limit_flag'] + ghi_out = irradiance.check_ghi_limits_qcrad(expected['ghi'], + expected['solar_zenith'], + expected['dni_extra']) + assert_series_equal(ghi_out, ghi_out_expected) + + +def test_check_dhi_limits_qcrad(irradiance_qcrad): + expected = irradiance_qcrad + dhi_out_expected = expected['dhi_limit_flag'] + dhi_out = irradiance.check_dhi_limits_qcrad(expected['dhi'], + expected['solar_zenith'], + expected['dni_extra']) + assert_series_equal(dhi_out, dhi_out_expected) + + +def test_check_dni_limits_qcrad(irradiance_qcrad): + expected = irradiance_qcrad + dni_out_expected = expected['dni_limit_flag'] + dni_out = irradiance.check_dni_limits_qcrad(expected['dni'], + expected['solar_zenith'], + expected['dni_extra']) + assert_series_equal(dni_out, dni_out_expected) + + +def test_check_irradiance_limits_qcrad(irradiance_qcrad): + expected = irradiance_qcrad + ghi_out_expected = expected['ghi_limit_flag'] + ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( + expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi']) + assert_series_equal(ghi_out, ghi_out_expected) + assert dhi_out is None + assert dni_out is None + + dhi_out_expected = expected['dhi_limit_flag'] + ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( + expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi'], + dhi=expected['dhi']) + assert_series_equal(dhi_out, dhi_out_expected) + + dni_out_expected = expected['dni_limit_flag'] + ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( + expected['solar_zenith'], expected['dni_extra'], + dni=expected['dni']) + assert_series_equal(dni_out, dni_out_expected) + + +def test_check_irradiance_consistency_qcrad(irradiance_qcrad): + expected = irradiance_qcrad + cons_comp, diffuse = irradiance.check_irradiance_consistency_qcrad( + expected['ghi'], expected['solar_zenith'], expected['dni_extra'], + expected['dhi'], expected['dni']) + assert_series_equal(cons_comp, expected['consistent_components']) + assert_series_equal(diffuse, expected['diffuse_ratio_limit']) + From e7e2102c31ef5157306f8f7ae2b2d9b22b7fa4ff Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 25 Feb 2020 13:28:21 -0700 Subject: [PATCH 02/21] Add numpy and pandas to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b137891..f6276d69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ - +numpy>=1.16.0 +pandas>=0.25.0 From 5e36e7bad9b5714c9db442cd7162a28591fad502 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 25 Feb 2020 13:28:56 -0700 Subject: [PATCH 03/21] Fix indentation in test for qcrad --- pvanalytics/tests/quality/test_irradiance.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py index 8180ad19..2ad237b9 100644 --- a/pvanalytics/tests/quality/test_irradiance.py +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -38,8 +38,8 @@ def test_check_ghi_limits_qcrad(irradiance_qcrad): expected = irradiance_qcrad ghi_out_expected = expected['ghi_limit_flag'] ghi_out = irradiance.check_ghi_limits_qcrad(expected['ghi'], - expected['solar_zenith'], - expected['dni_extra']) + expected['solar_zenith'], + expected['dni_extra']) assert_series_equal(ghi_out, ghi_out_expected) @@ -47,8 +47,8 @@ def test_check_dhi_limits_qcrad(irradiance_qcrad): expected = irradiance_qcrad dhi_out_expected = expected['dhi_limit_flag'] dhi_out = irradiance.check_dhi_limits_qcrad(expected['dhi'], - expected['solar_zenith'], - expected['dni_extra']) + expected['solar_zenith'], + expected['dni_extra']) assert_series_equal(dhi_out, dhi_out_expected) @@ -56,8 +56,8 @@ def test_check_dni_limits_qcrad(irradiance_qcrad): expected = irradiance_qcrad dni_out_expected = expected['dni_limit_flag'] dni_out = irradiance.check_dni_limits_qcrad(expected['dni'], - expected['solar_zenith'], - expected['dni_extra']) + expected['solar_zenith'], + expected['dni_extra']) assert_series_equal(dni_out, dni_out_expected) From 8c6bfe62f68295f89b06d095eec65cafc20e5c34 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 25 Feb 2020 13:33:46 -0700 Subject: [PATCH 04/21] Remove unused imports The QCRad implementation does not use pandas directly Clearsky functions are not implemented yet. --- pvanalytics/quality/irradiance.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 93a7e966..ce032c61 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -1,10 +1,7 @@ """Quality control functions for irradiance data.""" import numpy as np -import pandas as pd from pvlib.tools import cosd -from pvlib.irradiance import clearsky_index -from pvlib.clearsky import detect_clearsky as _detect_clearsky QCRAD_LIMITS = {'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100}, 'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50}, From 3d72c4c0e6410e2c2a42deabd6b6ad6684a01d1c Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 25 Feb 2020 13:36:38 -0700 Subject: [PATCH 05/21] Fix whitespace errors - Two lines before functions. - Remove trailing empty lines. --- pvanalytics/quality/irradiance.py | 1 + pvanalytics/tests/quality/test_irradiance.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index ce032c61..343374ef 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -28,6 +28,7 @@ 'ghi_bounds': [50, np.Inf], 'ratio_bounds': [0.0, 1.10]}}} + def _check_limits(val, lb=None, ub=None, lb_ge=False, ub_le=False): """ Returns True where lb < (or <=) val < (or <=) ub """ diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py index 2ad237b9..836fb751 100644 --- a/pvanalytics/tests/quality/test_irradiance.py +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -90,4 +90,3 @@ def test_check_irradiance_consistency_qcrad(irradiance_qcrad): expected['dhi'], expected['dni']) assert_series_equal(cons_comp, expected['consistent_components']) assert_series_equal(diffuse, expected['diffuse_ratio_limit']) - From 4938fb77c643e3101f0f1a8858a1e637e9d210b7 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 25 Feb 2020 14:18:17 -0700 Subject: [PATCH 06/21] Add package structure to tests adds __init__.py files in tests tree to satisfy pytest's package structure requirements. --- pvanalytics/tests/__init__.py | 0 pvanalytics/tests/quality/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 pvanalytics/tests/__init__.py create mode 100644 pvanalytics/tests/quality/__init__.py diff --git a/pvanalytics/tests/__init__.py b/pvanalytics/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pvanalytics/tests/quality/__init__.py b/pvanalytics/tests/quality/__init__.py new file mode 100644 index 00000000..e69de29b From 7c09bf1b1cd8785f320d4418cefa8e3f40d9f15b Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 25 Feb 2020 14:20:07 -0700 Subject: [PATCH 07/21] Add pvlib to dependencies --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f6276d69..8738885c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ numpy>=1.16.0 pandas>=0.25.0 +pvlib>=0.7.0 From 4f4601938a9f16b01e09f55d0357b537688cb5bf Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 27 Feb 2020 08:19:53 -0700 Subject: [PATCH 08/21] Add API documentation Enables the autodoc, autosummary, and napoleon sphinx extensions to generate docs from docstrings. Adds api file to documentation tree with documentation for the qcrad functions in `quality.irradiance` --- docs/api.rst | 42 ++++++++++++++++++++++++++++++++++++++++++ docs/conf.py | 11 ++++++++--- docs/index.rst | 1 + 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 docs/api.rst diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..32ce6b9a --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,42 @@ +.. currentmodule:: pvanalytics + +############# +API Reference +############# + +Quality +======= + +Irradiance +---------- + +The ``check_*_limits_qcrad`` functions use the QCRad algorithm [1]_ to +identify irradiance measurements that are beyond physical limits. + +.. autosummary:: + :toctree: generated/ + + quality.irradiance.check_ghi_limits_qcrad + quality.irradiance.check_dhi_limits_qcrad + quality.irradiance.check_dni_limits_qcrad + +All three checks can be combined into a single function call. + +.. autosummary:: + :toctree: generated/ + + quality.irradiance.check_irradiance_limits_qcrad + +Irradiance measurements can also be checked for consistency. + +.. autosummary:: + :toctree: generated/ + + quality.irradiance.check_irradiance_consistency_qcrad + +.. rubric:: References + +.. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control + Algorithm for Surface Radiation Measurements, The Open Atmospheric + Science Journal 2, pp. 23-37, 2008. + diff --git a/docs/conf.py b/docs/conf.py index 7597fdf5..b39f9b61 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,9 +10,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('..')) # -- Project information ----------------------------------------------------- @@ -28,8 +28,13 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.napoleon' ] +autosummary_generate = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/index.rst b/docs/index.rst index 1daac17a..95683a9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,6 +21,7 @@ Contents :maxdepth: 2 :caption: Contents: + api licenses From a4c5fb79f9d12f5b3bfc0607a0e6bcd111783a92 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 27 Feb 2020 09:10:26 -0700 Subject: [PATCH 09/21] Docstring improvements - Properly format math expressions. - Remove references from summary lines. - Clean up return types for check_irradiance_limits_qcrad - Format notes/warnings with reST directives --- pvanalytics/quality/irradiance.py | 68 +++++++++++++++++++------------ 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 343374ef..f7ee684e 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -3,6 +3,7 @@ import numpy as np from pvlib.tools import cosd + QCRAD_LIMITS = {'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100}, 'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50}, 'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0}, @@ -64,16 +65,16 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): are constant for all tests. Upper bounds are calculated as .. math:: - ub = min + mult * dni_extra * cos( solar_zenith)^exp + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} Parameters ---------- ghi : Series - Global horizontal irradiance in W/m^2 + Global horizontal irradiance in :math:`W/m^2` solar_zenith : Series Solar zenith angle in degrees dni_extra : Series - Extraterrestrial normal irradiance in W/m^2 + Extraterrestrial normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', @@ -103,16 +104,16 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): calculated as .. math:: - ub = min + mult * dni_extra * cos( solar_zenith)^exp + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} Parameters ---------- dhi : Series - Diffuse horizontal irradiance in W/m^2 + Diffuse horizontal irradiance in :math:`W/m^2` solar_zenith : Series Solar zenith angle in degrees dni_extra : Series - Extraterrestrial normal irradiance in W/m^2 + Extraterrestrial normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', @@ -143,16 +144,16 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): calculated as .. math:: - ub = min + mult * dni_extra * cos( solar_zenith)^exp + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} Parameters ---------- dni : Series - Direct normal irradiance in W/m^2 + Direct normal irradiance in :math:`W/m^2` solar_zenith : Series Solar zenith angle in degrees dni_extra : Series - Extraterrestrial normal irradiance in W/m^2 + Extraterrestrial normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', @@ -177,27 +178,31 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, dni=None, limits=None): - """Tests for physical limits on GHI, DHI or DNI using the QCRad - criteria. + """Tests for physical limits on GHI, DHI or DNI using the QCRad criteria. - Test passes if a value > lower bound and value < upper bound. Lower bounds - are constant for all tests. Upper bounds are calculated as + Criteria from [1]_ are used to determine lower and upper bounds + for physically plausible value. Test passes if a value > lower + bound and value < upper bound. Lower bounds are constant for all + tests. Upper bounds are calculated as .. math:: - ub = min + mult * dni_extra * cos( solar_zenith)^exp + ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} + + .. note:: If any of ``ghi``, ``dhi``, or ``dni`` are None, the + corresponding element of the returned tuple will also be None. Parameters ---------- solar_zenith : Series Solar zenith angle in degrees dni_extra : Series - Extraterrestrial normal irradiance in W/m^2 + Extraterrestrial normal irradiance in :math:`W/m^2` ghi : Series or None, default None - Global horizontal irradiance in W/m^2 + Global horizontal irradiance in :math:`W/m^2` dhi : Series or None, default None - Diffuse horizontal irradiance in W/m^2 + Diffuse horizontal irradiance in :math:`W/m^2` dni : Series or None, default None - Direct normal irradiance in W/m^2 + Direct normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', @@ -205,10 +210,12 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, Returns ------- - ghi_limit_flag : Series or None, default None - True if value passes physically-possible test - dhi_limit_flag : Series or None, default None - dhi_limit_flag : Series or None, default None + ghi_limit_flag : Series + True if value is physically possible. None if ``ghi`` is None. + dhi_limit_flag : Series + True if value is physically possible. None if ``dni`` is None. + dhi_limit_flag : Series + True if value is physically possible. None if ``dhi`` is None. References ---------- @@ -258,20 +265,27 @@ def _check_irrad_ratio(ratio, ghi, sza, bounds): def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, param=None): - """Checks consistency of GHI, DHI and DNI. Not valid for night time. + """Checks consistency of GHI, DHI and DNI using QCRad criteria. + + Uses criteria given in [1]_ to validate the ratio of irradiance + components. + + .. warning:: Not valid for night time. While you can pass data + from night time to this function, be aware that the truth + values returned for that data will not be valid. Parameters ---------- ghi : Series - Global horizontal irradiance in W/m^2 + Global horizontal irradiance in :math:`W/m^2` solar_zenith : Series Solar zenith angle in degrees dni_extra : Series - Extraterrestrial normal irradiance in W/m^2 + Extraterrestrial normal irradiance in :math:`W/m^2` dhi : Series - Diffuse horizontal irradiance in W/m^2 + Diffuse horizontal irradiance in :math:`W/m^2` dni : Series - Direct normal irradiance in W/m^2 + Direct normal irradiance in :math:`W/m^2` param : dict keys are 'ghi_ratio' and 'dhi_ratio'. For each key, value is a dict with keys 'high_zenith' and 'low_zenith'; for each of these keys, From 7978dd2747cf0c0860b6d0870182aded02438af1 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 27 Feb 2020 10:51:42 -0700 Subject: [PATCH 10/21] Use raw strings for docstrings with math These docstrings contian a '\' character (escapes the _ in the generated docs) that should be interpreted literally. --- pvanalytics/quality/irradiance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index f7ee684e..465c986b 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -59,7 +59,7 @@ def _qcrad_ub(dni_extra, sza, lim): def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): - """Tests for physical limits on GHI using the QCRad criteria. + r"""Tests for physical limits on GHI using the QCRad criteria. Test passes if a value > lower bound and value < upper bound. Lower bounds are constant for all tests. Upper bounds are calculated as @@ -97,7 +97,7 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): - """Tests for physical limits on DHI using the QCRad criteria. + r"""Tests for physical limits on DHI using the QCRad criteria. Test passes if a value > lower bound and value < upper bound. Lower bounds are constant for all tests. Upper bounds are @@ -137,7 +137,7 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): - """Tests for physical limits on DNI using the QCRad criteria. + r"""Tests for physical limits on DNI using the QCRad criteria. Test passes if a value > lower bound and value < upper bound. Lower bounds are constant for all tests. Upper bounds are @@ -178,7 +178,7 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, dni=None, limits=None): - """Tests for physical limits on GHI, DHI or DNI using the QCRad criteria. + r"""Tests for physical limits on GHI, DHI or DNI using the QCRad criteria. Criteria from [1]_ are used to determine lower and upper bounds for physically plausible value. Test passes if a value > lower From 1fbe77e0d51ac643807535a9d7d2aa63a8bb2371 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 09:57:49 -0700 Subject: [PATCH 11/21] Add test for the internal _check_limits function The qcrad checks do not fully excercise this function on their own. --- pvanalytics/tests/quality/test_irradiance.py | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py index 836fb751..d892bedf 100644 --- a/pvanalytics/tests/quality/test_irradiance.py +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -90,3 +90,28 @@ def test_check_irradiance_consistency_qcrad(irradiance_qcrad): expected['dhi'], expected['dni']) assert_series_equal(cons_comp, expected['consistent_components']) assert_series_equal(diffuse, expected['diffuse_ratio_limit']) + + +def test_check_limits(): + """Test the private check limits function.""" + expected = pd.Series(data=[True, False]) + data = pd.Series(data=[3, 2]) + result = irradiance._check_limits(val=data, lb=2.5) + assert_series_equal(expected, result) + result = irradiance._check_limits(val=data, lb=3, lb_ge=True) + assert_series_equal(expected, result) + + data = pd.Series(data=[3, 4]) + result = irradiance._check_limits(val=data, ub=3.5) + assert_series_equal(expected, result) + result = irradiance._check_limits(val=data, ub=3, ub_le=True) + assert_series_equal(expected, result) + + result = irradiance._check_limits(val=data, lb=3, ub=4, lb_ge=True, + ub_le=True) + assert all(result) + result = irradiance._check_limits(val=data, lb=3, ub=4) + assert not any(result) + + with pytest.raises(ValueError): + irradiance._check_limits(val=data) From 896c50ea347fc1a0c95059a31acb2bbb87c599a2 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 10:03:03 -0700 Subject: [PATCH 12/21] Break lines before binary operators PEP 8 recommends line breaks before binary operators rather than after. --- pvanalytics/quality/irradiance.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 465c986b..0e64317a 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -258,9 +258,9 @@ def _check_irrad_ratio(ratio, ghi, sza, bounds): # unpack bounds dict ghi_lb, ghi_ub, sza_lb, sza_ub, ratio_lb, ratio_ub = _get_bounds(bounds) # for zenith set lb_ge to handle edge cases, e.g., zenith=0 - return ((_check_limits(sza, lb=sza_lb, ub=sza_ub, lb_ge=True)) & - (_check_limits(ghi, lb=ghi_lb, ub=ghi_ub)) & - (_check_limits(ratio, lb=ratio_lb, ub=ratio_ub))) + return ((_check_limits(sza, lb=sza_lb, ub=sza_ub, lb_ge=True)) + & (_check_limits(ghi, lb=ghi_lb, ub=ghi_ub)) + & (_check_limits(ratio, lb=ratio_lb, ub=ratio_ub))) def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, @@ -319,17 +319,17 @@ def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, bounds = param['ghi_ratio'] consistent_components = ( _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, - sza=solar_zenith, bounds=bounds['high_zenith']) | - _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, - sza=solar_zenith, bounds=bounds['low_zenith'])) + sza=solar_zenith, bounds=bounds['high_zenith']) + | _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, + sza=solar_zenith, bounds=bounds['low_zenith'])) consistent_components.name = 'consistent_components' bounds = param['dhi_ratio'] diffuse_ratio_limit = ( _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, - bounds=bounds['high_zenith']) | - _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, - bounds=bounds['low_zenith'])) + bounds=bounds['high_zenith']) + | _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, + bounds=bounds['low_zenith'])) diffuse_ratio_limit.name = 'diffuse_ratio_limit' return consistent_components, diffuse_ratio_limit From 5ab3a064fd0c1b3b41c8b226c691a25ac0f4a475 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 10:32:09 -0700 Subject: [PATCH 13/21] Follow numpydoc rules for refering to parameters --- pvanalytics/quality/irradiance.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 0e64317a..3909706f 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -188,7 +188,7 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, .. math:: ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} - .. note:: If any of ``ghi``, ``dhi``, or ``dni`` are None, the + .. note:: If any of `ghi`, `dhi`, or `dni` are None, the corresponding element of the returned tuple will also be None. Parameters @@ -211,11 +211,11 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, Returns ------- ghi_limit_flag : Series - True if value is physically possible. None if ``ghi`` is None. + True if value is physically possible. None if `ghi` is None. dhi_limit_flag : Series - True if value is physically possible. None if ``dni`` is None. + True if value is physically possible. None if `dni` is None. dhi_limit_flag : Series - True if value is physically possible. None if ``dhi`` is None. + True if value is physically possible. None if `dhi` is None. References ---------- @@ -296,9 +296,9 @@ def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, Returns ------- consistent_components : Series - True if ghi, dhi and dni components are consistent. + True if `ghi`, `dhi` and `dni` components are consistent. diffuse_ratio_limit : Series - True if diffuse to ghi ratio passes limit test. + True if diffuse to GHI ratio passes limit test. References ---------- From 02dbbb526f60409796fad590d48b9529f4075b23 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 10:41:17 -0700 Subject: [PATCH 14/21] Follow PEP 257 guidelines for doc strings. - Use the imperative voice in first line ("test for..." rather than "tests for..."). - Single line doc strings fit on one line (including quotes) - No blank line after function doc string. --- pvanalytics/quality/irradiance.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 3909706f..6628bdaf 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -31,8 +31,7 @@ def _check_limits(val, lb=None, ub=None, lb_ge=False, ub_le=False): - """ Returns True where lb < (or <=) val < (or <=) ub - """ + """Return True where lb < (or <=) val < (or <=) ub.""" if lb_ge: lb_op = np.greater_equal else: @@ -59,7 +58,7 @@ def _qcrad_ub(dni_extra, sza, lim): def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): - r"""Tests for physical limits on GHI using the QCRad criteria. + r"""Test for physical limits on GHI using the QCRad criteria. Test passes if a value > lower bound and value < upper bound. Lower bounds are constant for all tests. Upper bounds are calculated as @@ -97,7 +96,7 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): - r"""Tests for physical limits on DHI using the QCRad criteria. + r"""Test for physical limits on DHI using the QCRad criteria. Test passes if a value > lower bound and value < upper bound. Lower bounds are constant for all tests. Upper bounds are @@ -137,7 +136,7 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): - r"""Tests for physical limits on DNI using the QCRad criteria. + r"""Test for physical limits on DNI using the QCRad criteria. Test passes if a value > lower bound and value < upper bound. Lower bounds are constant for all tests. Upper bounds are @@ -178,7 +177,7 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, dni=None, limits=None): - r"""Tests for physical limits on GHI, DHI or DNI using the QCRad criteria. + r"""Test for physical limits on GHI, DHI or DNI using the QCRad criteria. Criteria from [1]_ are used to determine lower and upper bounds for physically plausible value. Test passes if a value > lower @@ -265,7 +264,7 @@ def _check_irrad_ratio(ratio, ghi, sza, bounds): def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, param=None): - """Checks consistency of GHI, DHI and DNI using QCRad criteria. + """Check consistency of GHI, DHI and DNI using QCRad criteria. Uses criteria given in [1]_ to validate the ratio of irradiance components. @@ -307,7 +306,6 @@ def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, Science Journal 2, pp. 23-37, 2008. """ - if not param: param = QCRAD_CONSISTENCY From 5fdfeeabac17ddfaf224a6f17850a3d49dec2487 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 11:03:22 -0700 Subject: [PATCH 15/21] Add docstrings to irradiance tests. --- pvanalytics/tests/quality/test_irradiance.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py index d892bedf..d101ebd7 100644 --- a/pvanalytics/tests/quality/test_irradiance.py +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -1,3 +1,4 @@ +"""Tests for irradiance quality control functions.""" import pandas as pd import numpy as np @@ -9,6 +10,7 @@ @pytest.fixture def irradiance_qcrad(): + """Synthetic irradiance data and its expected quality flags.""" output = pd.DataFrame( columns=['ghi', 'dhi', 'dni', 'solar_zenith', 'dni_extra', 'ghi_limit_flag', 'dhi_limit_flag', 'dni_limit_flag', @@ -35,6 +37,7 @@ def irradiance_qcrad(): def test_check_ghi_limits_qcrad(irradiance_qcrad): + """Test that QCRad identifies out of bounds GHI values.""" expected = irradiance_qcrad ghi_out_expected = expected['ghi_limit_flag'] ghi_out = irradiance.check_ghi_limits_qcrad(expected['ghi'], @@ -44,6 +47,7 @@ def test_check_ghi_limits_qcrad(irradiance_qcrad): def test_check_dhi_limits_qcrad(irradiance_qcrad): + """Test that QCRad identifies out of bounds DHI values.""" expected = irradiance_qcrad dhi_out_expected = expected['dhi_limit_flag'] dhi_out = irradiance.check_dhi_limits_qcrad(expected['dhi'], @@ -53,6 +57,7 @@ def test_check_dhi_limits_qcrad(irradiance_qcrad): def test_check_dni_limits_qcrad(irradiance_qcrad): + """Test that QCRad identifies out of bounds DNI values.""" expected = irradiance_qcrad dni_out_expected = expected['dni_limit_flag'] dni_out = irradiance.check_dni_limits_qcrad(expected['dni'], @@ -62,6 +67,7 @@ def test_check_dni_limits_qcrad(irradiance_qcrad): def test_check_irradiance_limits_qcrad(irradiance_qcrad): + """Test different input combinations to check_irradiance_limits_qcrad.""" expected = irradiance_qcrad ghi_out_expected = expected['ghi_limit_flag'] ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( @@ -84,6 +90,7 @@ def test_check_irradiance_limits_qcrad(irradiance_qcrad): def test_check_irradiance_consistency_qcrad(irradiance_qcrad): + """Test that QCRad identifies consistent irradiance measurements.""" expected = irradiance_qcrad cons_comp, diffuse = irradiance.check_irradiance_consistency_qcrad( expected['ghi'], expected['solar_zenith'], expected['dni_extra'], From 4b5f6cfbc91fe0462d3d0709097b1dd42e5be55d Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 14:12:28 -0700 Subject: [PATCH 16/21] Limit the description of the limits dict to only relevant keys For check_*_limits_qcrad functions limits only needs to have keys corresponding to the specific measurement being checked (ghi/dhi/dni). This change makes it clear what keys are actually required for each function. Additional keys are allowed (i.e. we use QCRAD_LIMITS as the default for all functions) but are ignored. --- pvanalytics/quality/irradiance.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 6628bdaf..468b1ed4 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -75,9 +75,9 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): dni_extra : Series Extraterrestrial normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS - for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with - keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', - value is a float. + Must have keys 'ghi_ub' and 'ghi_lb'. For 'ghi_ub' value is a + dict with keys {'mult', 'exp', 'min'} and float values. For + 'ghi_lb' value is a float. Returns ------- @@ -114,9 +114,9 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): dni_extra : Series Extraterrestrial normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS - for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with - keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', - value is a float. + Must have keys 'dhi_ub' and 'dhi_lb'. For 'dhi_ub' value is a + dict with keys {'mult', 'exp', 'min'} and float values. For + 'dhi_lb' value is a float. Returns ------- @@ -154,9 +154,9 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): dni_extra : Series Extraterrestrial normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS - for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with - keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', - value is a float. + Must have keys 'dni_ub' and 'dni_lb'. For 'dni_ub' value is a + dict with keys {'mult', 'exp', 'min'} and float values. For + 'dni_lb' value is a float. Returns ------- From c41379317dbd3ed1fd328db1cb6c67ffa3e8fd52 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 14:35:11 -0700 Subject: [PATCH 17/21] Reword and clarify documentation for qcrad functions. Co-authored-by: Cliff Hansen --- pvanalytics/quality/irradiance.py | 41 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 468b1ed4..ccc458f6 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -60,8 +60,9 @@ def _qcrad_ub(dni_extra, sza, lim): def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): r"""Test for physical limits on GHI using the QCRad criteria. - Test passes if a value > lower bound and value < upper bound. Lower bounds - are constant for all tests. Upper bounds are calculated as + Test is applied to each GHI value. A GHI value passes if value > + lower bound and value < upper bound. Lower bounds are constant for + all tests. Upper bounds are calculated as .. math:: ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} @@ -82,7 +83,7 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): Returns ------- ghi_limit_flag : Series - True if value passes physically-possible test + True where value passes limits test. """ if not limits: @@ -98,9 +99,9 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): r"""Test for physical limits on DHI using the QCRad criteria. - Test passes if a value > lower bound and value < upper - bound. Lower bounds are constant for all tests. Upper bounds are - calculated as + Test is applied to each DHI value. A DHI value passes if value > + lower bound and value < upper bound. Lower bounds are constant for + all tests. Upper bounds are calculated as .. math:: ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} @@ -121,7 +122,7 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): Returns ------- dhi_limit_flag : Series - True if value passes physically-possible test + True where value passes limit test. """ if not limits: @@ -138,9 +139,9 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): r"""Test for physical limits on DNI using the QCRad criteria. - Test passes if a value > lower bound and value < upper - bound. Lower bounds are constant for all tests. Upper bounds are - calculated as + Test is applied to each DNI value. A DNI value passes if value > + lower bound and value < upper bound. Lower bounds are constant for + all tests. Upper bounds are calculated as .. math:: ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} @@ -161,7 +162,7 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): Returns ------- dni_limit_flag : Series - True if value passes physically-possible test + True where value passes limit test. """ if not limits: @@ -179,10 +180,10 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, dni=None, limits=None): r"""Test for physical limits on GHI, DHI or DNI using the QCRad criteria. - Criteria from [1]_ are used to determine lower and upper bounds - for physically plausible value. Test passes if a value > lower - bound and value < upper bound. Lower bounds are constant for all - tests. Upper bounds are calculated as + Criteria from [1]_ are used to determine physically plausible + lower and upper bounds. Each value is tested and a value passes if + value > lower bound and value < upper bound. Lower bounds are + constant for all tests. Upper bounds are calculated as .. math:: ub = min + mult * dni\_extra * cos( solar\_zenith)^{exp} @@ -210,11 +211,11 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, Returns ------- ghi_limit_flag : Series - True if value is physically possible. None if `ghi` is None. + True for each value that is physically possible. None if `ghi` is None. dhi_limit_flag : Series - True if value is physically possible. None if `dni` is None. + True for each value that is physically possible. None if `dni` is None. dhi_limit_flag : Series - True if value is physically possible. None if `dhi` is None. + True for each value that is physically possible. None if `dhi` is None. References ---------- @@ -295,9 +296,9 @@ def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, Returns ------- consistent_components : Series - True if `ghi`, `dhi` and `dni` components are consistent. + True where `ghi`, `dhi` and `dni` components are consistent. diffuse_ratio_limit : Series - True if diffuse to GHI ratio passes limit test. + True where diffuse to GHI ratio passes limit test. References ---------- From 4da03483962f46f857478ce97c32423cb1c1fc80 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 14:49:14 -0700 Subject: [PATCH 18/21] Do not set the name of the series returned by qcrad functions The name of the series was used in solarforecastarbiter, but it does not add value here. This removes it so it. Because the name is not set, we also have to update the tests to ignore `Series.name`. --- pvanalytics/quality/irradiance.py | 5 ---- pvanalytics/tests/quality/test_irradiance.py | 24 ++++++++++---------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index ccc458f6..fd6cb992 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -91,7 +91,6 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): ghi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['ghi_ub']) ghi_limit_flag = _check_limits(ghi, limits['ghi_lb'], ghi_ub) - ghi_limit_flag.name = 'ghi_limit_flag' return ghi_limit_flag @@ -131,7 +130,6 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): dhi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dhi_ub']) dhi_limit_flag = _check_limits(dhi, limits['dhi_lb'], dhi_ub) - dhi_limit_flag.name = 'dhi_limit_flag' return dhi_limit_flag @@ -171,7 +169,6 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): dni_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dni_ub']) dni_limit_flag = _check_limits(dni, limits['dni_lb'], dni_ub) - dni_limit_flag.name = 'dni_limit_flag' return dni_limit_flag @@ -321,7 +318,6 @@ def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, sza=solar_zenith, bounds=bounds['high_zenith']) | _check_irrad_ratio(ratio=ghi_ratio, ghi=component_sum, sza=solar_zenith, bounds=bounds['low_zenith'])) - consistent_components.name = 'consistent_components' bounds = param['dhi_ratio'] diffuse_ratio_limit = ( @@ -329,6 +325,5 @@ def check_irradiance_consistency_qcrad(ghi, solar_zenith, dni_extra, dhi, dni, bounds=bounds['high_zenith']) | _check_irrad_ratio(ratio=dhi_ratio, ghi=ghi, sza=solar_zenith, bounds=bounds['low_zenith'])) - diffuse_ratio_limit.name = 'diffuse_ratio_limit' return consistent_components, diffuse_ratio_limit diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py index d101ebd7..fbd1d27a 100644 --- a/pvanalytics/tests/quality/test_irradiance.py +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -43,7 +43,7 @@ def test_check_ghi_limits_qcrad(irradiance_qcrad): ghi_out = irradiance.check_ghi_limits_qcrad(expected['ghi'], expected['solar_zenith'], expected['dni_extra']) - assert_series_equal(ghi_out, ghi_out_expected) + assert_series_equal(ghi_out, ghi_out_expected, check_names=False) def test_check_dhi_limits_qcrad(irradiance_qcrad): @@ -53,7 +53,7 @@ def test_check_dhi_limits_qcrad(irradiance_qcrad): dhi_out = irradiance.check_dhi_limits_qcrad(expected['dhi'], expected['solar_zenith'], expected['dni_extra']) - assert_series_equal(dhi_out, dhi_out_expected) + assert_series_equal(dhi_out, dhi_out_expected, check_names=False) def test_check_dni_limits_qcrad(irradiance_qcrad): @@ -63,7 +63,7 @@ def test_check_dni_limits_qcrad(irradiance_qcrad): dni_out = irradiance.check_dni_limits_qcrad(expected['dni'], expected['solar_zenith'], expected['dni_extra']) - assert_series_equal(dni_out, dni_out_expected) + assert_series_equal(dni_out, dni_out_expected, check_names=False) def test_check_irradiance_limits_qcrad(irradiance_qcrad): @@ -72,7 +72,7 @@ def test_check_irradiance_limits_qcrad(irradiance_qcrad): ghi_out_expected = expected['ghi_limit_flag'] ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi']) - assert_series_equal(ghi_out, ghi_out_expected) + assert_series_equal(ghi_out, ghi_out_expected, check_names=False) assert dhi_out is None assert dni_out is None @@ -80,13 +80,13 @@ def test_check_irradiance_limits_qcrad(irradiance_qcrad): ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi'], dhi=expected['dhi']) - assert_series_equal(dhi_out, dhi_out_expected) + assert_series_equal(dhi_out, dhi_out_expected, check_names=False) dni_out_expected = expected['dni_limit_flag'] ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad( expected['solar_zenith'], expected['dni_extra'], dni=expected['dni']) - assert_series_equal(dni_out, dni_out_expected) + assert_series_equal(dni_out, dni_out_expected, check_names=False) def test_check_irradiance_consistency_qcrad(irradiance_qcrad): @@ -95,8 +95,8 @@ def test_check_irradiance_consistency_qcrad(irradiance_qcrad): cons_comp, diffuse = irradiance.check_irradiance_consistency_qcrad( expected['ghi'], expected['solar_zenith'], expected['dni_extra'], expected['dhi'], expected['dni']) - assert_series_equal(cons_comp, expected['consistent_components']) - assert_series_equal(diffuse, expected['diffuse_ratio_limit']) + assert_series_equal(cons_comp, expected['consistent_components'], check_names=False) + assert_series_equal(diffuse, expected['diffuse_ratio_limit'], check_names=False) def test_check_limits(): @@ -104,15 +104,15 @@ def test_check_limits(): expected = pd.Series(data=[True, False]) data = pd.Series(data=[3, 2]) result = irradiance._check_limits(val=data, lb=2.5) - assert_series_equal(expected, result) + assert_series_equal(expected, result, check_names=False) result = irradiance._check_limits(val=data, lb=3, lb_ge=True) - assert_series_equal(expected, result) + assert_series_equal(expected, result, check_names=False) data = pd.Series(data=[3, 4]) result = irradiance._check_limits(val=data, ub=3.5) - assert_series_equal(expected, result) + assert_series_equal(expected, result, check_names=False) result = irradiance._check_limits(val=data, ub=3, ub_le=True) - assert_series_equal(expected, result) + assert_series_equal(expected, result, check_names=False) result = irradiance._check_limits(val=data, lb=3, ub=4, lb_ge=True, ub_le=True) From 8c5eb987709a8e16e0136fa537bd79944837e946 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 14:56:12 -0700 Subject: [PATCH 19/21] Fix too long lines in tests Forgot to run the linter locally after updating the test suite. --- pvanalytics/tests/quality/test_irradiance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pvanalytics/tests/quality/test_irradiance.py b/pvanalytics/tests/quality/test_irradiance.py index fbd1d27a..31e800c9 100644 --- a/pvanalytics/tests/quality/test_irradiance.py +++ b/pvanalytics/tests/quality/test_irradiance.py @@ -95,8 +95,10 @@ def test_check_irradiance_consistency_qcrad(irradiance_qcrad): cons_comp, diffuse = irradiance.check_irradiance_consistency_qcrad( expected['ghi'], expected['solar_zenith'], expected['dni_extra'], expected['dhi'], expected['dni']) - assert_series_equal(cons_comp, expected['consistent_components'], check_names=False) - assert_series_equal(diffuse, expected['diffuse_ratio_limit'], check_names=False) + assert_series_equal(cons_comp, expected['consistent_components'], + check_names=False) + assert_series_equal(diffuse, expected['diffuse_ratio_limit'], + check_names=False) def test_check_limits(): From 011b2340b859495638bb276584fbae59adedc59e Mon Sep 17 00:00:00 2001 From: Will Vining Date: Thu, 5 Mar 2020 15:06:38 -0700 Subject: [PATCH 20/21] Note type of dict values in doc for check_irradiance_limits_qcrad --- pvanalytics/quality/irradiance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index fd6cb992..6e88d4e7 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -202,8 +202,8 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None, Direct normal irradiance in :math:`W/m^2` limits : dict, default QCRAD_LIMITS for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with - keys {'mult', 'exp', 'min'}. For keys 'ghi_lb', 'dhi_lb', 'dni_lb', - value is a float. + keys {'mult', 'exp', 'min'} and float values. For keys + 'ghi_lb', 'dhi_lb', 'dni_lb', value is a float. Returns ------- From a8ac82a793801104f742842aadf73f099d4f09b9 Mon Sep 17 00:00:00 2001 From: Will Vining Date: Tue, 10 Mar 2020 09:55:16 -0600 Subject: [PATCH 21/21] Remove name of return value from docstrings Including the name doesn't add any information for functions that return only one value. --- pvanalytics/quality/irradiance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvanalytics/quality/irradiance.py b/pvanalytics/quality/irradiance.py index 6e88d4e7..67186484 100644 --- a/pvanalytics/quality/irradiance.py +++ b/pvanalytics/quality/irradiance.py @@ -82,7 +82,7 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None): Returns ------- - ghi_limit_flag : Series + Series True where value passes limits test. """ @@ -120,7 +120,7 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None): Returns ------- - dhi_limit_flag : Series + Series True where value passes limit test. """ @@ -159,7 +159,7 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None): Returns ------- - dni_limit_flag : Series + Series True where value passes limit test. """