fix report quality flag tables (#602)

* initial attempt

* vertical middle

* fix text

* unique flags filter, pdf works

* rot parbox raggedright

* val results table test

* tabu for the win

* fix preprocessing total discard line

* clarify discard data lines

* remove undefined data string from preprocessing

* unneeded total

* hopefully clarify data_validation_pre

* endless screaming

* edits for clarity

* Update solarforecastarbiter/reports/templates/data_resampling_preamble

Co-authored-by: Tony Lorenzo <atlorenzo@email.arizona.edu>

* fix tests, older report compat

* fix section titles

* whatsnew

* maybe fix the latex issues

* maybe fix latex issue w/ tabu

* move validation and resampling down a level

Co-authored-by: Tony Lorenzo <atlorenzo@email.arizona.edu>

docs/source/whatsnew/1.0.0rc4.rst

@@ -34,6 +34,8 @@ API Changes
   an index for the computed aggregate. (:issue:`587`)(:pull:`590`)
 * Added :py:mod:`io.reference_observations.bsrn` to initialize reference data
   from the BSRN network (:issue:`541`) (:pull:`604`)
+* Removed ``preprocessing.VALIDATION_RESULT_TOTAL_STRING`` and
+  ``preprocessing.UNDEFINED_DATA_STRING``.
 
 
 Enhancements
@@ -58,6 +60,10 @@ Enhancements
   timeseries from a HTML report (:issue:`354`) (:pull:`601`)
 * Add :py:meth:`io.api.APISession.chunk_value_requests` for requesting large
   amounts of data. (:issue:`573`)(:pull:`600`)
+* Restructured and rewrote report data preprocessing text to account for
+  new filter/resample pattern and clarify the processes. Added a table that
+  summarizes the filter combinations applied in the report.
+  (:issue`589`, :pull:`602`)
 
 
 Bug fixes

solarforecastarbiter/conftest.py

@@ -2522,11 +2522,11 @@ def index(category):
 
 @pytest.fixture
 def preprocessing_result_types():
-    return (preprocessing.VALIDATION_RESULT_TOTAL_STRING,
-            'Forecast ' + preprocessing.DISCARD_DATA_STRING,
-            'Forecast ' + preprocessing.UNDEFINED_DATA_STRING,
-            'Observation ' + preprocessing.DISCARD_DATA_STRING,
-            'Observation ' + preprocessing.UNDEFINED_DATA_STRING)
+    return (
+        preprocessing.FILL_RESULT_TOTAL_STRING.format('', 'Discarded'),
+        preprocessing.DISCARD_DATA_STRING.format('Forecast'),
+        preprocessing.DISCARD_DATA_STRING.format('Observation')
+    )
 
 
 @pytest.fixture

solarforecastarbiter/metrics/preprocessing.py

@@ -14,12 +14,10 @@
 logger = logging.getLogger(__name__)
 
 # Titles to refer to counts of preprocessing results
-VALIDATION_RESULT_TOTAL_STRING = "TOTAL FLAGGED VALUES DISCARDED"
-FILL_RESULT_TOTAL_STRING = "Total {0}Forecast Values {1}"
-DISCARD_DATA_STRING = "Values Discarded by Alignment"
-UNDEFINED_DATA_STRING = "Undefined Values"
+FILL_RESULT_TOTAL_STRING = "Missing {0}Forecast Values {1}"
+DISCARD_DATA_STRING = "{0} Values Discarded by Alignment"
 FORECAST_FILL_CONST_STRING = "Filled with {0}"
-FORECAST_FILL_STRING_MAP = {'drop': "Dropped",
+FORECAST_FILL_STRING_MAP = {'drop': "Discarded",
                             'forward': "Forward Filled"}
 
 
@@ -298,6 +296,7 @@ def _calc_discard_before_resample(
     discard_before_resample = obs_flags[discard_before_resample_flags]
     counts = discard_before_resample.astype(int).sum(axis=0).to_dict()
     to_discard_before_resample = discard_before_resample.any(axis=1)
+    counts['TOTAL DISCARD BEFORE RESAMPLE'] = to_discard_before_resample.sum()
 
     # TODO: add filters for time of day and value, OR with
     # to_discard_before_resample, add discarded number to counts
@@ -380,6 +379,8 @@ def apply_flag(quality_flag):
         to_discard_after_resample |= flagged
         counts[filter_name] = flagged.sum()
 
+    counts['TOTAL DISCARD AFTER RESAMPLE'] = to_discard_after_resample.sum()
+
     return to_discard_after_resample, counts
 
 
@@ -503,7 +504,6 @@ def align(fx_obs, fx_data, obs_data, ref_data, tz):
 
     # Align (forecast is unchanged)
     # Remove non-corresponding observations and forecasts, and missing periods
-    undefined_obs = obs_data.isna().sum()
     obs_data = obs_data.dropna(how="any")
     obs_aligned, fx_aligned = obs_data.align(
         fx_data.dropna(how="any"), 'inner')
@@ -526,26 +526,17 @@ def align(fx_obs, fx_data, obs_data, ref_data, tz):
     forecast_values = fx_aligned.tz_convert(tz)
     observation_values = obs_aligned.tz_convert(tz)
 
-    # prob fx DataFrame needs to be summed across both dimensions
-    if isinstance(fx_data, pd.DataFrame):
-        undefined_fx = fx_data.isna().sum().sum()
-    else:
-        undefined_fx = fx_data.isna().sum()
-
     # Return dict summarizing results
+    discarded_fx_intervals = len(fx_data.dropna(how="any")) - len(fx_aligned)
+    discarded_obs_intervals = len(obs_data) - len(observation_values)
+    obs_blurb = "Validated, Resampled " + obs.__blurb__
     results = {
-        fx.__blurb__ + " " + DISCARD_DATA_STRING:
-            len(fx_data.dropna(how="any")) - len(fx_aligned),
-        obs.__blurb__ + " " + DISCARD_DATA_STRING:
-            len(obs_data) - len(observation_values),
-        fx.__blurb__ + " " + UNDEFINED_DATA_STRING:
-            int(undefined_fx),
-        obs.__blurb__ + " " + UNDEFINED_DATA_STRING:
-            int(undefined_obs)
+        DISCARD_DATA_STRING.format(fx.__blurb__): discarded_fx_intervals,
+        DISCARD_DATA_STRING.format(obs_blurb): discarded_obs_intervals
     }
 
     if ref_data is not None:
-        k = type(ref_fx).__name__ + " " + UNDEFINED_DATA_STRING
+        k = DISCARD_DATA_STRING.format("Reference " + ref_fx.__blurb__)
         results[k] = len(ref_data.dropna(how='any')) - len(ref_fx_aligned)
 
     return forecast_values, observation_values, ref_values, results
@@ -738,14 +729,31 @@ def process_forecast_observations(forecast_observations, filters,
         val_results = tuple(datamodel.ValidationResult(flag=k, count=int(v))
                             for k, v in counts.items())
 
-        # this count value no longer makes sense because the first object
-        # is at a different interval than the second.
-        # might need to add a 'total' to counts, exclude from the
-        # ValidationResult comprehension above, and use it here.
-        preproc_obs_results = datamodel.PreprocessingResult(
-            name=VALIDATION_RESULT_TOTAL_STRING,
-            count=(len(data[fxobs.data_object]) - len(observation_values)))
-        preproc_results.append(preproc_obs_results)
+        # the total count ultimately shows up in both the validation
+        # results table and the preprocessing summary table.
+        # use get for compatibility with older reports
+        try:
+            total_discard_before_resample = counts[
+                'TOTAL DISCARD BEFORE RESAMPLE']
+        except KeyError:
+            logging.warning(
+                'TOTAL DISCARD BEFORE RESAMPLE not available for pair '
+                '(%s, %s)', fxobs.forecast.name, fxobs.data_object.name)
+        else:
+            preproc_results.append(datamodel.PreprocessingResult(
+                name='Observation Values Discarded Before Resampling',
+                count=int(total_discard_before_resample)))
+        try:
+            total_discard_after_resample = counts[
+                'TOTAL DISCARD AFTER RESAMPLE']
+        except KeyError:
+            logging.warning(
+                'TOTAL DISCARD AFTER RESAMPLE not available for pair (%s, %s)',
+                fxobs.forecast.name, fxobs.data_object.name)
+        else:
+            preproc_results.append(datamodel.PreprocessingResult(
+                name='Resampled Observation Values Discarded',
+                count=int(total_discard_after_resample)))
 
         # Align and create processed pair
         try:

solarforecastarbiter/metrics/tests/test_preprocessing.py

@@ -76,10 +76,9 @@
 def create_preprocessing_result(counts):
     """Create preprocessing results in order that matches align function."""
     return {
-        "Forecast " + preprocessing.DISCARD_DATA_STRING: counts[0],
-        "Observation " + preprocessing.DISCARD_DATA_STRING: counts[1],
-        "Forecast " + preprocessing.UNDEFINED_DATA_STRING: counts[2],
-        "Observation " + preprocessing.UNDEFINED_DATA_STRING: counts[3],
+        preprocessing.DISCARD_DATA_STRING.format('Forecast'): counts[0],
+        preprocessing.DISCARD_DATA_STRING.format(
+            'Validated, Resampled Observation'): counts[1]
     }
 
 
@@ -88,16 +87,16 @@ def create_preprocessing_result(counts):
 @pytest.mark.parametrize('fx_interval_label',
                          ['beginning', 'ending'])
 @pytest.mark.parametrize('fx_series,obs_series,expected_dt,expected_res', [
-    (THREE_HOUR_SERIES, THREE_HOUR_SERIES, THREE_HOURS, [0]*4),
+    (THREE_HOUR_SERIES, THREE_HOUR_SERIES, THREE_HOURS, [0]*2),
     # document behavior in undesireable case with higher frequency obs data.
-    (THREE_HOUR_SERIES, THIRTEEN_10MIN_SERIES, THREE_HOURS, [0, 10, 0, 0]),
-    (THIRTEEN_10MIN_SERIES, THIRTEEN_10MIN_SERIES, THIRTEEN_10MIN, [0]*4),
-    (THREE_HOUR_SERIES, THREE_HOUR_NAN_SERIES, THREE_HOURS_NAN, [1, 0, 0, 1]),
-    (THREE_HOUR_NAN_SERIES, THREE_HOUR_SERIES, THREE_HOURS_NAN, [0, 1, 1, 0]),
-    (THREE_HOUR_NAN_SERIES, THREE_HOUR_NAN_SERIES, THREE_HOURS_NAN, [0, 0, 1, 1]),  # NOQA
-    (THREE_HOUR_SERIES, THREE_HOUR_EMPTY_SERIES, THREE_HOURS_EMPTY, [3, 0, 0, 0]),  # NOQA
-    (THREE_HOUR_EMPTY_SERIES, THREE_HOUR_SERIES, THREE_HOURS_EMPTY, [0, 3, 0, 0]),  # NOQA
-    (THREE_HOUR_SERIES, EMPTY_OBJ_SERIES, THREE_HOURS_EMPTY, [3, 0, 0, 0]),
+    (THREE_HOUR_SERIES, THIRTEEN_10MIN_SERIES, THREE_HOURS, [0, 10]),
+    (THIRTEEN_10MIN_SERIES, THIRTEEN_10MIN_SERIES, THIRTEEN_10MIN, [0]*2),
+    (THREE_HOUR_SERIES, THREE_HOUR_NAN_SERIES, THREE_HOURS_NAN, [1, 0]),
+    (THREE_HOUR_NAN_SERIES, THREE_HOUR_SERIES, THREE_HOURS_NAN, [0, 1]),
+    (THREE_HOUR_NAN_SERIES, THREE_HOUR_NAN_SERIES, THREE_HOURS_NAN, [0, 0]),
+    (THREE_HOUR_SERIES, THREE_HOUR_EMPTY_SERIES, THREE_HOURS_EMPTY, [3, 0]),
+    (THREE_HOUR_EMPTY_SERIES, THREE_HOUR_SERIES, THREE_HOURS_EMPTY, [0, 3]),
+    (THREE_HOUR_SERIES, EMPTY_OBJ_SERIES, THREE_HOURS_EMPTY, [3, 0]),
 ])
 def test_align(
         site_metadata, obs_interval_label, fx_interval_label, fx_series,
@@ -393,7 +392,9 @@ def test_align_prob_constant_value(
             60, 60, FOUR_HOUR_DF, FOUR_HOUR_SERIES,
             pd.Series([1.], index=pd.DatetimeIndex(["20200301T03Z"])),
             {'NIGHTTIME': 2, 'ISNAN': 1, 'USER FLAGGED': 2,
-             'DISCARD BEFORE RESAMPLE: NIGHTTIME OR USER FLAGGED OR ISNAN': 3}
+             'DISCARD BEFORE RESAMPLE: NIGHTTIME OR USER FLAGGED OR ISNAN': 3,
+             'TOTAL DISCARD BEFORE RESAMPLE': 3,
+             'TOTAL DISCARD AFTER RESAMPLE': 3}
         ),
         (
             (datamodel.QualityFlagFilter(('NIGHTTIME', )),
@@ -402,7 +403,9 @@ def test_align_prob_constant_value(
             pd.Series([1.], index=pd.DatetimeIndex(["20200301T03Z"])),
             {'NIGHTTIME': 2, 'ISNAN': 1, 'USER FLAGGED': 2,
              'DISCARD BEFORE RESAMPLE: NIGHTTIME OR ISNAN': 2,
-             'DISCARD BEFORE RESAMPLE: USER FLAGGED OR ISNAN': 3}
+             'DISCARD BEFORE RESAMPLE: USER FLAGGED OR ISNAN': 3,
+             'TOTAL DISCARD BEFORE RESAMPLE': 3,
+             'TOTAL DISCARD AFTER RESAMPLE': 3}
         ),
         (
             [datamodel.QualityFlagFilter(('NIGHTTIME', 'USER FLAGGED'))],
@@ -411,7 +414,9 @@ def test_align_prob_constant_value(
             # resample_threshold_percentage exceeded
             pd.Series([14.5], index=pd.DatetimeIndex(["20200301T03Z"])),
             {'NIGHTTIME': 3, 'ISNAN': 5, 'USER FLAGGED': 3,
-             'DISCARD BEFORE RESAMPLE: NIGHTTIME OR USER FLAGGED OR ISNAN': 3}
+             'DISCARD BEFORE RESAMPLE: NIGHTTIME OR USER FLAGGED OR ISNAN': 3,
+             'TOTAL DISCARD BEFORE RESAMPLE': 8,
+             'TOTAL DISCARD AFTER RESAMPLE': 3}
         ),
         (
             [datamodel.QualityFlagFilter(
@@ -421,7 +426,9 @@ def test_align_prob_constant_value(
             # only first all-nan interval is discarded
             pd.Series([7.5, 11.5, 14.5], index=FOUR_HOUR_SERIES.index[1:]),
             {'NIGHTTIME': 3, 'ISNAN': 5, 'USER FLAGGED': 3,
-             'DISCARD BEFORE RESAMPLE: NIGHTTIME OR USER FLAGGED OR ISNAN': 1}
+             'DISCARD BEFORE RESAMPLE: NIGHTTIME OR USER FLAGGED OR ISNAN': 1,
+             'TOTAL DISCARD BEFORE RESAMPLE': 8,
+             'TOTAL DISCARD AFTER RESAMPLE': 1}
         ),
         (
             (datamodel.QualityFlagFilter(('NIGHTTIME', )),
@@ -430,7 +437,9 @@ def test_align_prob_constant_value(
             pd.Series([14.5], index=pd.DatetimeIndex(["20200301T03Z"])),
             {'NIGHTTIME': 3, 'ISNAN': 5, 'USER FLAGGED': 3,
              'DISCARD BEFORE RESAMPLE: NIGHTTIME OR ISNAN': 3,
-             'DISCARD BEFORE RESAMPLE: USER FLAGGED OR ISNAN': 3}
+             'DISCARD BEFORE RESAMPLE: USER FLAGGED OR ISNAN': 3,
+             'TOTAL DISCARD BEFORE RESAMPLE': 8,
+             'TOTAL DISCARD AFTER RESAMPLE': 3}
         ),
         (
             [datamodel.QualityFlagFilter(
@@ -439,7 +448,9 @@ def test_align_prob_constant_value(
                 resample_threshold_percentage=30)],
             60, 15, SIXTEEN_15MIN_DF, FOUR_HOUR_SERIES,
             pd.Series([14.5], index=pd.DatetimeIndex(["20200301T03Z"])),
-            {'ISNAN': 5, 'NIGHTTIME OR USER FLAGGED OR ISNAN': 3}
+            {'ISNAN': 5, 'NIGHTTIME OR USER FLAGGED OR ISNAN': 3,
+             'TOTAL DISCARD BEFORE RESAMPLE': 5,
+             'TOTAL DISCARD AFTER RESAMPLE': 3}
         ),
         (
             (datamodel.QualityFlagFilter(
@@ -452,7 +463,9 @@ def test_align_prob_constant_value(
                 resample_threshold_percentage=30)),
             60, 15, SIXTEEN_15MIN_DF, FOUR_HOUR_SERIES,
             pd.Series([10.5, 14.5], index=FOUR_HOUR_SERIES.index[2:]),
-            {'ISNAN': 5, 'NIGHTTIME OR ISNAN': 2, 'USER FLAGGED OR ISNAN': 2}
+            {'ISNAN': 5, 'NIGHTTIME OR ISNAN': 2, 'USER FLAGGED OR ISNAN': 2,
+             'TOTAL DISCARD BEFORE RESAMPLE': 5,
+             'TOTAL DISCARD AFTER RESAMPLE': 2}
         ),
         (
             [datamodel.QualityFlagFilter(
@@ -462,7 +475,9 @@ def test_align_prob_constant_value(
             60, 15, SIXTEEN_15MIN_DF, FOUR_HOUR_SERIES,
             # only first all-nan interval is discarded
             pd.Series([7.0, 10.5, 14.5], index=FOUR_HOUR_SERIES.index[1:]),
-            {'ISNAN': 5, 'NIGHTTIME OR USER FLAGGED OR ISNAN': 0}
+            {'ISNAN': 5, 'NIGHTTIME OR USER FLAGGED OR ISNAN': 0,
+             'TOTAL DISCARD BEFORE RESAMPLE': 5,
+             'TOTAL DISCARD AFTER RESAMPLE': 0}
         ),
         (
             [datamodel.QualityFlagFilter(
@@ -479,7 +494,9 @@ def test_align_prob_constant_value(
             pd.Series([10.5], index=[FOUR_HOUR_SERIES.index[2]]),
             {'ISNAN': 5, 'CLEARSKY EXCEEDED': 1,
              'NIGHTTIME OR USER FLAGGED OR ISNAN': 0,
-             'DISCARD BEFORE RESAMPLE: CLEARSKY EXCEEDED OR ISNAN': 3}
+             'DISCARD BEFORE RESAMPLE: CLEARSKY EXCEEDED OR ISNAN': 3,
+             'TOTAL DISCARD BEFORE RESAMPLE': 6,
+             'TOTAL DISCARD AFTER RESAMPLE': 3}
         ),
     ]
 )

solarforecastarbiter/reports/template.py

@@ -198,6 +198,16 @@ def _figure_name_filter(value):
     return out
 
 
+def _unique_flags_filter(proc_fxobs_list):
+    # use a dict to preserve order
+    names = {}
+    for proc_fxobs in proc_fxobs_list:
+        for val_result in proc_fxobs.validation_results:
+            names[val_result.flag] = None
+    unique_names = list(names.keys())
+    return unique_names
+
+
 def get_template_and_kwargs(report, dash_url, with_timeseries, body_only):
     """Returns the jinja2 Template object and a dict of template variables for
     the report. If the report failed to compute, the template and kwargs will
@@ -233,6 +243,7 @@ def get_template_and_kwargs(report, dash_url, with_timeseries, body_only):
     )
     env.filters['pretty_json'] = _pretty_json
     env.filters['figure_name_filter'] = _figure_name_filter
+    env.filters['unique_flags_filter'] = _unique_flags_filter
     kwargs = _get_render_kwargs(report, dash_url, with_timeseries)
     if report.status == 'complete':
         template = env.get_template('body.html')
@@ -390,6 +401,7 @@ def render_pdf(report, dash_url, max_runs=5):
     env.filters['html_to_tex'] = _html_to_tex
     env.filters['link_filter'] = _link_filter
     env.filters['pretty_json'] = _pretty_json
+    env.filters['unique_flags_filter'] = _unique_flags_filter
     kwargs = _get_render_kwargs(report, dash_url, False)
     with tempfile.TemporaryDirectory() as _tmpdir:
         tmpdir = Path(_tmpdir)

solarforecastarbiter/reports/templates/data_resampling_postamble

@@ -1,5 +1,5 @@
 <p>
-Users may wish to fix the data issues by having new or missing data uploaded.
-The metrics computed in this report will remain unchanged, however, a user may
-generate a new report after the data provider submits new data.
+Users may address data issues by uploading new, corrected, or missing data.
+The metrics computed in this report will remain unchanged unless a user elects
+to recompute the report. Alternatively, a user may generate a new report.
 </p>

solarforecastarbiter/reports/templates/data_resampling_preamble

@@ -1,21 +1,22 @@
-<p>The Solar Forecast Arbiter's preprocessing applies the following
+<p>
+The Solar Forecast Arbiter's preprocessing algorithms applied the following
 operations to the data:
-
+</p>
 <ol>
-  <li>Apply the <a href="#data-validation">data validation tests</a> and exclude
-    the matched data.</li>
-  <li>For deterministic forecasts with interval lengths that are longer than
-    the observations interval lengths,
-    <ol>
-      <li>Resample the observations to the forecast using the mean.</li>
-      <li>Remove resampled observation data if more than 10% of the
-        points in the resampled interval are missing. For example, if 1-minute
-        observations are resampled to 60-minute means, then a 60 minute period must
-        have no more than 6 minutes of missing data.</li>
-    </ol>
-  <li>Remove times that do not exist in both the resampled observations,
-    the forecasts, and, if selected, the reference forecasts.
-  </li>
+  <li>Drop or fill missing forecast and reference forecast data points according to the report's forecast fill method selection.</li>
+  <li>Apply the <a href="#data-validation-resampling">data validation tests</a> specified with <em>discard before resample</em> == True to the observation data and discard
+    the matched data. This procedure is typically used to exclude erroneous observation data.
+    </li>
+  <li>Resample the observation data to the forecast interval length using the mean.</li>
+  <li>Apply the <a href="#data-validation-resampling">data validation tests</a> specified with <em>discard before resample</em> == False to the observation data and discard
+    the resampled intervals where the percentage of matched points exceeds <em>resample threshold percentage</em>.
+    This procedure is typically used to exclude intervals that contain valid but undesirable data, such as the hours of sunrise/sunset or hours that are mostly clear.
+    This step also discards intervals where the percentage of points discarded before resampling exceeds the <em>resample threshold percentage</em>.
+    </li>
+  <li>Align the time series of the resampled, filtered observations, the forecast, and, if selected, the reference forecast.
+    Discard intervals that do not exist in all time series.</li>
 </ol>
-The table below summarizes the data preprocessing results.
+<p>
+The <a href="#data-preprocessing-results-table">table of data preprocessing results</a>
+summarizes the number of points matched by each of these operations.
 </p>

solarforecastarbiter/reports/templates/data_validation_post_text

@@ -1,10 +1,13 @@
-<p>These intervals were removed from the raw time series before
-resampling and realignment. For more details on the data validation
-results for each observation, please see the observation page linked
-to in the table above.</p>
-<p>Data providers may elect to reupload data to
+<p>
+For more details on the data validation results for each observation,
+please see the observation pages linked to in the
+<a href="#observations-and-forecasts">observations and forecasts table</a>.
+</p>
+<p>
+Data providers may elect to reupload data to
 fix issues identified by the validation toolkit. The metrics computed
 in this report will remain unchanged, however, a user may generate a
 new report after the data provider submits new data. The online
 version of this report verifies that the data was not modified after
-the metrics were computed.</p>
+the metrics were computed.
+</p>