[DOING] renier waterbalance enable current interception (#300)

[DONE] renier waterbalance enable current interception

CHANGES.rst

@@ -4,7 +4,9 @@ ThreeDiToolBox changelog
 
 1.4 (unreleased)
 ----------------
-- Upgrade threedigrid from 0.2.6 to 1.0.6 (latest version)
+- Enable intercepted_volume through aggregation NetCDF
+
+- Upgrade threedigrid from 0.2.6 to 1.0.7 (current latest version)
 
 - Add '(de)activate all layers' buttons in Waterbalancetool
 

datasource/netcdf_groundwater_h5py.py

@@ -176,23 +176,14 @@ def available_aggregation_vars(self):
             # all available fields, including hdf5 fields
             available_vars = (agg.nodes._field_names | agg.lines._field_names)
 
-        # TODO: fix infiltration and leakage aggregation vars in threedigrid
+        # TODO: fix infiltration, leakage, and interception aggregation vars
+        # in threedigrid's agg.nodes.Meta.composite_fields.keys()
         known_vars.add('infiltration_rate_simple_cum')
         known_vars.add('leak_cum')
+        known_vars.add('intercepted_volume_current')
 
         available_known_vars = available_vars & known_vars
 
-        # renier: dit kan nu weg toch?
-        # # TODO: this is a bug in threedigrid:
-        # # https://nelen-schuurmans.atlassian.net/browse/THREEDI-486
-        # # until it is fixed, we add simple_infiltration to known_vars
-        # if agg.has_simple_infiltration:
-        #     available_known_vars.add(str('infiltration_rate_simple_cum'))
-        # # TODO: a simulation with groundwater does not have leakage per-se
-        # # (only when leakage is forced (global or raster) so
-        # # agg.has_groundwater is not bullet-proof
-        # if agg.has_groundwater:
-        #     available_known_vars.add(str('leak_cum'))
         return list(available_known_vars)
 
     def get_available_variables(self):

requirements.txt

@@ -2,4 +2,4 @@ git+https://github.com/nens/geoalchemy2.git#egg=geoalchemy2
 SQLAlchemy==1.1.0b3
 pyqtgraph==0.9.10
 git+https://github.com/lizardsystem/lizard-connector.git@0.6#egg=lizard_connector
-threedigrid==1.0.6
+threedigrid==1.0.7

water_balance/config/waterbalance/sum_configs.py

@@ -137,6 +137,13 @@
             'def_pen_color': '121,160,191' + pen_transp,
             'series': ['2d_vertical_infiltration_pos',
                        '2d_vertical_infiltration_neg'],
+        }, {
+            'name': 'intercepted volume',
+            'default_method': 'net',
+            'order': 10.7,
+            'def_fill_color': '181,60,221' + fill_transp,
+            'def_pen_color': '181,60,221' + pen_transp,
+            'series': ['intercepted_volume'],
         }, {
             'name': 'inflow 1d from rain',
             'default_method': 'net',
@@ -208,6 +215,13 @@
             'def_fill_color': '0,0,128' + fill_transp,
             'def_pen_color': '0,0,128' + pen_transp,
             'series': ['2d_groundwater_in', '2d_groundwater_out'],
+        }, {
+            'name': 'intercepted volume',
+            'default_method': 'net',
+            'order': 1.6,
+            'def_fill_color': '181,60,221' + fill_transp,
+            'def_pen_color': '181,60,221' + pen_transp,
+            'series': ['intercepted_volume'],
         }, {
             'name': 'leakage',
             'default_method': 'net',

water_balance/tools/waterbalance.py

@@ -735,23 +735,25 @@ def get_aggregated_flows(self, link_ids, pump_ids, node_ids, model_part):
             # NOTE: can be a source or sink depending on sign
             ('leak', np_2d_groundwater_node, 26, 1),
             ('rain', np_1d_node, 27, 1),
+            ('intercepted_volume', np_2d_node, 34, -1),
         ]:
 
             if node.size > 0:
-                skip = False
-                if parameter + '_cum' not in ds.get_available_variables():
-                    skip = True
-                    log.warning('%s_cum not available! skip it', parameter)
-                    # todo: fallback on not aggregated version
+                skip = True
+                if parameter + '_cum' in ds.get_available_variables():
+                    addition = '_cum'
+                    skip = False
+                elif parameter + '_current' in ds.get_available_variables():
+                    addition = '_current'
+                    skip = False
                 if not skip:
                     values_pref = 0
                     for ts_idx, t in enumerate(ts):
                         values = ds.get_values_by_timestep_nr(
-                            parameter + '_cum', ts_idx, node).sum()  # * dt
+                            parameter + addition, ts_idx, node).sum()  # * dt
                         values_dt = values - values_pref
                         values_pref = values
                         total_time[ts_idx, pnr] = values_dt * factor
-
         t_pref = 0
 
         for ts_idx, t in enumerate(ts):
@@ -874,45 +876,58 @@ def pop_up_no_agg_found(self):
               "but this file could not be found. Please make sure you run " \
               "your simulation using the 'v2_aggregation_settings' table " \
               "with the following variables:" \
+              "\n\ncurrent:" \
+              "\n- volume" \
+              "\n- interception (in case model has interception)" \
               "\n\ncumulative:"\
               "\n- rain"\
               "\n- discharge"\
               "\n- leakage (in case model has leakage)" \
               "\n- laterals (in case model has laterals)"\
               "\n- pump discharge (in case model has pumps)" \
-              "\n- interception (in case model has interception)" \
-              "\n- infiltration (in case model has (simple_)infiltration)"\
-              "\n\npositive cumulative:\n- discharge"\
-              "\n\nnegative cumulative:\n- discharge"\
-              "\n\nNOTE: for 100% closure of the balance, please make sure " \
-              "that the v2_aggregation_settings' timestep for all " \
-              "aggregation variables is a multiple of the v2_global_settings'"\
-              " output_time_step"
+              "\n- simple_infiltration (in case model has simple_infiltration"\
+              "\n\npositive cumulative:" \
+              "\n- discharge"\
+              "\n\nnegative cumulative:" \
+              "\n- discharge"
         QMessageBox.warning(None, header, msg)
 
     def pop_up_missing_agg_vars(self):
         header = 'Error: Missing aggregation settings'
         missing_vars = self.get_missing_agg_vars()
-        msg = "The WaterBalanceTool found the 'aggregate_results_3di.nc' but" \
-              " the file does not include all required aggregation " \
+        msg = "The WaterBalanceTool found the 'aggregate_results_3di.nc' but "\
+              "the file does not include all required aggregation " \
               "variables. Please add them to the sqlite table " \
               "'v2_aggregation_settings' and run your simulation again. The " \
               "required variables are:" \
+              "\n\ncurrent:" \
+              "\n- volume" \
+              "\n- interception (in case model has interception)" \
               "\n\ncumulative:"\
               "\n- rain"\
               "\n- discharge"\
               "\n- leakage (in case model has leakage)" \
               "\n- laterals (in case model has laterals)"\
               "\n- pump discharge (in case model has pumps)" \
-              "\n- interception (in case model has interception)" \
-              "\n- infiltration (in case model has (simple_)infiltration)"\
-              "\n\npositive cumulative:\n- discharge"\
-              "\n\nnegative cumulative:\n- discharge" \
-              "\n\nYour aggregation .nc misses the following variables: " + \
+              "\n- simple_infiltration (in case model has simple_infiltration"\
+              "\n\npositive cumulative:" \
+              "\n- discharge"\
+              "\n\nnegative cumulative:" \
+              "\n- discharge" \
+              "\n\nYour aggregation .nc misses the following variables:\n" + \
               ', '.join(missing_vars)
         QMessageBox.warning(None, header, msg)
 
     def get_missing_agg_vars(self):
+        """Returns a list with tuples of aggregation vars (vol, discharge) +
+        methods (cum, current, etc) that are not (but should be) in the
+        v2_aggregation_settings
+
+        1.  some vars_methods are always required: minimum_agg_vars
+        2.  some vars methods are required when included in the model
+            schematisation (e.g. pumps, laterals).
+        """
+
         selected_ds = self.ts_datasource.rows[0].datasource()
         check_available_vars = selected_ds.get_available_variables()
         nc_path = self.ts_datasource.rows[0].datasource().file_path
@@ -926,28 +941,41 @@ def get_missing_agg_vars(self):
             ('vol_current', 'current volume')
             ]
 
+        # some vars must be aggregated when included in the model
+        # schematisation (e.g. pumps, laterals). problem is that threedigrid
+        # does not support e.g. ga.has_lateral, ga.has_leakage etc. For those
+        # fields, we read the threedigrid metadata.
+        simulated_vars_nodes = ga.nodes._meta.get_fields(only_names=True)
+
         if ga.has_pumpstations:
             to_add = ('q_pump_cum', 'cumulative pump discharge')
             minimum_agg_vars.append(to_add)
 
-        if ga.has_simple_infiltration:
-            to_add = ('infiltration_rate_simple_cum',
-                      'cumulative infiltration rate')
-            minimum_agg_vars.append(to_add)
-
-        if ga.has_interception:
-            to_add = ('intercepted_volume', 'cumulative interception')
-            minimum_agg_vars.append(to_add)
-
-        # we cant check whether a simulation used rain (since this depends on
-        # user action during simulation). Moreover, threedigrid does not
-        # support e.g. ga.has_lateral, ga.has_leakage etc. For those fields,
-        # we read the threedigrid metadata
-        simulated_vars_nodes = ga.nodes._meta.get_fields(only_names=True)
-
-        # we do not check whether vertical_infiltration must be added to the
-        # minimum_agg_vars since this flow is covered in 'q_cum_negative',
-        # 'q_cum_positive', and 'q_cum'.
+        # ga.has_simple_infiltration and ga.has_interception are added to
+        # threedigrid some months after groundwater release. To coop with the
+        # .h5 that has been created in that period we use the meta data
+        try:
+            if ga.has_simple_infiltration:
+                to_add = ('infiltration_rate_simple_cum',
+                          'cumulative infiltration rate')
+                minimum_agg_vars.append(to_add)
+        except AttributeError:
+            if 'infiltration' in simulated_vars_nodes:
+                to_add = ('infiltration_rate_simple_cum',
+                          'cumulative infiltration rate')
+                minimum_agg_vars.append(to_add)
+
+        try:
+            if ga.has_interception:
+                to_add = ('intercepted_volume', 'current interception')
+                minimum_agg_vars.append(to_add)
+        except AttributeError:
+            # ga.has_interception is added to threedigrid some months after
+            # groundwater release. To coop with .h5 that has been created in
+            # that period we read the simulated_vars_nodes
+            if 'intercepted_volume' in simulated_vars_nodes:
+                to_add = ('intercepted_volume', 'current interception')
+                minimum_agg_vars.append(to_add)
 
         if 'q_lat' in simulated_vars_nodes:
             to_add = ('q_lat_cum', 'cumulative lateral discharge')

water_balance/views/waterbalance_widget.py

@@ -72,6 +72,7 @@ def _translate(context, text, disambig):
     ('2d__1d_2d_flow_out', 31, '2d', '1d2d'),
     ('2d__1d_2d_exch_in', 32, '2d', '1d2d'),
     ('2d__1d_2d_exch_out', 33, '2d', '1d2d'),
+    ('intercepted_volume', 34, '2d', '2d'),
 ]
 
 
@@ -621,6 +622,11 @@ class WaterBalanceWidget(QDockWidget):
             'in': ['rain'],
             'out': ['rain'],
             'type': '2d',
+        }, {
+            'label_name': 'intercepted_volume',
+            'in': ['intercepted_volume'],
+            'out': ['intercepted_volume'],
+            'type': '2d',
         }
     ]
 
@@ -734,8 +740,10 @@ def show_barchart(self):
         # init figure
         plt.close()
         fig = plt.figure(1)
+
+        t_start = max(0, t1)
         plt.suptitle("Water balance from t=%.2f to t=%.2f\n Model name: %s" %
-                     (t1, t2, ga.model_name))
+                     (t_start, t2, ga.model_name))
         # prevent clipping of tick-labels, among others
         plt.subplots_adjust(
             bottom=.3, top=.9, left=.125, right=.9, hspace=1, wspace=.4)
@@ -918,6 +926,7 @@ def hover_enter_map_visualization(self, name):
             'lateral 1d': ['1d'],
             'lateral 2d': ['2d'],
             'leakage': ['2d'],
+            'intercepted_volume': ['2d'],
             'infiltration': ['2d'],
             'external (rain and laterals)': ['1d', '2d'],
         }