Manual Entry: Recalculate Area-derived Values on Land Cover Updates

README.md

@@ -152,6 +152,13 @@ $ ./scripts/manage.sh test apps.app_name.tests
 
 More info [here](https://docs.djangoproject.com/en/1.8/topics/testing/).
 
+Run MapShed tests, which require MapShed tables installed in the local database
+(using `setupdb.sh`):
+
+```console
+$ ./scripts/manage.sh test_mapshed
+```
+
 ##### JavaScript
 
 To check for JavaScript lint:

scripts/test.sh

@@ -12,7 +12,7 @@ set -x
 vagrant ssh app -c "flake8 /opt/app/apps --exclude migrations || echo flake8 check failed"
 
 # Run the Django test suite with --noinput flag.
-vagrant ssh app -c "cd /opt/app && envdir /etc/mmw.d/env ./manage.py test --noinput"
+vagrant ssh app -c "cd /opt/app && envdir /etc/mmw.d/env ./manage.py test --noinput --exclude-tag=mapshed"
 
 # Check for client-side JS lint.
 # vagrant ssh app -c "cd /opt/app && npm run lint"

src/mmw/apps/modeling/calcs.py

@@ -12,6 +12,8 @@
 
 from django.contrib.gis.geos import WKBReader
 
+from apps.modeling.mapshed.calcs import area_calculations
+
 
 HECTARES_PER_SQM = 0.0001
 
@@ -103,6 +105,9 @@ def apply_gwlfe_modifications(gms, modifications):
     for mod in key_mods:
         modified_gms.update(mod)
 
+    # Update keys that derive values from other keys
+    modified_gms = area_calculations(modified_gms['Area'], modified_gms)
+
     return modified_gms
 
 

src/mmw/apps/modeling/management/commands/test_data/geop-results-list.json

@@ -0,0 +1 @@
+[{"landuse_pcts": [0.0017858831192024457, 0.0004832389616665441, 0.03657908835919362, 0.016114968852097362, 0, 0, 0.002332153249782017, 0.00014707272746373082, 0, 0, 0.13098927419608997, 0.30799130169869, 0.33113424588458995, 0.13048502484478575, 0, 0], "cn": [75, 82, 69.75, 87.0, 0, 0, 87, 89.0, 0, 0, 0, 0, 0, 0, 0, 0]}, {"soiln": 1056.3471074380166}, {"soilp": 702.0881355932204}, {"recess_coef": 0.04235508474576272}, {"gr_phos_conc": 0.06670789571848966, "gr_nitr_conc": 8.714170460479961}, {"avg_awc": 1.7660511808298869}, {"n41": 0.25462962962962965, "ag_slope_3_8_pct": 0.0003571766238404891, "ag_slope_3_pct": 0.0005777857150360854}, {"avg_slope": 2.708598957407307}, {"avg_kf": 0.03660172029234699, "kf": [0.0021370587980045993, 0.0, 0.16766063067354733, 0.0006469204300002656, 0.0, 0.0, 0.01606418877034574, 0.007790000177919865, 0.0, 0.0, 0.06440528029078053, 0.024080129089020078, 0.010029554109157535, 0.0, 0.0, 0.0]}, {"lu_stream_pct": [0.003416856492027335, 0.0, 0.02733485193621868, 0.02733485193621868, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1958997722095672, 0.08656036446469248, 0.02847380410022779, 0.0, 0.0, 0.0], "med_high_urban_stream_pct": 0.11503416856492027, "ag_stream_pct": 0.003416856492027335, "low_urban_stream_pct": 0.1958997722095672}]

src/mmw/apps/modeling/management/commands/test_mapshed.py

@@ -0,0 +1,38 @@
+import unittest
+import json
+
+from os.path import join, dirname, abspath
+
+from django.core.management.base import BaseCommand
+from django.test import tag
+
+from dictdiffer import diff
+
+from apps.modeling.mapshed import tasks
+
+
+class Command(BaseCommand):
+    def handle(self, **options):
+        suite = unittest.TestLoader().loadTestsFromTestCase(TestMapshed)
+        unittest.TextTestRunner().run(suite)
+
+
+class TestMapshed(unittest.TestCase):
+    def setUp(self):
+        with open(join(dirname(abspath(__file__)),
+                  'test_data/huc12__55174.geojson')) as f:
+            self.geojson = f.read()
+        with open(join(dirname(abspath(__file__)),
+                  'test_data/geop-results-list.json')) as f:
+            self.geop_results = json.load(f)
+        with open(join(dirname(abspath(__file__)),
+                  'test_data/mapshed-dict.json')) as f:
+            self.z = json.load(f)
+
+    @tag('mapshed')
+    def test_collect_data(self):
+        z = tasks.collect_data(self.geop_results, self.geojson)
+
+        diffs = list(diff(self.z, z, tolerance=1e-15))
+
+        self.assertEqual(diffs, [])

src/mmw/apps/modeling/mapshed/calcs.py

@@ -690,3 +690,29 @@ def phosphorus_conc(sed_phos):
         0.01,          # Unpaved
         0, 0, 0, 0, 0, 0  # Urban Land Use Types
     ]
+
+
+def area_calculations(areas_h, z):
+    """
+    Given a list of areas per land use in hectares and MapShed Dictionary z,
+    calculates all fields that rely on the land use distribution, and returns
+    an updated dictionary z.
+
+    Useful when users customize the land use distribution.
+    """
+    percents = [a_h / z['TotArea'] for a_h in areas_h]
+
+    z['UrbAreaTotal'] = sum(areas_h[NRur:])
+    z['NumNormalSys'] = num_normal_sys(areas_h)
+    z['KV'] = kv_coefficient(percents, z['Grow'])
+
+    # Original at Class1.vb@1.3.0:9803-9807
+    z['n23'] = areas_h[1]    # Row Crops Area
+    z['n23b'] = areas_h[13]  # High Density Mixed Urban Area
+    z['n24'] = areas_h[0]    # Hay/Pasture Area
+    z['n24b'] = areas_h[11]  # Low Density Mixed Urban Area
+
+    z['SedAFactor'] = sed_a_factor(percents,
+                                   z['CN'], z['AEU'], z['AvKF'], z['AvSlope'])
+
+    return z

src/mmw/apps/modeling/mapshed/tasks.py

@@ -14,7 +14,6 @@
                                          growing_season,
                                          erosion_coeff,
                                          et_adjustment,
-                                         kv_coefficient,
                                          animal_energy_units,
                                          ag_ls_c_p,
                                          ls_factors,
@@ -30,8 +29,7 @@
                                          groundwater_nitrogen_conc,
                                          sediment_delivery_ratio,
                                          landuse_pcts,
-                                         num_normal_sys,
-                                         sed_a_factor
+                                         area_calculations,
                                          )
 
 
@@ -139,11 +137,8 @@ def collect_data(geop_results, geojson, watershed_id=None, weather=None):
     if sum(z['Area']) == 0:
         raise Exception(NO_LAND_COVER)
 
-    z['UrbAreaTotal'] = sum(z['Area'][NRur:])
     z['PhosConc'] = phosphorus_conc(z['SedPhos'])
 
-    z['NumNormalSys'] = num_normal_sys(z['Area'])
-
     z['AgSlope3'] = geop_result['ag_slope_3_pct'] * area * HECTARES_PER_SQM
     z['AgSlope3To8'] = (geop_result['ag_slope_3_8_pct'] *
                         area * HECTARES_PER_SQM)
@@ -154,21 +149,14 @@ def collect_data(geop_results, geojson, watershed_id=None, weather=None):
     z['AvKF'] = geop_result['avg_kf']
     z['KF'] = geop_result['kf']
 
-    z['KV'] = kv_coefficient(geop_result['landuse_pcts'], z['Grow'])
-
-    # Original at Class1.vb@1.3.0:9803-9807
-    z['n23'] = z['Area'][1]    # Row Crops Area
-    z['n23b'] = z['Area'][13]  # High Density Mixed Urban Area
-    z['n24'] = z['Area'][0]    # Hay/Pasture Area
-    z['n24b'] = z['Area'][11]  # Low Density Mixed Urban Area
-
     z['SedDelivRatio'] = sediment_delivery_ratio(area * SQKM_PER_SQM)
     z['TotArea'] = area * HECTARES_PER_SQM
     z['GrNitrConc'] = geop_result['gr_nitr_conc']
     z['GrPhosConc'] = geop_result['gr_phos_conc']
     z['MaxWaterCap'] = geop_result['avg_awc']
-    z['SedAFactor'] = sed_a_factor(geop_result['landuse_pcts'],
-                                   z['CN'], z['AEU'], z['AvKF'], z['AvSlope'])
+
+    # Calculate fields derived from land use distribution
+    z = area_calculations(z['Area'], z)
 
     # Use zeroed out stream variables if there are no streams in the AoI
     if 'lu_stream_pct' in geop_result:

src/mmw/js/src/modeling/controls.js

@@ -475,6 +475,10 @@ var GwlfeLandCoverView = ControlView.extend({
 
 var GwlfeConservationPracticeView = ModificationsView.extend({
     initialize: function(options) {
+        var gis_data = App.currentProject.get('gis_data'),
+            currentScenario = App.currentProject.get('scenarios').getActiveScenario(),
+            mods = currentScenario && currentScenario.get('modifications');
+
         ModificationsView.prototype.initialize.apply(this, [options]);
         this.model.set({
             controlName: this.getControlName(),
@@ -494,7 +498,7 @@ var GwlfeConservationPracticeView = ModificationsView.extend({
                     rows: [['urban_buffer_strips', 'urban_streambank_stabilization', 'water_retention', 'infiltration']]
                 }
             ],
-            dataModel: gwlfeConfig.cleanDataModel(App.currentProject.get('gis_data')),
+            dataModel: gwlfeConfig.cleanDataModel(gis_data, mods),
             errorMessages: null,
             infoMessages: null
         });

src/mmw/js/src/modeling/gwlfeModificationConfig.js

@@ -6,6 +6,9 @@ var _ = require('lodash'),
 // These variable names include 'Name' to indicate
 // that they reference the name of the variable, and not the value.
 var n23Name = 'n23',
+    n23bName = 'n23b',
+    n24Name = 'n24',
+    n24bName = 'n24b',
     n25Name = 'n25',
     n26Name = 'n26',
     n28bName = 'n28b',
@@ -94,10 +97,31 @@ var shortDisplayNames = fromPairs([
     [UrbLengthName, 'length of streams in non-ag areas']
 ]);
 
-function cleanDataModel(dataModel) {
-    return _.mapValues(dataModel, function(val, varName) {
-        return varName === UrbLengthName ? val / 1000 : val;
-    });
+function cleanDataModel(dataModel, mods) {
+    var newModel = _.mapValues(dataModel, function(val, varName) {
+            return varName === UrbLengthName ? val / 1000 : val;
+        }),
+        landCoverChanges = mods && mods.findWhere({ modKey: 'entry_landcover' }),
+        landCoverOutput = landCoverChanges && landCoverChanges.get('output');
+
+    if (landCoverOutput) {
+        // Same as in apps.modeling.mapshed.calcs.area_calculations
+        // Update with modified values if available, else use default values
+        newModel[n23Name] = _.get(landCoverOutput, 'Area__1', newModel.Area[1]);
+        newModel[n23bName] = _.get(landCoverOutput, 'Area__13', newModel.Area[13]);
+        newModel[n24Name] = _.get(landCoverOutput, 'Area__0', newModel.Area[0]);
+        newModel[n24bName] = _.get(landCoverOutput, 'Area__11', newModel.Area[11]);
+        newModel[UrbAreaTotalName] = _.sum([
+            _.get(landCoverOutput, 'Area__10', newModel.Area[10]),
+            _.get(landCoverOutput, 'Area__11', newModel.Area[11]),
+            _.get(landCoverOutput, 'Area__12', newModel.Area[12]),
+            _.get(landCoverOutput, 'Area__13', newModel.Area[13]),
+            _.get(landCoverOutput, 'Area__14', newModel.Area[14]),
+            _.get(landCoverOutput, 'Area__15', newModel.Area[15]),
+        ]);
+    }
+
+    return newModel;
 }
 
 function getPercentStr(fraction, dataModelName) {

src/mmw/requirements/base.txt

@@ -28,3 +28,4 @@ six==1.11.0
 fiona==1.7.11
 redis==2.10.6
 numpy==1.16.4
+dictdiffer==0.8.0