Refactor: Unify Getting GIS / Mapshed / GMS Data for a Scenario

[rajadain]

Currently, we calculate Mapshed data once for a project, and save it in its gis_data field. Each scenario can have modifications that affect parts of this object. This is currently done in multiple places. Here is sampling of some of them:

model-my-watershed/src/mmw/js/src/modeling/models.js

Lines 1579 to 1637 in e966cc9

	/**
	* Converts current modifications into a single object with key / value
	* pairs for overriding the baseline data model. Useful for aggregating
	* the final value when multiple BMPs target the same value in a GMS file.
	*/
	aggregateGwlfeModifications: function() {
	var gisData = App.currentProject.get('gis_data'),
	overrides = this.get('modifications').pluck('output'),
	input = {
	'CN__1': [], // Cropland Curve Number
	'n26': [], // Crop Tillage Practice Application Percentage
	'n65': [], // Crop Tillage Practice Nitrogen Efficiency
	'n73': [], // Crop Tillage Practice Phosphorus Efficiency
	'n81': [], // Crop Tillage Practice Sediments Efficiency
	},
	output = {};
	// For every override, if its key is present in `input`, add its
	// value to the array to be aggregated later. Else, just add it
	// to `output` directly.
	_.forEach(overrides, function(o) {
	_.forEach(o, function(value, key) {
	if (['CN__1', 'n26'].indexOf(key) >= 0) {
	input[key].push(value);
	} else if (['n65', 'n73', 'n81'].indexOf(key) >= 0) {
	// Multiply efficiency with its applied area
	// so it can be weighted by area later
	input[key].push(value * o['n26']);
	} else {
	output[key] = value;
	}
	});
	});
	if (input['CN__1'].length > 0) {
	// Curve Number aggregation is described in
	// https://github.com/WikiWatershed/model-my-watershed/issues/2942
	output['CN__1'] = _.sum(input['CN__1']) -
	(input['CN__1'].length - 1) * gisData['CN'][1];
	}
	if (input['n26'].length > 0) {
	// Area weight efficiencies
	var n26 = _.sum(input['n26']);
	_.extend(output, {
	'n26': n26,
	'n65': _.sum(input['n65']) / n26,
	'n73': _.sum(input['n73']) / n26,
	'n81': _.sum(input['n81']) / n26,
	});
	}
	this.validateGwlfeModifications(output);
	// Technically `output` is a singleton that contains everything,
	// but we put it in an array to maintain backwards compatibility.
	return [output];
	},

model-my-watershed/src/mmw/js/src/modeling/models.js

Lines 1639 to 1662 in e966cc9

	/**
	* Returns a `gis_data` object with the overriding modifications
	* of this scenario applied.
	*/
	getModifiedGwlfeGisData: function() {
	var gisData = _.cloneDeep(App.currentProject.get('gis_data')),
	modifications = this.aggregateGwlfeModifications();
	_.forEach(modifications, function(override) {
	_.forEach(override, function(value, key) {
	if (key.indexOf('__') > 0) {
	var split = key.split('__'),
	gmskey = split[0],
	index = parseInt(split[1]);
	gisData[gmskey][index] = value;
	} else {
	gisData[key] = value;
	}
	});
	});
	return gisData;
	},

model-my-watershed/src/mmw/js/src/modeling/models.js

Lines 1664 to 1705 in e966cc9

	getGisData: function(isSubbasinMode) {
	var self = this,
	project = App.currentProject,
	aoi = project.get('area_of_interest'),
	wkaoi = project.get('wkaoi');
	switch(App.currentProject.get('model_package')) {
	case utils.TR55_PACKAGE:
	var nonZeroModifications = self.get('modifications').filter(function(mod) {
	return mod.get('effectiveArea') > 0;
	}),
	modelInput = {
	inputs: self.get('inputs').toJSON(),
	modification_pieces: alterModifications(nonZeroModifications, self.get('modification_hash')),
	aoi_census: self.get('aoi_census'),
	modification_censuses: self.get('modification_censuses'),
	inputmod_hash: self.get('inputmod_hash'),
	modification_hash: self.get('modification_hash')
	};
	if (utils.isWKAoIValid(wkaoi)) {
	modelInput.wkaoi = wkaoi;
	} else {
	modelInput.area_of_interest = aoi;
	}
	return {
	model_input: JSON.stringify(modelInput)
	};
	case utils.GWLFE:
	var modifications = self.aggregateGwlfeModifications();
	return {
	inputmod_hash: self.get('inputmod_hash'),
	modifications: JSON.stringify(modifications),
	mapshed_job_uuid: isSubbasinMode ?
	project.get('subbasin_mapshed_job_uuid') :
	project.get('mapshed_job_uuid'),
	};
	}
	}

model-my-watershed/src/mmw/js/src/modeling/gwlfeModificationConfig.js

Lines 100 to 125 in e966cc9

	function cleanDataModel(dataModel, mods) {
	var newModel = _.mapValues(dataModel, function(val, varName) {
	return varName === UrbLengthName ? val / 1000 : val;
	}),
	landCoverChanges = mods && mods.findWhere({ modKey: 'entry_landcover' }),
	landCoverOutput = landCoverChanges && landCoverChanges.get('output');
	if (landCoverOutput) {
	// Same as in apps.modeling.mapshed.calcs.area_calculations
	// Update with modified values if available, else use default values
	newModel[n23Name] = _.get(landCoverOutput, 'Area__1', newModel.Area[1]);
	newModel[n23bName] = _.get(landCoverOutput, 'Area__13', newModel.Area[13]);
	newModel[n24Name] = _.get(landCoverOutput, 'Area__0', newModel.Area[0]);
	newModel[n24bName] = _.get(landCoverOutput, 'Area__11', newModel.Area[11]);
	newModel[UrbAreaTotalName] = _.sum([
	_.get(landCoverOutput, 'Area__10', newModel.Area[10]),
	_.get(landCoverOutput, 'Area__11', newModel.Area[11]),
	_.get(landCoverOutput, 'Area__12', newModel.Area[12]),
	_.get(landCoverOutput, 'Area__13', newModel.Area[13]),
	_.get(landCoverOutput, 'Area__14', newModel.Area[14]),
	_.get(landCoverOutput, 'Area__15', newModel.Area[15]),
	]);
	}
	return newModel;
	}

model-my-watershed/src/mmw/apps/modeling/calcs.py

Lines 261 to 288 in e966cc9

	def apply_gwlfe_modifications(gms, modifications):
	# Partition modifications into array and key modifications.
	# Array modifications target gms arrays, and have keys like
	# {array}__{index}. Key modifications target gms keys and
	# can be used by simply updating modified_gms.
	array_mods = []
	key_mods = []
	modified_gms = deepcopy(gms)
	for mod in modifications:
	for key, value in mod.iteritems():
	if '__' in key:
	array_mods.append({key: value})
	else:
	key_mods.append({key: value})
	for mod in array_mods:
	for key, value in mod.iteritems():
	gmskey, i = key.split('__')
	modified_gms[gmskey][int(i)] = value
	for mod in key_mods:
	modified_gms.update(mod)
	# Update keys that derive values from other keys
	modified_gms = area_calculations(modified_gms['Area'], modified_gms)
	return modified_gms

with their many usages.

Refactor this so there is ONE canonical place where all the modifications for a scenario are combined into a single object, giving the canonical state for gis_data for that scenario.

There should be at most two places for such code to exist: once in the client and once on the server, but no more.