Gui input processing

conda-requirements.txt

@@ -1,5 +1,5 @@
 nomkl
-taxcalc==0.9.0
+taxcalc==0.10.2
 btax==0.1.8
 numba==0.33.0
 pandas

webapp/apps/btax/views.py

@@ -43,7 +43,7 @@
                                 is_wildcard,
                                 convert_val,
                                 make_bool)
-from ..taxbrain.views import (benefit_surtax_fixup,
+from ..taxbrain.views import (benefit_switch_fixup,
                               denormalize, normalize,taxcalc_version as TAXCALC_VERSION)
 from .compute import DropqComputeBtax, MockComputeBtax, JobFailError
 

webapp/apps/dynamic/tests/test_behavioral.py

@@ -77,7 +77,7 @@ def test_behavioral_reform_with_wildcard(self):
                 u'ID_BenefitSurtax_Switch_5': [u'True'],
                 u'ID_BenefitSurtax_Switch_4': [u'True'],
                 u'ID_BenefitSurtax_Switch_6': [u'True'],
-                u'SS_Earnings_c': [u'*,*,*,*,1.0e99'],
+                u'SS_Earnings_c': [u'*,*,*,*,15000'],
                 u'has_errors': [u'False'],
                 u'start_year': u'2016', 'csrfmiddlewaretoken': 'abc123'}
 
@@ -92,7 +92,7 @@ def test_behavioral_reform_with_wildcard(self):
         # Verify that partial equilibrium job submitted with proper
         # SS_Earnings_c with wildcards filled in properly
         reform = json.loads(post['user_mods'])
-        assert reform["2016"][u'_SS_Earnings_c'][0]  == 118500.0
+        assert reform["2020"][u'_SS_Earnings_c'][0]  == 15000.0
 
 
     def test_behavioral_reform_from_file(self):

webapp/apps/dynamic/views.py

@@ -28,16 +28,16 @@
 from django import forms
 
 from djqscsv import render_to_csv_response
-from .forms import (DynamicInputsModelForm, DynamicBehavioralInputsModelForm, 
+from .forms import (DynamicInputsModelForm, DynamicBehavioralInputsModelForm,
                     has_field_errors, DynamicElasticityInputsModelForm)
 from .models import (DynamicSaveInputs, DynamicOutputUrl,
                      DynamicBehaviorSaveInputs, DynamicBehaviorOutputUrl,
                      DynamicElasticitySaveInputs, DynamicElasticityOutputUrl)
 from ..taxbrain.models import TaxSaveInputs, OutputUrl, ErrorMessageTaxCalculator
-from ..taxbrain.views import (growth_fixup, benefit_surtax_fixup, make_bool, dropq_compute,
-                              JOB_PROC_TIME_IN_SECONDS)
+from ..taxbrain.views import (growth_fixup, benefit_switch_fixup, make_bool, dropq_compute,
+                              JOB_PROC_TIME_IN_SECONDS, get_reform_from_gui)
 from ..taxbrain.helpers import (default_policy, taxcalc_results_to_tables, default_behavior,
-                                convert_val)
+                                convert_val, to_json_reform)
 
 from .helpers import (default_parameters, job_submitted,
                       ogusa_results_to_tables, success_text,
@@ -123,7 +123,7 @@ def dynamic_input(request, pk):
                 if 'tax_result' in microsim_data:
                     del microsim_data['tax_result']
 
-                benefit_surtax_fixup(request.REQUEST, microsim_data, taxbrain_model)
+                benefit_switch_fixup(request.REQUEST, microsim_data, taxbrain_model)
 
                 # start calc job
                 submitted_ids, guids = dynamic_compute.submit_ogusa_calculation(worker_data, int(start_year), microsim_data)
@@ -186,7 +186,7 @@ def dynamic_behavioral(request, pk):
     handles the calculation on the inputs.
     """
 
-    if request.method=='POST':
+    if request.method == 'POST':
         # Client is attempting to send inputs, validate as form data
         fields = dict(request.REQUEST)
         strip_empty_lists(fields)
@@ -196,56 +196,35 @@ def dynamic_behavioral(request, pk):
 
         if dyn_mod_form.is_valid():
             model = dyn_mod_form.save()
-
-            curr_dict = dict(model.__dict__)
-            for key, value in curr_dict.items():
-                print "got this ", key, value
-
-            for key, value in curr_dict.items():
-                if type(value) == type(unicode()):
-                    curr_dict[key] = [convert_val(x) for x in value.split(',') if x]
-                else:
-                    print "missing this: ", key
-
-            # get macrosim data from form
-            worker_data = {k:v for k, v in curr_dict.items() if v not in (u'', None, [])}
-
-            #get microsim data 
+            #get microsim data
             outputsurl = OutputUrl.objects.get(pk=pk)
             model.micro_sim = outputsurl
             taxbrain_model = outputsurl.unique_inputs
+            is_file = False
+             # necessary for simulations before PR 641
             if not taxbrain_model.json_text:
-
-                taxbrain_dict = dict(taxbrain_model.__dict__)
-                growth_fixup(taxbrain_dict)
-                for key, value in taxbrain_dict.items():
-                    if type(value) == type(unicode()):
-                        taxbrain_dict[key] = [convert_val(x) for x in value.split(',') if x]
-                    else:
-                        print "missing this: ", key
-
-                microsim_data = {k:v for k, v in taxbrain_dict.items() if not (v == [] or v == None)}
-                behavior_params = format_dynamic_params(worker_data)
-
-                #Don't need to pass around the microsim results
-                if 'tax_result' in microsim_data:
-                    del microsim_data['tax_result']
-
-                benefit_surtax_fixup(request.REQUEST, microsim_data, taxbrain_model)
-                microsim_data.update(worker_data)
-                # start calc job
-                submitted_ids, max_q_length = dropq_compute.submit_dropq_calculation(microsim_data, int(start_year), additional_data=behavior_params)
-
-            else:
-                behavior_params = format_dynamic_params(worker_data)
-                # start calc job
-                submitted_ids, max_q_length = dropq_compute.submit_dropq_calculation(
-                    json.loads(taxbrain_model.json_text.reform_text),
-                    int(start_year),
-                    is_file=True,
-                    additional_data=behavior_params,
+                (reform_dict, assumptions_dict, _, _,
+                    errors_warnings) = get_reform_from_gui(
+                        request,
+                        taxbrain_model=taxbrain_model,
+                        behavior_model=model
                 )
-
+            else:
+                is_file = True
+                reform_dict = json.loads(taxbrain_model.json_text.reform_text)
+                (_, assumptions_dict, _, _,
+                    errors_warnings) = get_reform_from_gui(
+                        request,
+                        taxbrain_model=None,
+                        behavior_model=model
+                    )
+            # start calc job
+            submitted_ids, max_q_length = dropq_compute.submit_dropq_calculation(
+                reform_dict,
+                int(start_year),
+                is_file=is_file,
+                additional_data=assumptions_dict
+            )
 
             if not submitted_ids:
                 no_inputs = True
@@ -302,7 +281,7 @@ def dynamic_behavioral(request, pk):
 
 def dynamic_elasticities(request, pk):
     """
-    This view handles the dynamic macro elasticities input page and 
+    This view handles the dynamic macro elasticities input page and
     calls the function that handles the calculation on the inputs.
     """
 
@@ -342,7 +321,7 @@ def dynamic_elasticities(request, pk):
             # get macrosim data from form
             worker_data = {k:v for k, v in curr_dict.items() if v not in (u'', None, [])}
 
-            #get microsim data 
+            #get microsim data
             outputsurl = OutputUrl.objects.get(pk=pk)
             model.micro_sim = outputsurl
             taxbrain_model = outputsurl.unique_inputs
@@ -361,7 +340,7 @@ def dynamic_elasticities(request, pk):
                 if 'tax_result' in microsim_data:
                     del microsim_data['tax_result']
 
-                benefit_surtax_fixup(request.REQUEST, microsim_data, taxbrain_model)
+                benefit_switch_fixup(request.REQUEST, microsim_data, taxbrain_model)
                 microsim_data.update(worker_data)
                 # start calc job
                 submitted_ids, max_q_length = dropq_compute.submit_elastic_calculation(microsim_data,

webapp/apps/taxbrain/compute.py

@@ -35,10 +35,6 @@ class DropqCompute(object):
 
     num_budget_years = NUM_BUDGET_YEARS
 
-    # Override if needed, e.g. btax
-    def package_up_vars(self, *args, **kwargs):
-        return _package_up_vars(*args, **kwargs)
-
     def remote_submit_job(self, theurl, data, timeout=TIMEOUT_IN_SECONDS):
         response = requests.post(theurl, data=data, timeout=timeout)
         return response
@@ -51,59 +47,51 @@ def remote_retrieve_results(self, theurl, params):
         job_response = requests.get(theurl, params=params)
         return job_response
 
-    def submit_json_dropq_calculation(self, mods, first_budget_year, additional_data=None):
+    def submit_json_dropq_calculation(self, user_mods, first_budget_year, additional_data=None):
         url_template = "http://{hn}" + DROPQ_URL
-        return self.submit_calculation(mods, first_budget_year, url_template,
+        return self.submit_calculation(user_mods, first_budget_year, url_template,
                                        num_years=NUM_BUDGET_YEARS,
                                        pack_up_user_mods=False,
                                        additional_data=additional_data)
 
-    def submit_dropq_calculation(self, mods, first_budget_year, additional_data={}, is_file=False):
+    def submit_dropq_calculation(self, user_mods, first_budget_year, additional_data={}, is_file=False):
         url_template = "http://{hn}" + DROPQ_URL
-        return self.submit_calculation(mods, first_budget_year, url_template,
+        return self.submit_calculation(user_mods, first_budget_year, url_template,
                                        num_years=NUM_BUDGET_YEARS,
                                        additional_data=additional_data,
                                        pack_up_user_mods=not is_file)
 
-    def submit_json_dropq_small_calculation(self, mods, first_budget_year):
+    def submit_json_dropq_small_calculation(self, user_mods, first_budget_year):
         url_template = "http://{hn}" + DROPQ_SMALL_URL
-        return self.submit_calculation(mods, first_budget_year, url_template,
+        return self.submit_calculation(user_mods, first_budget_year, url_template,
                                        num_years=NUM_BUDGET_YEARS_QUICK,
                                        increment_counter=False,
                                        pack_up_user_mods=False)
 
-    def submit_dropq_small_calculation(self, mods, first_budget_year, additional_data={}, is_file=False):
+    def submit_dropq_small_calculation(self, user_mods, first_budget_year, additional_data={}, is_file=False):
         url_template = "http://{hn}" + DROPQ_SMALL_URL
-        return self.submit_calculation(mods, first_budget_year, url_template,
+        return self.submit_calculation(user_mods, first_budget_year, url_template,
                                        num_years=NUM_BUDGET_YEARS_QUICK,
                                        additional_data=additional_data,
                                        increment_counter=False,
                                        pack_up_user_mods=not is_file)
 
-    def submit_elastic_calculation(self, mods, first_budget_year, is_file=False, additional_data={}):
+    def submit_elastic_calculation(self, user_mods, first_budget_year, is_file=False, additional_data={}):
         url_template = "http://{hn}/elastic_gdp_start_job"
-        return self.submit_calculation(mods, first_budget_year, url_template,
+        return self.submit_calculation(user_mods, first_budget_year, url_template,
                                        start_budget_year=1,
                                        additional_data=additional_data,
                                        pack_up_user_mods=not is_file)
 
 
-    def submit_calculation(self, mods, first_budget_year, url_template,
+    def submit_calculation(self, user_mods, first_budget_year, url_template,
                            start_budget_year=0, num_years=NUM_BUDGET_YEARS,
                            workers=DROPQ_WORKERS,
                            increment_counter=True,
                            use_wnc_offset=True,
                            pack_up_user_mods=True,
                            additional_data={}):
         data = {}
-        if pack_up_user_mods:
-            user_mods = self.package_up_vars(mods, first_budget_year)
-            if not bool(user_mods):
-                return False
-            user_mods = {first_budget_year:user_mods}
-        else:
-            user_mods = mods
-
         years = self._get_years(start_budget_year, num_years, first_budget_year)
         if use_wnc_offset:
             wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1)

webapp/apps/taxbrain/forms.py

@@ -270,7 +270,6 @@ def do_taxcalc_validations(self):
 
             # First make sure the text parses OK
             BOOLEAN_FLAGS = (u'True', u'False')
-            found_parse_error = False
             for col, col_field in enumerate(param.col_fields):
                 if col_field.id not in self.cleaned_data:
                     continue
@@ -283,71 +282,6 @@ def do_taxcalc_validations(self):
                     except ParseException as pe:
                         # Parse Error - we don't recognize what they gave us
                         self.add_error(col_field.id, "Unrecognized value: {}".format(submitted_col_values_raw))
-                        found_parse_error = True
-
-            if found_parse_error:
-                continue
-
-            # Move on if there is no min/max validation necessary
-            if param.max is None and param.min is None:
-                continue
-
-            for col, col_field in enumerate(param.col_fields):
-                submitted_col_values_raw = self.cleaned_data[col_field.id]
-                try:
-                    submitted_col_values = string_to_float_array(submitted_col_values_raw)
-                except ValueError as ve:
-                    # Assuming wildcard notation here
-                    submitted_col_values_list = submitted_col_values_raw.split(',')
-                    param_name = parameter_name(col_field.id)
-                    submitted_col_values = expand_unless_empty(submitted_col_values_list, param_name, col_field.id, self, len(submitted_col_values_list))
-                default_col_values = col_field.values
-
-                # If we change a different field which this field relies on for
-                # validation, we must ensure this is validated even if unchanged
-                # from defaults
-                if submitted_col_values:
-                    col_values = submitted_col_values
-                else:
-                    col_values = default_col_values
-
-                if param.max is not None:
-                    comp = self.get_comp_data(param.max, param_id, col, col_values)
-                    source = comp['source']
-                    maxes = comp['comp_data']
-                    exp_col_values = comp['exp_col_values']
-
-                    for i, value in enumerate(exp_col_values):
-                        if value > maxes[i]:
-                            if len(col_values) == 1:
-                                self.add_error(col_field.id,
-                                               u"Must be \u2264 {0} of {1}".
-                                               format(source, maxes[i]))
-                            else:
-                                self.add_error(col_field.id,
-                                               u"{0} value must be \u2264 \
-                                               {1}'s {0} value of {2}".format(
-                                                   int_to_nth(i + 1),
-                                                   source, maxes[i]))
-
-                if param.min is not None:
-                    comp = self.get_comp_data(param.min, param_id, col, col_values)
-                    source = comp['source']
-                    mins = comp['comp_data']
-                    exp_col_values = comp['exp_col_values']
-
-                    for i, value in enumerate(exp_col_values):
-                        if value < mins[i]:
-                            if len(col_values) == 1:
-                                self.add_error(col_field.id,
-                                               u"Must be \u2265 {0} of {1}".
-                                               format(source, mins[i]))
-                            else:
-                                self.add_error(col_field.id,
-                                               u"{0} value must be \u2265 \
-                                               {1}'s {0} value of {2}".format(
-                                                   int_to_nth(i + 1),
-                                                   source, mins[i]))
 
     class Meta:
         model = TaxSaveInputs