-
Notifications
You must be signed in to change notification settings - Fork 32
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Refactor warning/error parsing #800
Changes from 4 commits
d0fe0a7
04c336d
0578c21
a32dfcc
d2014d5
8ae2b43
4102204
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -151,6 +151,36 @@ def normalize(x): | |
return ans | ||
|
||
|
||
def append_ew_file_input(error_list, msg, *args): | ||
""" | ||
Append errors to list for file input interface | ||
""" | ||
error_list.append(msg) | ||
|
||
|
||
def append_ew_personal_inputs(personal_inputs, msg, param_name): | ||
""" | ||
Use PersonalExemptionForm.add_error to append errors for GUI input | ||
interface | ||
""" | ||
personal_inputs.add_error(param_name, msg) | ||
|
||
|
||
def append_errors_warnings(errors_warnings, append_obj, append_func): | ||
""" | ||
Appends warning/error messages to some object, append_obj, according to | ||
the provided function, append_func | ||
""" | ||
for action in ['warnings', 'errors']: | ||
for param in errors_warnings[action]: | ||
for year in sorted( | ||
errors_warnings[action][param].keys(), | ||
key=lambda x: int(x) | ||
): | ||
msg = errors_warnings[action][param][year] | ||
append_func(append_obj, msg, param) | ||
|
||
|
||
def parse_errors_warnings(errors_warnings, map_back_to_tb): | ||
""" | ||
Parse error messages so that they can be mapped to Taxbrain param ID. This | ||
|
@@ -171,11 +201,11 @@ def parse_errors_warnings(errors_warnings, map_back_to_tb): | |
msg_spl = msg.split() | ||
msg_action = msg_spl[0] | ||
year = msg_spl[1] | ||
curr_id = msg_spl[2] | ||
curr_id = msg_spl[2][1:] | ||
msg_parse = msg_spl[2:] | ||
if year not in parsed[action]: | ||
parsed[action][year] = {} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can use a from collections import defaultdict # at the top
...
parsed = {'errors': defaultdict(dict), 'warnings': defaultdict(dict)} Then you can get rid of the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Oh nice. |
||
parsed[action][year][curr_id[1:]] = ' '.join([msg_action] + msg_parse + | ||
if curr_id not in parsed[action]: | ||
parsed[action][curr_id] = {} | ||
parsed[action][curr_id][year] = ' '.join([msg_action] + msg_parse + | ||
['for', year]) | ||
|
||
return parsed | ||
|
@@ -467,14 +497,8 @@ def submit_reform(request, user=None, json_reform_id=None): | |
# only handle GUI errors for now | ||
if ((taxcalc_errors or taxcalc_warnings) | ||
and personal_inputs is not None): | ||
for action in errors_warnings: | ||
for year in sorted(errors_warnings[action].keys(), | ||
key=lambda x: float(x)): | ||
for param in errors_warnings[action][year]: | ||
personal_inputs.add_error( | ||
param, | ||
errors_warnings[action][year][param] | ||
) | ||
append_errors_warnings(errors_warnings, personal_inputs, | ||
append_ew_personal_inputs) | ||
has_parse_errors = any(['Unrecognize value' in e[0] | ||
for e in personal_inputs.errors.values()]) | ||
if no_inputs: | ||
|
@@ -592,22 +616,8 @@ def file_input(request): | |
if (has_errors and | ||
(errors_warnings['warnings'] or errors_warnings['errors'])): | ||
errors.append(OUT_OF_RANGE_ERROR_MSG) | ||
# group messages by parameter name | ||
group_by_param = {} | ||
for action in errors_warnings: | ||
for year in sorted(errors_warnings[action].keys(), | ||
key=lambda x: float(x)): | ||
for param in errors_warnings[action][year]: | ||
if param in group_by_param: | ||
group_by_param[param].append( | ||
errors_warnings[action][year][param] | ||
) | ||
else: | ||
group_by_param[param] = \ | ||
[errors_warnings[action][year][param]] | ||
# append to errors | ||
for param in group_by_param: | ||
errors += group_by_param[param] | ||
append_errors_warnings(errors_warnings, errors, | ||
append_ew_file_input) | ||
else: | ||
return redirect(unique_url) | ||
else: | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,7 +3,10 @@ | |
import taxcalc | ||
import numpy as np | ||
|
||
from ..taxbrain.views import read_json_reform, parse_errors_warnings | ||
from ..taxbrain.views import (read_json_reform, parse_errors_warnings, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. One handy software engineering tool you might be interested in is "blame". Basically, it's a tool that shows the last person to make changes to each line of code. GitHub has a more thorough explanation here. This isn't a review comment per se, but one thing to keep in mind when using that tool is writing code in a way that "preserves blame", in other words, so that you don't make changes to lines that you don't edit. In this case, the best way to "preserve blame" would be to write each import on a new line, so that if you add your own import, it doesn't update the blame on someone else's import, so you know the last person who thought that import was important to change. i.e. from ..taxbrain.views import (
read_json_reform,
parse_errors_warnings,
...
) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I hadn't thought about preserving "blame" before. That makes a lot of sense. Thanks for the pointer. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Check out the blame tool, it is useful for figuring out who wrote something if you're trying to figure out why it was done that way (i.e. poorly documented code) |
||
append_errors_warnings, append_ew_file_input, | ||
append_ew_personal_inputs) | ||
|
||
from ..taxbrain.helpers import (get_default_policy_param_name, to_json_reform) | ||
|
||
from test_reform import (test_coverage_fields, test_coverage_reform, | ||
|
@@ -93,6 +96,51 @@ def test_parse_errors_warnings(): | |
np.testing.assert_equal(act, exp_errors_warnings) | ||
|
||
|
||
def test_append_ew_file_input(): | ||
""" | ||
Tests append_errors_warnings when add warning/error messages from the | ||
file input interface | ||
""" | ||
errors_warnings = {"warnings": {"param1": {"2017": "msg2", "2016": "msg1"}}, | ||
"errors": {"param2": {"2018": "msg3", "2019": "msg4"}} | ||
} | ||
error_list = [] | ||
exp = ["msg1", "msg2", "msg3", "msg4"] | ||
|
||
append_errors_warnings(errors_warnings, error_list, append_ew_file_input) | ||
|
||
assert error_list == exp | ||
|
||
def test_append_ew_personal_inputs(): | ||
""" | ||
Tests append_errors_warnings when adding warning/error messages from the | ||
GUI input interface | ||
""" | ||
errors_warnings = {"warnings": {"param1": {"2017": "msg2", "2016": "msg1"}}, | ||
"errors": {"param2": {"2018": "msg3", "2019": "msg4"}} | ||
} | ||
# fake PersonalExemptionForm object to simulate add_error method | ||
class FakeForm: | ||
def __init__(self): | ||
self.errors = {} | ||
|
||
def add_error(self, param_name, msg): | ||
if param_name in self.errors: | ||
self.errors[param_name].append(msg) | ||
else: | ||
self.errors[param_name] = [msg] | ||
|
||
exp = {"param1": ["msg1", "msg2"], | ||
"param2": ["msg3", "msg4"]} | ||
|
||
fake_form = FakeForm() | ||
append_errors_warnings(errors_warnings, fake_form, | ||
append_ew_personal_inputs) | ||
|
||
assert (exp["param1"] == fake_form.errors["param1"] and | ||
exp["param2"] == fake_form.errors["param2"]) | ||
|
||
|
||
############################################################################### | ||
# Test read_json_reform | ||
# 3 Cases: | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -372,30 +372,34 @@ | |
|
||
exp_errors_warnings = { | ||
'errors': { | ||
'2017': { | ||
'FICA_ss_trt': 'ERROR: _FICA_ss_trt value -1.0 < min value 0 for 2017', | ||
'II_brk4_0': 'ERROR: _II_brk4_0 value 500.0 < min value 91900.0 for _II_brk3_0 for 2017'}, | ||
'2018': { | ||
'FICA_ss_trt': 'ERROR: _FICA_ss_trt value -1.0 < min value 0 for 2018', | ||
'II_brk4_0': 'ERROR: _II_brk4_0 value 511.25 < min value 93967.75 for _II_brk3_0 for 2018'}, | ||
'2019': {'II_brk4_0': 'ERROR: _II_brk4_0 value 522.7 < min value 96072.63 for _II_brk3_0 for 2019'}, | ||
'2020': {'II_brk4_0': 'ERROR: _II_brk4_0 value 534.77 < min value 98291.91 for _II_brk3_0 for 2020'}, | ||
'2021': {'II_brk4_0': 'ERROR: _II_brk4_0 value 547.5 < min value 100631.26 for _II_brk3_0 for 2021'}, | ||
'2022': {'II_brk4_0': 'ERROR: _II_brk4_0 value 560.8 < min value 103076.6 for _II_brk3_0 for 2022'}, | ||
'2023': {'II_brk4_0': 'ERROR: _II_brk4_0 value 574.09 < min value 105519.52 for _II_brk3_0 for 2023'}, | ||
'2024': {'II_brk4_0': 'ERROR: _II_brk4_0 value 587.87 < min value 108051.99 for _II_brk3_0 for 2024'}, | ||
'2025': {'II_brk4_0': 'ERROR: _II_brk4_0 value 601.86 < min value 110623.63 for _II_brk3_0 for 2025'}, | ||
'2026': {'II_brk4_0': 'ERROR: _II_brk4_0 value 616.18 < min value 113256.47 for _II_brk3_0 for 2026'}, | ||
'2027': {'II_brk4_0': 'ERROR: _II_brk4_0 value 630.97 < min value 115974.63 for _II_brk3_0 for 2027'} | ||
'FICA_ss_trt': { | ||
'2017': 'ERROR: _FICA_ss_trt value -1.0 < min value 0 for 2017', | ||
'2018': 'ERROR: _FICA_ss_trt value -1.0 < min value 0 for 2018' | ||
}, | ||
'II_brk4_0': { | ||
'2017': 'ERROR: _II_brk4_0 value 500.0 < min value 91900.0 for _II_brk3_0 for 2017', | ||
'2018': 'ERROR: _II_brk4_0 value 511.25 < min value 93967.75 for _II_brk3_0 for 2018', | ||
'2019': 'ERROR: _II_brk4_0 value 522.7 < min value 96072.63 for _II_brk3_0 for 2019', | ||
'2020': 'ERROR: _II_brk4_0 value 534.77 < min value 98291.91 for _II_brk3_0 for 2020', | ||
'2021': 'ERROR: _II_brk4_0 value 547.5 < min value 100631.26 for _II_brk3_0 for 2021', | ||
'2022': 'ERROR: _II_brk4_0 value 560.8 < min value 103076.6 for _II_brk3_0 for 2022', | ||
'2023': 'ERROR: _II_brk4_0 value 574.09 < min value 105519.52 for _II_brk3_0 for 2023', | ||
'2024': 'ERROR: _II_brk4_0 value 587.87 < min value 108051.99 for _II_brk3_0 for 2024', | ||
'2025': 'ERROR: _II_brk4_0 value 601.86 < min value 110623.63 for _II_brk3_0 for 2025', | ||
'2026': 'ERROR: _II_brk4_0 value 616.18 < min value 113256.47 for _II_brk3_0 for 2026', | ||
'2027': 'ERROR: _II_brk4_0 value 630.97 < min value 115974.63 for _II_brk3_0 for 2027' | ||
} | ||
}, | ||
'warnings': { | ||
'2020': {'STD_3': 'WARNING: _STD_3 value 150.0 < min value 9900.32 for 2020'}, | ||
'2021': {'STD_3': 'WARNING: _STD_3 value 153.57 < min value 10138.33 for 2021'}, | ||
'2022': {'STD_3': 'WARNING: _STD_3 value 157.3 < min value 10387.12 for 2022'}, | ||
'2023': {'STD_3': 'WARNING: _STD_3 value 161.03 < min value 10635.66 for 2023'}, | ||
'2024': {'STD_3': 'WARNING: _STD_3 value 164.89 < min value 10893.32 for 2024'}, | ||
'2025': {'STD_3': 'WARNING: _STD_3 value 168.81 < min value 11154.96 for 2025'}, | ||
'2026': {'STD_3': 'WARNING: _STD_3 value 172.83 < min value 11422.83 for 2026'}, | ||
'2027': {'STD_3': 'WARNING: _STD_3 value 176.98 < min value 11699.38 for 2027'} | ||
'STD_3': { | ||
'2020': 'WARNING: _STD_3 value 150.0 < min value 9900.32 for 2020', | ||
'2021': 'WARNING: _STD_3 value 153.57 < min value 10138.33 for 2021', | ||
'2022': 'WARNING: _STD_3 value 157.3 < min value 10387.12 for 2022', | ||
'2023': 'WARNING: _STD_3 value 161.03 < min value 10635.66 for 2023', | ||
'2024': 'WARNING: _STD_3 value 164.89 < min value 10893.32 for 2024', | ||
'2025': 'WARNING: _STD_3 value 168.81 < min value 11154.96 for 2025', | ||
'2026': 'WARNING: _STD_3 value 172.83 < min value 11422.83 for 2026', | ||
'2027': 'WARNING: _STD_3 value 176.98 < min value 11699.38 for 2027' | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. One software engineering practice you might want to consider is separating "code" from "data". In this case the "data" is these warnings, which could be saved as like a separate JSON file somewhere and loaded into Python (using the default There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok, I see. I'll open a follow up PR to clean up |
||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This seems redundant, since you only use this function once... Probably a cleaner way to do this would be to make the third argument of
append_errors_warnings
be a method with two arguments,msg
andparam
Then the
append_errors_warnings
function would beAdmittedly lambdas are frowned upon by some people but I think it makes for less redundant code here.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't have any aversion to lambdas. I think this looks a little cleaner.
Out of curiosity, why do some people frown upon lambda functions?