Skip to content

Commit

Permalink
Merge pull request #738 from hdoupe/update_tc
Browse files Browse the repository at this point in the history 
Merged #738
  • Loading branch information
@hdoupe
hdoupe committed Nov 20, 2017
2 parents a43ad0c + 4dc66c0 commit 5a9430e
Show file tree
Hide file tree
Showing 50 changed files with 3,609 additions and 808 deletions.
2 changes: 1 addition & 1 deletion conda-requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
nomkl
taxcalc==0.10.2
taxcalc==0.13.2
btax==0.1.9
numba==0.33.0
pandas
Expand Down
123 changes: 50 additions & 73 deletions deploy/taxbrain_server/celery_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,7 @@ def task(func):
return celery_app_like
celery_app = Namespace(task=task)

#Create a Public Use File object
rn_seed = 80
if not MOCK_CELERY:
tax_dta_full = pd.read_csv("puf.csv.gz", compression='gzip')
tax_dta = tax_dta_full.sample(frac=0.02, random_state=rn_seed)
else:
tax_dta_full = tax_dta = pd.DataFrame({})
tax_dta_full = tax_dta = pd.DataFrame({})


def convert_int_key(user_mods):
Expand All @@ -52,48 +46,27 @@ def convert_int_key(user_mods):
return user_mods


def dropq_task(year, user_mods, first_budget_year, beh_params, tax_data):
print("user mods: ", user_mods)
# The reform style indicates what kind of reform we ran.
# A list of size 1 with 'True' indicates a standard TaxBrain run
# A list of size > 1 indicates a file-based reform was run, where each
# index indicates whether the reform dictionary was non-empty
# The four reform dictionaries from file-based reforms are:
# policy, behavior, growth, consumption (in that order)
first_budget_year = int(first_budget_year)
user_mods = convert_int_key(user_mods)
print('first_year', first_budget_year)
for reform_year in user_mods.keys():
if user_mods.get(reform_year):
for key in set(user_mods[reform_year]):
if key.startswith('_BE_'):
user_mods[reform_year].pop(key)
user_reform = {"policy": user_mods}
print('user_reform', user_reform, user_mods)
if beh_params:
for x, y in beh_params.items():
for z in y.keys():
if z.isdigit():
convert_int_key(y)
user_reform.update(beh_params)
for key in EXPECTED_KEYS:
if key not in user_reform:
user_reform[key] = {}
kw = dict(year_n=year, start_year=first_budget_year,
taxrec_df=tax_data, user_mods=user_reform)
print('keywords to dropq', {k: v for k, v in kw.items()
if k not in ('taxrec_df',)})
(mY_dec_i, mX_dec_i, df_dec_i, pdf_dec_i, cdf_dec_i, mY_bin_i, mX_bin_i,
df_bin_i, pdf_bin_i, cdf_bin_i, fiscal_tot_i,
fiscal_tot_i_bl, fiscal_tot_i_ref) = taxcalc.dropq.run_nth_year_tax_calc_model(**kw)

results = {'mY_dec': mY_dec_i, 'mX_dec': mX_dec_i, 'df_dec': df_dec_i,
'pdf_dec': pdf_dec_i, 'cdf_dec': cdf_dec_i, 'mY_bin': mY_bin_i,
'mX_bin': mX_bin_i, 'df_bin': df_bin_i, 'pdf_bin': pdf_bin_i,
'cdf_bin': cdf_bin_i, 'fiscal_tot_diffs': fiscal_tot_i,
'fiscal_tot_base': fiscal_tot_i_bl,
'fiscal_tot_ref': fiscal_tot_i_ref}

def dropq_task(year_n, user_mods, first_budget_year, use_puf_not_cps=True, use_full_sample=True):
user_mods["policy"] = convert_int_key(user_mods["policy"])

print(
'keywords to dropq',
dict(
year_n=year_n,
start_year=int(first_budget_year),
use_puf_not_cps=use_puf_not_cps,
use_full_sample=use_full_sample,
user_mods=user_mods
)
)

results = taxcalc.tbi.run_nth_year_tax_calc_model(
year_n=year_n,
start_year=int(first_budget_year),
use_puf_not_cps=use_puf_not_cps,
use_full_sample=use_full_sample,
user_mods=user_mods
)

#Add taxcalc version to results
vinfo = taxcalc._version.get_versions()
Expand All @@ -105,36 +78,40 @@ def dropq_task(year, user_mods, first_budget_year, beh_params, tax_data):


@celery_app.task
def dropq_task_async(year, user_mods, first_budget_year, beh_params):
print('dropq_task_async', year, user_mods, first_budget_year, beh_params)
return dropq_task(year, user_mods, first_budget_year, beh_params, tax_dta_full)
def dropq_task_async(year, user_mods, first_budget_year):
print('dropq_task_async', year, user_mods, first_budget_year)
return dropq_task(year, user_mods, first_budget_year,
use_puf_not_cps=True, use_full_sample=True)


@celery_app.task
def dropq_task_small_async(year, user_mods, first_budget_year, beh_params):
return dropq_task(year, user_mods, first_budget_year, beh_params, tax_dta)
def dropq_task_small_async(year, user_mods, first_budget_year):
return dropq_task(year, user_mods, first_budget_year,
use_puf_not_cps=True, use_full_sample=False)


@celery_app.task
def elasticity_gdp_task_async(year, user_mods, first_budget_year, elast_params):

if first_budget_year:
first_year = int(first_budget_year)
else:
first_year = int(user_mods.keys()[0])
user_mods = convert_int_key(user_mods)
user_reform = {'policy': user_mods, 'gdp_elasticity': {'value': elast_params}}
# combine elast_params with user_mods
for key in EXPECTED_KEYS:
if key not in user_reform:
user_reform[key] = {}
# combine elast_params with user_mods
print("ELASTICITY user mods: ", user_reform)

gdp_elast_i = taxcalc.dropq.run_nth_year_gdp_elast_model(year_n=year,
start_year=first_year,
taxrec_df=tax_dta,
user_mods=user_reform)
def elasticity_gdp_task_async(year_n, user_mods, first_budget_year,
gdp_elasticity, use_puf_not_cps=True):

user_mods["policy"] = convert_int_key(user_mods["policy"])
print("kw to dropq", dict(
year_n=year_n,
start_year=int(first_budget_year),
use_puf_not_cps=use_puf_not_cps,
use_full_sample=True,
user_mods=user_mods,
gdp_elasticity=gdp_elasticity
))

gdp_elast_i = taxcalc.tbi.run_nth_year_gdp_elast_model(
year_n=year_n,
start_year=int(first_budget_year),
use_puf_not_cps=use_puf_not_cps,
use_full_sample=True,
user_mods=user_mods,
gdp_elasticity=gdp_elasticity
)

results = {'elasticity_gdp': gdp_elast_i}
#Add taxcalc version to results
Expand Down
22 changes: 7 additions & 15 deletions deploy/taxbrain_server/flask_server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,16 @@ def dropq_endpoint(dropq_task):
if request.method == 'POST':
year_n = request.form['year']
user_mods = json.loads(request.form['user_mods'])
beh_params = None if 'behavior_params' not in request.form else json.loads(request.form['behavior_params'])
first_budget_year = None if 'first_budget_year' not in request.form else request.form['first_budget_year']
else:
year_n = request.args.get('year', '')
first_budget_year = None
beh_params = None

year_n = int(year_n)
print("beh params", beh_params, " user_mods: ", user_mods)
raw_results = dropq_task.delay(year_n, user_mods, first_budget_year, beh_params)
print("year_n", year_n)
print("user_mods", user_mods)
print("first_budget_year", first_budget_year)
raw_results = dropq_task.delay(year_n, user_mods, first_budget_year)
RUNNING_JOBS[raw_results.id] = raw_results
length = client.llen(queue_name) + 1
results = {'job_id':str(raw_results), 'qlength':length}
Expand Down Expand Up @@ -108,19 +108,11 @@ def btax_endpoint():

@app.route("/elastic_gdp_start_job", methods=['POST'])
def elastic_endpoint():

# TODO: this assumes a single year is the key at highest
# level.
user_mods = tuple(json.loads(request.form['user_mods']).values())[0]
user_mods = json.loads(request.form['user_mods'])
year_n = int(request.form['year'])
print('user_mods', user_mods, 'year_n', year_n)
user_mods = json.loads(request.form['user_mods'])
elast_params = None if 'elasticity_params' not in request.form else json.loads(request.form['elasticity_params'])
first_budget_year = None if 'first_budget_year' not in request.form else request.form['first_budget_year']
user_mods = {int(k): v for k, v in user_mods.iteritems()}
elast_params = elast_params or user_mods.values()[0].pop('elastic_gdp', elast_params)
if not isinstance(elast_params, (float, int)):
elast_params = float(elast_params[0])
elast_params = json.loads(request.form['gdp_elasticity'])
first_budget_year = request.form['first_budget_year']
print("elast params", elast_params, " user_mods: ", user_mods)
raw_results = elasticity_gdp_task_async.delay(year_n, user_mods, first_budget_year, elast_params)
RUNNING_JOBS[raw_results.id] = raw_results
Expand Down
42 changes: 21 additions & 21 deletions static/js/taxbrain-tablebuilder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,31 +106,31 @@ $(function() {
rowLabels: function() {
if (this.get('grouping') == 'bin') {
return [
'Less than 10',
'10-20',
'20-30',
'30-40',
'40-50',
'50-75',
'75-100',
'100-200',
'200-500',
'500-1000',
'1000+',
'<$10K',
'$10-20K',
'$20-30K',
'$30-40K',
'$40-50K',
'$50-75K',
'$75-100K',
'$100-200K',
'$200-500K',
'$500-1000K',
'>$1000K',
'All'
];
} else if (this.get('grouping') == 'dec') {
return [
'0-10%',
'10-20%',
'20-30%',
'30-40%',
'40-50%',
'50-60%',
'60-70%',
'70-80%',
'80-90%',
'90-100%',
'0-10',
'10-20',
'20-30',
'30-40',
'40-50',
'50-60',
'60-70',
'70-80',
'80-90',
'90-100',
'All'
];
}
Expand Down
42 changes: 21 additions & 21 deletions staticfiles/js/taxbrain-tablebuilder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,31 +106,31 @@ $(function() {
rowLabels: function() {
if (this.get('grouping') == 'bin') {
return [
'Less than 10',
'10-20',
'20-30',
'30-40',
'40-50',
'50-75',
'75-100',
'100-200',
'200-500',
'500-1000',
'1000+',
'<$10K',
'$10-20K',
'$20-30K',
'$30-40K',
'$40-50K',
'$50-75K',
'$75-100K',
'$100-200K',
'$200-500K',
'$500-1000K',
'>$1000K',
'All'
];
} else if (this.get('grouping') == 'dec') {
return [
'0-10%',
'10-20%',
'20-30%',
'30-40%',
'40-50%',
'50-60%',
'60-70%',
'70-80%',
'80-90%',
'90-100%',
'0-10',
'10-20',
'20-30',
'30-40',
'40-50',
'50-60',
'60-70',
'70-80',
'80-90',
'90-100',
'All'
];
}
Expand Down
17 changes: 11 additions & 6 deletions webapp/apps/btax/compute.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,18 @@ class DropqComputeBtax(DropqCompute):
num_budget_years = 1
package_up_vars = package_up_vars

def submit_btax_calculation(self, mods, first_budget_year=2015):
def submit_btax_calculation(self, user_mods, first_budget_year=2015):
url_template = "http://{hn}/btax_start_job"
return self.submit_calculation(mods, first_budget_year, url_template,
start_budget_year=None, num_years=1,
data = {}
user_mods = self.package_up_vars(user_mods, first_budget_year)
if not bool(user_mods):
return False
user_mods = {first_budget_year: user_mods}
data['user_mods'] = json.dumps(user_mods)
data['first_budget_year'] = str(first_budget_year)
data['start_budget_year'] = '0'
data['num_budget_years'] = 1
return self.submit_calculation(data, url_template,
workers=BTAX_WORKERS,
increment_counter=False,
use_wnc_offset=False)
Expand Down Expand Up @@ -97,6 +105,3 @@ class NodeDownComputeBtax(NodeDownCompute, DropqComputeBtax):
dropq_get_results = partial(mock_dropq_get_results, 'Failure message')
submit_calculation = mock_submit_calculation
dropq_results_ready = partial(mock_dropq_results_ready, "FAIL")



2 changes: 1 addition & 1 deletion webapp/apps/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,5 @@
COC_TOOLTIP = "The cost of capital is calculated as the net-of-depreciation, before-tax rate of return."
DPRC_TOOLTIP = "Net present value of depreciation deductions."

START_YEARS = ('2013', '2014', '2015', '2016', '2017')
START_YEARS = ('2013', '2014', '2015', '2016', '2017', '2018')
START_YEAR = os.environ.get('START_YEAR', '2017')
4 changes: 2 additions & 2 deletions webapp/apps/dynamic/compute.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
ENFORCE_REMOTE_VERSION_CHECK = os.environ.get('ENFORCE_VERSION', 'False') == 'True'
TIMEOUT_IN_SECONDS = 1.0
MAX_ATTEMPTS_SUBMIT_JOB = 20
TAXCALC_RESULTS_TOTAL_ROW_KEYS = taxcalc.dropq.TOTAL_ROW_NAMES
ELASTIC_RESULTS_TOTAL_ROW_KEYS = ["gdp_elasticity"]
AGG_ROW_NAMES = taxcalc.tbi_utils.AGGR_ROW_NAMES
GDP_ELAST_ROW_NAMES = taxcalc.tbi.GDP_ELAST_ROW_NAMES
ogusa_workers = os.environ.get('OGUSA_WORKERS', '')
OGUSA_WORKERS = ogusa_workers.split(",")
CALLBACK_HOSTNAME = os.environ.get('CALLBACK_HOSTNAME', 'localhost:8000')
Expand Down

0 comments on commit 5a9430e

Please sign in to comment.