Skip to content

Commit

Permalink
Merge branch 'master' into fix-requirements-files-and-tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@desilinguist
desilinguist committed Sep 14, 2019
2 parents 2fcd1d7 + fdf5ea6 commit fdea1f4
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 26 deletions.
4 changes: 4 additions & 0 deletions skll/experiments.py
View file
Expand Up @@ -433,6 +433,10 @@ def _load_featureset(dir_path, feat_files, suffix, id_col='id', label_col='y',
num_features=num_features,
logger=logger).read()
else:
if len(feat_files) > 1 and feature_hasher:
logger.warning("Since there are multiple feature files, "
"feature hashing applies to each specified "
"feature file separately.")
merged_set = None
for file_name in sorted(join(dir_path, featfile + suffix) for
featfile in feat_files):
Expand Down
7 changes: 7 additions & 0 deletions tests/configs/test_warning_multiple_featuresets.template.cfg
View file
@@ -0,0 +1,7 @@
[General]

[Input]

[Tuning]

[Output]
152 changes: 127 additions & 25 deletions tests/test_output.py
View file
Expand Up @@ -13,6 +13,7 @@
import csv
import json
import os
import re
import sys
import warnings

Expand Down Expand Up @@ -77,8 +78,8 @@ def tearDown():
test_dir = join(_my_dir, 'test')
output_dir = join(_my_dir, 'output')
config_dir = join(_my_dir, 'configs')

for suffix in ['learning_curve', 'summary', 'fancy_xval']:
for suffix in ['learning_curve', 'summary', 'fancy_xval',
'warning_multiple_featuresets']:
if exists(join(train_dir, 'test_{}.jsonlines'.format(suffix))):
os.unlink(join(train_dir, 'test_{}.jsonlines'.format(suffix)))

Expand All @@ -94,8 +95,9 @@ def tearDown():
if exists(join(config_dir, cf)):
os.unlink(join(config_dir, cf))

for output_file in (glob(join(output_dir, 'test_{}_*'.format(suffix))) +
glob(join(output_dir, 'test_majority_class_custom_learner_*'))):
for output_file in (glob(join(output_dir, 'test_{}_*'.format(suffix)))
+ glob(join(output_dir, 'test_{}.log'.format(suffix)))
+ glob(join(output_dir, 'test_majority_class_custom_learner_*'))):
os.unlink(output_file)

for suffix in _VALID_TASKS:
Expand Down Expand Up @@ -366,7 +368,7 @@ def check_xval_fancy_results_file(do_grid_search,
with open(results_file_path, 'r') as resultsf:
results_lines = resultsf.readlines()
end_idx = [results_lines.index(l) for l in results_lines if l.startswith('Total Time:')][0]
results_lines = results_lines[:end_idx+1]
results_lines = results_lines[:end_idx + 1]

# read in the "keys" and "values" separated by colons into a dictionary
results_dict = dict([rl.strip().split(': ') for rl in results_lines])
Expand Down Expand Up @@ -407,6 +409,53 @@ def check_xval_fancy_results_file(do_grid_search,
sorted(expected_metrics))


def test_multiple_featuresets_and_featurehasher_throws_warning():
'''
test using multiple feature sets with feature hasher throws warning
'''
train_dir = join(_my_dir, 'train')
output_dir = join(_my_dir, 'output')

# make a simple config file for feature hasher warning test
values_to_fill_dict = {'experiment_name': 'test_warning_multiple_featuresets',
'train_directory': train_dir,
'task': 'train',
'grid_search': 'false',
'objectives': "['f1_score_micro']",
'learners': "['LogisticRegression']",
'featuresets': ("[['test_input_3examples_1', "
"'test_input_3examples_2']]"),
"featureset_names": "['feature_hasher']",
'suffix': '.jsonlines',
'log': output_dir,
'models': output_dir,
'feature_hasher': "true",
"hasher_features": "4"
}

config_template_path = join(_my_dir,
'configs',
'test_warning_multiple_featuresets.template.cfg')

config_path = fill_in_config_options(config_template_path,
values_to_fill_dict,
'feature_hasher')

# run the experiment
print(config_path)
run_configuration(config_path, quiet=True)

# test if it throws any warning
logfile_path = join(_my_dir, "output",
"test_warning_multiple_featuresets_feature_hasher_LogisticRegression.log")
with open(logfile_path) as f:
warning_pattern = re.compile('Since there are multiple feature files, '
'feature hashing applies to each '
'specified feature file separately.')
matches = re.findall(warning_pattern, f.read())
eq_(len(matches), 1)


def test_xval_fancy_results_file():

for (do_grid_search,
Expand All @@ -425,7 +474,10 @@ def test_xval_fancy_results_file():
def check_grid_search_cv_results(task, do_grid_search):
learners = ['LogisticRegression', 'SVC']
expected_path = join(_my_dir, 'other', 'cv_results')
time_field = lambda x: x.endswith('_time')

def time_field(x):
return x.endswith('_time')

train_path = join(_my_dir, 'train', 'f0.jsonlines')
output_dir = join(_my_dir, 'output')

Expand Down Expand Up @@ -490,7 +542,7 @@ def check_grid_search_cv_results(task, do_grid_search):
results_file_name)
ok_(exists(actual_results_file_path))
with open(expected_results_file_path) as expected, \
open(actual_results_file_path) as actual:
open(actual_results_file_path) as actual:
expected_lines = [json.loads(line) for line in expected][0]
actual_lines = [json.loads(line) for line in actual][0]
assert len(expected_lines) == len(actual_lines)
Expand All @@ -503,17 +555,17 @@ def check_grid_search_cv_results(task, do_grid_search):
assert len(expected_gs_cv_results) == len(actual_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if do_grid_search:
assert (set(expected_gs_cv_results)
.intersection(actual_gs_cv_results) ==
set(expected_gs_cv_results))
assert set(expected_gs_cv_results).intersection(actual_gs_cv_results) == \
set(expected_gs_cv_results)
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
if time_field(subfield):
continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
actual_gs_cv_results[field][subfield]
else:
if field == 'grid_score':
assert actual_gs_cv_results[field] == 0.0
Expand All @@ -531,17 +583,17 @@ def check_grid_search_cv_results(task, do_grid_search):
assert len(expected_gs_cv_results) == len(actual_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if do_grid_search:
assert (set(expected_gs_cv_results)
.intersection(actual_gs_cv_results) ==
set(expected_gs_cv_results))
assert set(expected_gs_cv_results).intersection(actual_gs_cv_results) == \
set(expected_gs_cv_results)
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
if time_field(subfield):
continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
actual_gs_cv_results[field][subfield]
else:
if field == 'grid_score':
assert actual_gs_cv_results[field] == 0.0
Expand All @@ -551,23 +603,24 @@ def check_grid_search_cv_results(task, do_grid_search):
expected_gs_cv_results = expected_lines
actual_gs_cv_results = actual_lines
assert set(expected_gs_cv_results).intersection(actual_gs_cv_results) == \
set(expected_gs_cv_results)
set(expected_gs_cv_results)
for field in ['grid_score', 'grid_search_cv_results']:
if field == 'grid_score':
assert expected_gs_cv_results[field] == \
actual_gs_cv_results[field]
actual_gs_cv_results[field]
else:
for subfield in expected_gs_cv_results[field]:
if time_field(subfield): continue
if time_field(subfield):
continue
assert expected_gs_cv_results[field][subfield] == \
actual_gs_cv_results[field][subfield]
actual_gs_cv_results[field][subfield]
else:
for expected_line, actual_line in zip(expected_lines,
actual_lines):
expected_fields = set(list(expected_line))
actual_fields = set(list(actual_line))
assert expected_fields.intersection(actual_fields) == \
expected_fields
expected_fields
assert all(field not in actual_fields
for field in ['grid_score',
'grid_search_cv_results'])
Expand Down Expand Up @@ -781,7 +834,56 @@ def test_learning_curve_ylimits():
"""

# create a test data frame
df_test = pd.DataFrame.from_dict({'test_score_std': {0: 0.16809136190418694, 1: 0.18556201422712379, 2: 0.15002727816517414, 3: 0.15301923832338646, 4: 0.15589815327205431, 5: 0.68205316443171948, 6: 0.77441075727706354, 7: 0.83838056331276678, 8: 0.84770116657005623, 9: 0.8708014559726478}, 'value': {0: 0.4092496971394447, 1: 0.2820507715115001, 2: 0.24533811547921261, 3: 0.21808651942296109, 4: 0.19767367891431534, 5: -2.3540980769230773, 6: -3.1312445327182394, 7: -3.2956790939674137, 8: -3.4843050005436713, 9: -3.6357879085645455}, 'train_score_std': {0: 0.15950199460682787, 1: 0.090992452273091703, 2: 0.068488654201949981, 3: 0.055223120652733763, 4: 0.03172452509259388, 5: 1.0561586240460523, 6: 0.53955995320300709, 7: 0.40477740983901211, 8: 0.34148185048394258, 9: 0.20791478156554272}, 'variable': {0: 'train_score_mean', 1: 'train_score_mean', 2: 'train_score_mean', 3: 'train_score_mean', 4: 'train_score_mean', 5: 'train_score_mean', 6: 'train_score_mean', 7: 'train_score_mean', 8: 'train_score_mean', 9: 'train_score_mean'}, 'metric': {0: 'r2', 1: 'r2', 2: 'r2', 3: 'r2', 4: 'r2', 5: 'neg_mean_squared_error', 6: 'neg_mean_squared_error', 7: 'neg_mean_squared_error', 8: 'neg_mean_squared_error', 9: 'neg_mean_squared_error'}})
df_test = pd.DataFrame.from_dict({'test_score_std': {0: 0.16809136190418694,
1: 0.18556201422712379,
2: 0.15002727816517414,
3: 0.15301923832338646,
4: 0.15589815327205431,
5: 0.68205316443171948,
6: 0.77441075727706354,
7: 0.83838056331276678,
8: 0.84770116657005623,
9: 0.8708014559726478},
'value': {0: 0.4092496971394447,
1: 0.2820507715115001,
2: 0.24533811547921261,
3: 0.21808651942296109,
4: 0.19767367891431534,
5: -2.3540980769230773,
6: -3.1312445327182394,
7: -3.2956790939674137,
8: -3.4843050005436713,
9: -3.6357879085645455},
'train_score_std': {0: 0.15950199460682787,
1: 0.090992452273091703,
2: 0.068488654201949981,
3: 0.055223120652733763,
4: 0.03172452509259388,
5: 1.0561586240460523,
6: 0.53955995320300709,
7: 0.40477740983901211,
8: 0.34148185048394258,
9: 0.20791478156554272},
'variable': {0: 'train_score_mean',
1: 'train_score_mean',
2: 'train_score_mean',
3: 'train_score_mean',
4: 'train_score_mean',
5: 'train_score_mean',
6: 'train_score_mean',
7: 'train_score_mean',
8: 'train_score_mean',
9: 'train_score_mean'},
'metric': {0: 'r2',
1: 'r2',
2: 'r2',
3: 'r2',
4: 'r2',
5: 'neg_mean_squared_error',
6: 'neg_mean_squared_error',
7: 'neg_mean_squared_error',
8: 'neg_mean_squared_error',
9: 'neg_mean_squared_error'}})

# compute the y-limits
ylimits_dict = _compute_ylimits_for_featureset(df_test, ['r2', 'neg_mean_squared_error'])
Expand Down
3 changes: 2 additions & 1 deletion tests/utils.py
View file
Expand Up @@ -151,7 +151,8 @@ def fill_in_config_options(config_template_path,
'sampler', 'shuffle', 'feature_scaling',
'learning_curve_cv_folds_list', 'folds_file',
'learning_curve_train_sizes', 'fixed_parameters',
'num_cv_folds', 'bad_option', 'duplicate_option'],
'num_cv_folds', 'bad_option', 'duplicate_option',
'suffix'],
'Tuning': ['probability', 'grid_search', 'objective',
'use_folds_file_for_grid_search', 'grid_search_folds',
'param_grids', 'objectives', 'duplicate_option'],
Expand Down

0 comments on commit fdea1f4

Please sign in to comment.