From 091b149af85c0c74784e8cc4d5f5fb004cb73e78 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Tue, 15 Sep 2020 15:49:14 -0400 Subject: [PATCH 01/15] Try to address nbconvert API change. --- rsmtool/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsmtool/reporter.py b/rsmtool/reporter.py index 852166c9e..34e4d933d 100644 --- a/rsmtool/reporter.py +++ b/rsmtool/reporter.py @@ -320,7 +320,7 @@ def convert_ipynb_to_html(notebook_file, html_file): # set a high timeout for datasets with a large number of features report_config = Config({'ExecutePreprocessor': {'enabled': True, 'timeout': 3600}, - 'HTMLExporter': {'template_path': [template_path], + 'HTMLExporter': {'template_paths': [template_path], 'template_file': 'report.tpl'}}) exportHtml = HTMLExporter(config=report_config) From 8502369f86fff71c27bf2221f6016e2475d8fd0f Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Mon, 21 Sep 2020 14:29:41 -0400 Subject: [PATCH 02/15] Revert change for newer version of nbconvert. --- rsmtool/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsmtool/reporter.py b/rsmtool/reporter.py index 34e4d933d..852166c9e 100644 --- a/rsmtool/reporter.py +++ b/rsmtool/reporter.py @@ -320,7 +320,7 @@ def convert_ipynb_to_html(notebook_file, html_file): # set a high timeout for datasets with a large number of features report_config = Config({'ExecutePreprocessor': {'enabled': True, 'timeout': 3600}, - 'HTMLExporter': {'template_paths': [template_path], + 'HTMLExporter': {'template_path': [template_path], 'template_file': 'report.tpl'}}) exportHtml = HTMLExporter(config=report_config) From aa5e6cc474a234c043cbc5782f15c779717457e5 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Mon, 21 Sep 2020 14:29:52 -0400 Subject: [PATCH 03/15] Pin nbconvert to older version --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 58c835a82..cfbdf28c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ doc2dash jupyter nose notebook +nbconvert<6.0 numpy pandas seaborn From 8971bf51f2bf40c9f934bb9a1c966ecd0137c913 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Mon, 21 Sep 2020 14:30:51 -0400 Subject: [PATCH 04/15] Improve seaborn calls to use new API --- rsmtool/notebooks/builtin_model.ipynb | 2 +- rsmtool/notebooks/comparison/evaluation.ipynb | 2 +- rsmtool/notebooks/evaluation.ipynb | 2 +- rsmtool/notebooks/evaluation_by_group.ipynb | 2 +- rsmtool/notebooks/feature_descriptives.ipynb | 2 +- rsmtool/notebooks/preprocessed_features.ipynb | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rsmtool/notebooks/builtin_model.ipynb b/rsmtool/notebooks/builtin_model.ipynb index 4f3523f7f..ed72bc1b7 100644 --- a/rsmtool/notebooks/builtin_model.ipynb +++ b/rsmtool/notebooks/builtin_model.ipynb @@ -144,7 +144,7 @@ "grey_colors = sns.color_palette('Greys', len(features_used))[::-1]\n", "with sns.axes_style('whitegrid'):\n", " ax1=fig.add_subplot(121)\n", - " sns.barplot(\"feature\",\"standardized\", data=df_betas_sorted, \n", + " sns.barplot(x=\"feature\", y=\"standardized\", data=df_betas_sorted, \n", " order=df_betas_sorted['feature'].values,\n", " palette=sns.color_palette(\"Greys\", 1), ax=ax1)\n", " ax1.set_xticklabels(df_betas_sorted['feature'].values, rotation=90)\n", diff --git a/rsmtool/notebooks/comparison/evaluation.ipynb b/rsmtool/notebooks/comparison/evaluation.ipynb index 29aefb483..85ad8d887 100644 --- a/rsmtool/notebooks/comparison/evaluation.ipynb +++ b/rsmtool/notebooks/comparison/evaluation.ipynb @@ -105,7 +105,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.6.11" } }, "nbformat": 4, diff --git a/rsmtool/notebooks/evaluation.ipynb b/rsmtool/notebooks/evaluation.ipynb index 9863f2463..71beaa14e 100644 --- a/rsmtool/notebooks/evaluation.ipynb +++ b/rsmtool/notebooks/evaluation.ipynb @@ -191,7 +191,7 @@ "\n", " # make a barplot without a legend since we will \n", " # add one manually later\n", - " p = sns.catplot(\"score\", \"value\", \"variable\", kind=\"bar\",\n", + " p = sns.catplot(x=\"score\", y=\"value\", hue=\"variable\", kind=\"bar\",\n", " palette=colors, data=df_scoredist_melted, \n", " height=3, aspect=2, legend=False)\n", " p.set_axis_labels('score', '% of responses')\n", diff --git a/rsmtool/notebooks/evaluation_by_group.ipynb b/rsmtool/notebooks/evaluation_by_group.ipynb index 576be5eca..2312d8640 100644 --- a/rsmtool/notebooks/evaluation_by_group.ipynb +++ b/rsmtool/notebooks/evaluation_by_group.ipynb @@ -76,7 +76,7 @@ " ax = fig.add_subplot(num_rows, num_columns, i + 1)\n", " for lineval in metrics[metric]:\n", " ax.axhline(y=float(lineval), linestyle='--', linewidth=0.5, color='black')\n", - " sns.barplot(df_plot[group], df_plot[metric], color='grey', ax=ax, order=bar_names)\n", + " sns.barplot(x=df_plot[group], y=df_plot[metric], color='grey', ax=ax, order=bar_names)\n", " ax.set_xticklabels(wrapped_bar_names, rotation=90) \n", " ax.set_xlabel('')\n", " ax.set_ylabel('')\n", diff --git a/rsmtool/notebooks/feature_descriptives.ipynb b/rsmtool/notebooks/feature_descriptives.ipynb index c7abedb4f..9244897af 100644 --- a/rsmtool/notebooks/feature_descriptives.ipynb +++ b/rsmtool/notebooks/feature_descriptives.ipynb @@ -80,7 +80,7 @@ "with sns.axes_style('whitegrid'):\n", " # create a barplot without a legend since we will manually\n", " # add one later\n", - " p = sns.catplot(\"feature\", \"value\", \"variable\", kind=\"bar\", \n", + " p = sns.catplot(x=\"feature\", y=\"value\", hue=\"variable\", kind=\"bar\", \n", " palette=colors, data=df_outliers, height=height, \n", " aspect=aspect, legend=False)\n", " p.set_axis_labels('', '% cases truncated\\nto mean +/- 4*sd')\n", diff --git a/rsmtool/notebooks/preprocessed_features.ipynb b/rsmtool/notebooks/preprocessed_features.ipynb index b3208507f..622768d3c 100644 --- a/rsmtool/notebooks/preprocessed_features.ipynb +++ b/rsmtool/notebooks/preprocessed_features.ipynb @@ -208,7 +208,7 @@ "\n", " # generate a bar plot but without the legend since we will\n", " # manually add one later\n", - " p = sns.catplot(\"feature\", \"value\", \"variable\", kind=\"bar\",\n", + " p = sns.catplot(x=\"feature\", y=\"value\", hue=\"variable\", kind=\"bar\",\n", " palette=colors, data=df_mpcor, height=height, \n", " aspect=aspect, legend=False)\n", " p.set_axis_labels('', 'Correlation with score')\n", @@ -289,7 +289,7 @@ " \n", " # create a barplot but without the legend since\n", " # we will manually add one later\n", - " p = sns.catplot(\"feature\", \"value\", \"variable\", kind=\"bar\",\n", + " p = sns.catplot(x=\"feature\", y=\"value\", hue=\"variable\", kind=\"bar\",\n", " palette=colors, data=df_mpcor, height=height, \n", " aspect=aspect, legend=False)\n", " p.set_axis_labels('', 'Correlation with length')\n", From 2a776fa949784c44f93d17c94ffaaba2ccc41214 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Tue, 22 Sep 2020 10:28:38 -0400 Subject: [PATCH 05/15] Revert unnecessary change. --- rsmtool/notebooks/comparison/evaluation.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsmtool/notebooks/comparison/evaluation.ipynb b/rsmtool/notebooks/comparison/evaluation.ipynb index 85ad8d887..29aefb483 100644 --- a/rsmtool/notebooks/comparison/evaluation.ipynb +++ b/rsmtool/notebooks/comparison/evaluation.ipynb @@ -105,7 +105,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.11" + "version": "3.7.6" } }, "nbformat": 4, From efda9ff55f42a930fa6d0d1192b90716f8cb1b40 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Mon, 28 Sep 2020 13:38:36 -0400 Subject: [PATCH 06/15] Pin ipython to keep compatibility with python 3.6 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index cfbdf28c3..13a0355b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ doc2dash jupyter +ipython<7.17 nose notebook nbconvert<6.0 From 7688efca9c45e76418b5de731add50441b483600 Mon Sep 17 00:00:00 2001 From: Anastassia Loukina Date: Mon, 28 Sep 2020 16:10:08 -0400 Subject: [PATCH 07/15] Add explicit normalization --- rsmtool/notebooks/builtin_model.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rsmtool/notebooks/builtin_model.ipynb b/rsmtool/notebooks/builtin_model.ipynb index ed72bc1b7..47e29a429 100644 --- a/rsmtool/notebooks/builtin_model.ipynb +++ b/rsmtool/notebooks/builtin_model.ipynb @@ -154,8 +154,10 @@ " # no pie chart if we have more than 15 features or if the feature names are long (pie chart looks ugly)\n", " if len(features_used) <= 15 and longest_feature_name <= 10:\n", " ax2=fig.add_subplot(133, aspect=True)\n", + " # we set normalize to False to make sure we have a partial pie in case \n", + " # of negative coefficients.\n", " ax2.pie(abs(df_betas_sorted['relative'].values), colors=grey_colors, \n", - " labels=df_betas_sorted['feature'].values)\n", + " labels=df_betas_sorted['feature'].values, normalize=False)\n", " ax2.set_title('Proportional contribution of each feature')\n", " else:\n", " fig.set_size_inches(len(features_used), 3)\n", @@ -245,7 +247,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.8.5" } }, "nbformat": 4, From f9c3f589cb0aa9e5abbad3707438d27ac24c8d2c Mon Sep 17 00:00:00 2001 From: Anastassia Loukina Date: Mon, 28 Sep 2020 16:28:34 -0400 Subject: [PATCH 08/15] 466-add-info-about-strict-mode-to-section-about-tests --- doc/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index b271c97e9..f96623a67 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -179,6 +179,8 @@ Here are some advanced tips and tricks when working with RSMTool tests. 4. The ``--pdb-errors`` and ``--pdb-failures`` options for ``nosetests`` are your friends. If you encounter test errors or test failures where the cause may not be immediately clear, re-run the ``nosetests`` command with the appropriate option. Doing so will drop you into an interactive PDB session as soon as a error (or failure) is encountered and then you inspect the variables at that point (or use "u" and "d" to go up and down the call stack). This may be particularly useful for tests in ``tests/test_cli.py`` that use ``subprocess.run()``. If these tests are erroring out, use ``--pdb-errors`` and inspect the "stderr" variable in the resulting PDB session to see what the error is. +5. In RSMTool 8.0.1 and later by default the tests will pass even if the reports contain warnings. To catch any warnings that may appear in the reports run the tests in strict mode (``STRICT=1 nosetests --nologcapture tests``). + .. rubric:: Footnotes .. [#] For older versions of conda, you may have to do ``source activate rsmtool`` on Linux/macOS and ``activate rsmtool`` on Windows. From 619d9fad43a4690fd614752c9934c648a20450ee Mon Sep 17 00:00:00 2001 From: Anastassia Loukina Date: Mon, 28 Sep 2020 17:01:38 -0400 Subject: [PATCH 09/15] Remove pie chart for negative coefficients; set normalize to true --- rsmtool/notebooks/builtin_model.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rsmtool/notebooks/builtin_model.ipynb b/rsmtool/notebooks/builtin_model.ipynb index 47e29a429..cdf3572d8 100644 --- a/rsmtool/notebooks/builtin_model.ipynb +++ b/rsmtool/notebooks/builtin_model.ipynb @@ -151,13 +151,13 @@ " ax1.set_title('Values of standardized coefficients')\n", " ax1.set_xlabel('')\n", " ax1.set_ylabel('')\n", - " # no pie chart if we have more than 15 features or if the feature names are long (pie chart looks ugly)\n", - " if len(features_used) <= 15 and longest_feature_name <= 10:\n", + " # no pie chart if we have more than 15 features,\n", + " # if the feature names are long (pie chart looks ugly)\n", + " # or if there are any negative coefficients.\n", + " if len(features_used) <= 15 and longest_feature_name <= 10 and (df_betas_sorted['relative']>=0).all():\n", " ax2=fig.add_subplot(133, aspect=True)\n", - " # we set normalize to False to make sure we have a partial pie in case \n", - " # of negative coefficients.\n", " ax2.pie(abs(df_betas_sorted['relative'].values), colors=grey_colors, \n", - " labels=df_betas_sorted['feature'].values, normalize=False)\n", + " labels=df_betas_sorted['feature'].values, normalize=True)\n", " ax2.set_title('Proportional contribution of each feature')\n", " else:\n", " fig.set_size_inches(len(features_used), 3)\n", From 9d866887510cafd44631ad64c02656ef0fa51cfe Mon Sep 17 00:00:00 2001 From: Anastassia Loukina Date: Mon, 28 Sep 2020 17:53:29 -0400 Subject: [PATCH 10/15] Pep8 fixes and use of warnings where appropriate --- rsmtool/configuration_parser.py | 83 ++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/rsmtool/configuration_parser.py b/rsmtool/configuration_parser.py index 11783b073..917c0ddf3 100644 --- a/rsmtool/configuration_parser.py +++ b/rsmtool/configuration_parser.py @@ -1,4 +1,6 @@ """ +Configuration parser. + Classes related to parsing configuration files and creating configuration objects. @@ -11,6 +13,7 @@ import json import logging +import warnings import re from copy import copy, deepcopy @@ -41,6 +44,8 @@ def configure(context, config_file_or_obj_or_dict): """ + Create a Configuration object. + Get the configuration for ``context`` from the input ``config_file_or_obj_or_dict``. @@ -112,7 +117,9 @@ def configure(context, config_file_or_obj_or_dict): class Configuration: """ - Configuration class, which encapsulates all of the + Configuration class. + + Encapsulates all of the configuration parameters and methods to access these parameters. """ @@ -145,7 +152,6 @@ def __init__(self, The context of the tool. Defaults to 'rsmtool'. """ - if not isinstance(configdict, dict): raise TypeError('The input must be a dictionary.') @@ -166,6 +172,8 @@ def __init__(self, def __contains__(self, key): """ + Key check. + Check if configuration object contains a given key. @@ -223,6 +231,8 @@ def __len__(self): def __str__(self): """ + Return string representation. + Return a string representation of the underlying configuration dictionary. @@ -255,6 +265,8 @@ def __iter__(self): @property def configdir(self): """ + Get the path to configuration directory. + Get the path to the configuration reference directory that will be used to resolve any relative paths in the configuration. @@ -269,7 +281,7 @@ def configdir(self): @configdir.setter def configdir(self, new_path): """ - Set a new configuration reference directory + Set a new configuration reference directory. Parameters ---------- @@ -278,7 +290,6 @@ def configdir(self, new_path): directory used to resolve any relative paths in the configuration object. """ - if new_path is None: raise ValueError("The `configdir` attribute cannot be set to `None` ") @@ -286,18 +297,15 @@ def configdir(self, new_path): # once this Windows bug is fixed: https://bugs.python.org/issue38671 self._configdir = Path(abspath(new_path)) - @property def context(self): - """ - Get the context. - """ + """Get the context.""" return self._context @context.setter def context(self, new_context): """ - Set a new context + Set a new context. Parameters ---------- @@ -373,6 +381,8 @@ def items(self): def pop(self, key, default=None): """ + Remove and return value. + Remove and returns an element from the object having the given key. @@ -419,7 +429,6 @@ def save(self, output_dir=None): output_dir : str The path to the output directory. """ - # save a copy of the main config into the output directory if output_dir is None: output_dir = Path(getcwd()) @@ -438,6 +447,8 @@ def save(self, output_dir=None): def check_exclude_listwise(self): """ + Check for candidate exclusion. + Check if we are excluding candidates based on number of responses, and add this to the configuration file @@ -456,8 +467,9 @@ def check_flag_column(self, flag_column='flag_column', partition='unknown'): """ - Make sure the `flag_column` field is in the correct format. Get - flag columns and values for filtering if any and convert single + Make sure the `flag_column` field is in the correct format. + + Get flag columns and values for filtering if any and convert single values to lists. Raises an exception if `flag_column` is not correctly specified. @@ -549,9 +561,10 @@ def check_flag_column(self, model_eval)) return new_filter_dict - def get_trim_min_max_tolerance(self): """ + Get trim min, trim max and tolerance. + Get the specified trim min and max, and trim_tolerance if any, and make sure they are numeric. @@ -621,6 +634,7 @@ def get_default_converter(self): def get_names_and_paths(self, keys, names): """ Get a a list of values, given keys. + Remove any values that are None. Parameters @@ -641,7 +655,6 @@ def get_names_and_paths(self, keys, names): ValueError If there are any duplicate keys or names. """ - assert len(keys) == len(names) # Make sure keys are not duplicated @@ -676,9 +689,7 @@ def get_names_and_paths(self, keys, names): class ConfigurationParser: - """ - A `ConfigurationParser` class to create a `Configuration` object. - """ + """A `ConfigurationParser` class to create a `Configuration` object.""" def __init__(self, pathlike): """ @@ -728,7 +739,9 @@ def __init__(self, pathlike): @staticmethod def _fix_json(json_string): """ - Takes a bit of JSON that might have bad quotes + Fix json. + + Take a bit of JSON that might have bad quotes or capitalized booleans and fixes that stuff. Parameters @@ -748,6 +761,8 @@ def _fix_json(json_string): def _parse_json_file(self, filepath): """ + Parse json. + A private method to parse JSON configuration files and return a Python dictionary. @@ -778,9 +793,10 @@ def _parse_json_file(self, filepath): return configdict - def parse(self, context='rsmtool'): """ + Parse the configuration file. + Parse the configuration file for which this parser was instantiated. @@ -801,7 +817,6 @@ def parse(self, context='rsmtool'): A Configuration object containing the parameters in the file that we instantiated the parser for. """ - filepath = self._configdir / self._filename configdict = self._parse_json_file(filepath) @@ -815,6 +830,8 @@ def parse(self, context='rsmtool'): @classmethod def validate_config(cls, config, context='rsmtool'): """ + Validate configuration file. + Ensure that all required fields are specified, add default values values for all unspecified fields, and ensure that all specified fields are valid. @@ -844,7 +861,6 @@ def validate_config(cls, config, context='rsmtool'): ValueError If config does not exist, and no config passed. """ - # make a copy of the given parameter dictionary new_config = deepcopy(config) @@ -954,8 +970,8 @@ def validate_config(cls, config, context='rsmtool'): # linear regression model if new_config['skll_objective']: if not is_skll_model(new_config['model']): - logging.warning("You specified a custom SKLL objective but also chose a " - "non-SKLL model. The objective will be ignored.") + warnings.warn("You specified a custom SKLL objective but also chose a " + "non-SKLL model. The objective will be ignored.") else: if new_config['skll_objective'] not in SCORERS: raise ValueError("Invalid SKLL objective. Please refer to the SKLL " @@ -968,9 +984,9 @@ def validate_config(cls, config, context='rsmtool'): # at run time for any invalid parameters if new_config['skll_fixed_parameters']: if not is_skll_model(new_config['model']): - logging.warning("You specified custom SKLL fixed parameters but " - "also chose a non-SKLL model. The parameters will " - "be ignored.") + warnings.warn("You specified custom SKLL fixed parameters but " + "also chose a non-SKLL model. The parameters will " + "be ignored.") # 10. Check that if we are running rsmtool to ask for # expected scores then the SKLL model type must actually @@ -994,12 +1010,12 @@ def validate_config(cls, config, context='rsmtool'): # 12. Raise a warning if we are specifiying a feature file but also # telling the system to automatically select transformations if new_config['features'] and new_config['select_transformations']: - logging.warning("You specified a feature file but also set " - "`select_transformations` to True. Any " - "transformations or signs specified in " - "the feature file will be overwritten by " - "the automatically selected transformations " - "and signs.") + warnings.warn("You specified a feature file but also set " + "`select_transformations` to True. Any " + "transformations or signs specified in " + "the feature file will be overwritten by " + "the automatically selected transformations " + "and signs.") # 13. If we have `experiment_names`, check that the length of the list # matches the list of experiment_dirs. @@ -1039,6 +1055,8 @@ def validate_config(cls, config, context='rsmtool'): @classmethod def process_config(cls, config): """ + Process configuration file. + Converts fields which are read in as string to the appropriate format. Fields which can take multiple string values are converted to lists if they have @@ -1061,7 +1079,6 @@ def process_config(cls, config): NameError If config does not exist, or no config read. """ - # Get the parameter dictionary new_config = deepcopy(config) From f4a22cb68a0494856b2fed64e30a6952c81dfb26 Mon Sep 17 00:00:00 2001 From: Anastassia Loukina Date: Mon, 28 Sep 2020 17:56:54 -0400 Subject: [PATCH 11/15] 460-tweak-warning --- rsmtool/configuration_parser.py | 14 ++++++++------ tests/test_configuration_parser.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/rsmtool/configuration_parser.py b/rsmtool/configuration_parser.py index 917c0ddf3..1bc814b82 100644 --- a/rsmtool/configuration_parser.py +++ b/rsmtool/configuration_parser.py @@ -1010,12 +1010,14 @@ def validate_config(cls, config, context='rsmtool'): # 12. Raise a warning if we are specifiying a feature file but also # telling the system to automatically select transformations if new_config['features'] and new_config['select_transformations']: - warnings.warn("You specified a feature file but also set " - "`select_transformations` to True. Any " - "transformations or signs specified in " - "the feature file will be overwritten by " - "the automatically selected transformations " - "and signs.") + # Show a warning unless a user passed a list of features. + if not isinstance(new_config['features'], list): + warnings.warn("You specified a feature file but also set " + "`select_transformations` to True. Any " + "transformations or signs specified in " + "the feature file will be overwritten by " + "the automatically selected transformations " + "and signs.") # 13. If we have `experiment_names`, check that the length of the list # matches the list of experiment_dirs. diff --git a/tests/test_configuration_parser.py b/tests/test_configuration_parser.py index 255f97f7f..12a5f5360 100644 --- a/tests/test_configuration_parser.py +++ b/tests/test_configuration_parser.py @@ -3,6 +3,7 @@ import os import tempfile import pandas as pd +import warnings from io import StringIO from os import getcwd @@ -305,6 +306,34 @@ def test_validate_config_min_n_without_subgroups(self): _ = ConfigurationParser.validate_config(data) + def test_validate_config_warning_feature_file_and_transformations(self): + data = {'experiment_id': 'experiment_1', + 'train_file': 'data/rsmtool_smTrain.csv', + 'test_file': 'data/rsmtool_smEval.csv', + 'model': 'LinearRegression', + 'select_transformations': True, + 'features': 'some_file.csv'} + + with warnings.catch_warnings(record=True) as warning_list: + _ = ConfigurationParser.validate_config(data) + eq_(len(warning_list), 1) + ok_(issubclass(warning_list[0].category, UserWarning)) + + + def test_validate_config_warning_feature_list_and_transformations(self): + # this should no show warnings + data = {'experiment_id': 'experiment_1', + 'train_file': 'data/rsmtool_smTrain.csv', + 'test_file': 'data/rsmtool_smEval.csv', + 'model': 'LinearRegression', + 'select_transformations': True, + 'features': ['feature1', 'feature2']} + + with warnings.catch_warnings(record=True) as warning_list: + _ = ConfigurationParser.validate_config(data) + eq_(len(warning_list), 0) + + def test_process_fields(self): data = {'experiment_id': 'experiment_1', 'train_file': 'data/rsmtool_smTrain.csv', From ee4b0e4a6b9ef6473374479addbff11714a96948 Mon Sep 17 00:00:00 2001 From: Anastassia Loukina Date: Tue, 29 Sep 2020 10:00:02 -0400 Subject: [PATCH 12/15] Apply suggestions from code review Co-authored-by: Nitin Madnani --- doc/contributing.rst | 2 +- rsmtool/configuration_parser.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index f96623a67..2bae46adc 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -179,7 +179,7 @@ Here are some advanced tips and tricks when working with RSMTool tests. 4. The ``--pdb-errors`` and ``--pdb-failures`` options for ``nosetests`` are your friends. If you encounter test errors or test failures where the cause may not be immediately clear, re-run the ``nosetests`` command with the appropriate option. Doing so will drop you into an interactive PDB session as soon as a error (or failure) is encountered and then you inspect the variables at that point (or use "u" and "d" to go up and down the call stack). This may be particularly useful for tests in ``tests/test_cli.py`` that use ``subprocess.run()``. If these tests are erroring out, use ``--pdb-errors`` and inspect the "stderr" variable in the resulting PDB session to see what the error is. -5. In RSMTool 8.0.1 and later by default the tests will pass even if the reports contain warnings. To catch any warnings that may appear in the reports run the tests in strict mode (``STRICT=1 nosetests --nologcapture tests``). +5. In RSMTool 8.0.1 and later, the tests will pass even if any of the reports contain warnings. To catch any warnings that may appear in the reports, run the tests in strict mode (``STRICT=1 nosetests --nologcapture tests``). .. rubric:: Footnotes diff --git a/rsmtool/configuration_parser.py b/rsmtool/configuration_parser.py index 1bc814b82..3e08a7811 100644 --- a/rsmtool/configuration_parser.py +++ b/rsmtool/configuration_parser.py @@ -563,7 +563,7 @@ def check_flag_column(self, def get_trim_min_max_tolerance(self): """ - Get trim min, trim max and tolerance. + Get trim min, trim max, and tolerance. Get the specified trim min and max, and trim_tolerance if any, @@ -689,7 +689,7 @@ def get_names_and_paths(self, keys, names): class ConfigurationParser: - """A `ConfigurationParser` class to create a `Configuration` object.""" + """A ``ConfigurationParser`` class to create a ``Configuration`` object.""" def __init__(self, pathlike): """ @@ -795,7 +795,7 @@ def _parse_json_file(self, filepath): def parse(self, context='rsmtool'): """ - Parse the configuration file. + Parse configuration file. Parse the configuration file for which this parser was instantiated. From 95d02d1c8d197c1df6ab357b9eda3b5c3c581d24 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Wed, 30 Sep 2020 11:14:35 -0400 Subject: [PATCH 13/15] Add azure pipelines badge to readme. --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index cb596642e..6d8bf198e 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,10 @@ Rater Scoring Modeling Tool :target: https://travis-ci.org/EducationalTestingService/rsmtool :alt: Build status +.. image:: https://dev.azure.com/EducationalTestingService/RSMTool/_apis/build/status/EducationalTestingService.rsmtool + :target: https://dev.azure.com/EducationalTestingService/RSMTool/_build?view=runs + :alt: Build status + .. image:: https://img.shields.io/coveralls/EducationalTestingService/rsmtool/master.svg :target: https://coveralls.io/r/EducationalTestingService/rsmtool :alt: Coverage status From b33229029fb67e8d821a9950a944436b750639b7 Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Wed, 30 Sep 2020 11:14:42 -0400 Subject: [PATCH 14/15] Fix typo in docstring. --- rsmtool/configuration_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsmtool/configuration_parser.py b/rsmtool/configuration_parser.py index 3e08a7811..cd855b299 100644 --- a/rsmtool/configuration_parser.py +++ b/rsmtool/configuration_parser.py @@ -596,8 +596,8 @@ def get_rater_error_variance(self): """ Get specified rater error variance, if any, and make sure it's numeric. - Returns: - -------- + Returns + ------- rater_error_variance : float specified rater error variance """ From 24357464342d1127beedfde4403061c18f9584ff Mon Sep 17 00:00:00 2001 From: Nitin Madnani Date: Wed, 30 Sep 2020 11:14:51 -0400 Subject: [PATCH 15/15] Update version number. --- rsmtool/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsmtool/version.py b/rsmtool/version.py index d9cd859b0..7338ba18e 100644 --- a/rsmtool/version.py +++ b/rsmtool/version.py @@ -3,5 +3,5 @@ in one place. Based on the suggestion `here. `_ """ -__version__ = '8.0.1' +__version__ = '8.0.2' VERSION = tuple(int(x) for x in __version__.split('.'))