Merge pull request #246 from EducationalTestingService/release/6.1.0

v6.1.0 Release

conda-recipe/rsmtool/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: rsmtool
-  version: 6.0.1
+  version: 6.1.0
 
 source:
   path: ../../../rsmtool
@@ -22,19 +22,19 @@ build:
 requirements:
   build:
     - python
-    - numpy>=1.13.3,<1.13.4
-    - scipy>=1.0.0,<1.0.1
-    - ipython>=6.3.1,<6.4
+    - numpy>=1.14.0,<1.15
+    - scipy>=1.1.0,<1.2.0
+    - ipython>=6.5.0,<6.5.1
     - jupyter>=1.0.0,<1.1
     - joblib>=0.11,<0.12
     - matplotlib>=2.1.2,<2.2
     - nose>=1.3.7,<1.4
-    - notebook>=5.4.1,<5.5
-    - pandas>=0.22.0,<0.22.1
+    - notebook>=5.7.2,<5.8
+    - pandas>=0.23.4,<0.23.5
     - scikit-learn>=0.19.1,<0.19.2
-    - seaborn>=0.8.1,<0.9
+    - seaborn>=0.9.0,<0.10.0
     - skll>=1.5,<1.6
-    - statsmodels>=0.8.0,<0.8.1
+    - statsmodels>=0.9.0,<0.9.1
     - sphinx
     - sphinx_rtd_theme
     - zeromq
@@ -47,25 +47,24 @@ requirements:
 
   run:
     - python
-    - numpy>=1.13.3,<1.13.4
-    - scipy>=1.0.0,<1.0.1
-    - ipython>=6.3.1,<6.4
+    - numpy>=1.14.0,<1.15
+    - scipy>=1.1.0,<1.2.0
+    - ipython>=6.5.0,<6.5.1
     - jupyter>=1.0.0,<1.1
     - joblib>=0.11,<0.12
     - matplotlib>=2.1.2,<2.2
     - nose>=1.3.7,<1.4
-    - notebook>=5.4.1,<5.5
-    - pandas>=0.22.0,<0.22.1
+    - notebook>=5.7.2,<5.8
+    - pandas>=0.23.4,<0.23.5
     - scikit-learn>=0.19.1,<0.19.2
-    - seaborn>=0.8.1,<0.9
+    - seaborn>=0.9.0,<0.10.0
     - skll>=1.5,<1.6
-    - statsmodels>=0.8.0,<0.8.1
+    - statsmodels>=0.9.0,<0.9.1
     - sphinx
     - sphinx_rtd_theme
     - zeromq
     - coverage
     - parameterized
-    - setuptools
     - openpyxl
     - xlrd
     - xlwt

doc/config_rsmtool.rst

@@ -13,7 +13,7 @@ An identifier for the experiment that will be used to name the report and all :r
 
 model
 """""
-The machine learner you want to use to build the scoring model. Possible values include :ref:`built-in linear regression models <builtin_models>` as well as all of the learners available via `SKLL <http://skll.readthedocs.io/en/latest/run_experiment.html#learners>`_. With SKLL learners, you can customize the :ref:`tuning objective <skll_objective>` and also :ref:`compute expected scores as predictions <predict_expected_scores>`.
+The machine learner you want to use to build the scoring model. Possible values include :ref:`built-in linear regression models <builtin_models>` as well as all of the learners available via `SKLL <https://skll.readthedocs.io/en/latest/run_experiment.html#learners>`_. With SKLL learners, you can customize the :ref:`tuning objective <skll_objective>` and also :ref:`compute expected scores as predictions <predict_expected_scores>`.
 
 train_file
 """"""""""
@@ -27,7 +27,7 @@ The path to the evaluation data feature file in one of the :ref:`supported forma
 
 skll_objective *(Optional)*
 """""""""""""""""""""""""""
-The tuning objective to use if a SKLL model is chosen to build the scoring model. Possible values are the objectives available via `SKLL <http://skll.readthedocs.io/en/latest/run_experiment.html#objectives>`_. Defaults to ``neg_mean_squared_error`` for SKLL regressors and ``f1_score_micro`` for SKLL classifiers. Note that if this option is specified with the :ref:`built-in linear regression models <builtin_models>`, it will simply be ignored. 
+The tuning objective to use if a SKLL model is chosen to build the scoring model. Possible values are the objectives available via `SKLL <https://skll.readthedocs.io/en/latest/run_experiment.html#objectives>`_. Defaults to ``neg_mean_squared_error`` for SKLL regressors and ``f1_score_micro`` for SKLL classifiers. Note that if this option is specified with the :ref:`built-in linear regression models <builtin_models>`, it will simply be ignored. 
 
 .. _predict_expected_scores:
 
@@ -216,7 +216,7 @@ If set to ``true``, certain evaluations (confusion matrices, score distributions
 .. _use_truncation_thresholds:
 
 use_truncation_thresholds *(Optional)*
-"""""""""""""""""""""""""""""""""""
+""""""""""""""""""""""""""""""""""""""
 If set to ``true``, use the ``min`` and ``max`` columns specified in the ``features`` file to clamp outlier feature values. This is useful if users would like to clamp feature values based on some pre-defined boundaries, rather than having these boundaries calculated based on the training set. Defaults to ``false``.
 
 .. note::

doc/contributing.rst

@@ -10,7 +10,7 @@ To set up a local development environment, follow the steps below:
 
 1. Pull the latest version of RSMTool from GitHub and switch to the ``master`` branch.
 
-2. If you already have the ``conda`` package manager installed, skip to the next step. If you do not, follow the instructions on `this page <http://conda.pydata.org/docs/install/quick.html>`_ to install conda.
+2. If you already have the ``conda`` package manager installed, skip to the next step. If you do not, follow the instructions on `this page <https://conda.io/docs/user-guide/install/index.html>`_ to install conda.
 
 3. Create a new conda environment (say, ``rsmtool``) and install the packages specified in the ``conda_requirements.txt`` file by running::
 

doc/getting_started.rst

@@ -9,7 +9,7 @@ Installing with conda
 
 Currently, the recommended way to install RSMTool is by using the ``conda`` package manager. If you have already installed ``conda``, you can skip straight to Step 2.
 
-1. To install ``conda``, follow the instructions on `this page <http://conda.pydata.org/docs/install/quick.html>`_.
+1. To install ``conda``, follow the instructions on `this page <https://conda.io/docs/install/quick.html>`_.
 
 2. Create a new conda environment (say, ``rsmtool``) and install the RSMTool conda package by running::
 
@@ -20,7 +20,7 @@ Currently, the recommended way to install RSMTool is by using the ``conda`` pack
 4. From now on, you will need to activate this conda environment whenever you want to use RSMTool. This will ensure that the packages required by RSMTool will not affect other projects.
 
 RSMTool can also be downloaded directly from
-`GitHub <http://github.com/EducationalTestingService/rsmtool>`_.
+`GitHub <https://github.com/EducationalTestingService/rsmtool>`_.
 
 Installing with pip
 -------------------

doc/index.rst

@@ -15,7 +15,7 @@ Rater Scoring Modeling Tool (RSMTool)
 
     .. image:: spacer.png
 
-Automated scoring of written and spoken responses is a growing field in educational natural language processing. Automated scoring engines employ machine learning models to predict scores for such responses based on features extracted from the text/audio of these responses. Examples of automated scoring engines include `Project Essay Grade <http://pegwriting.com/about>`_ for written responses and `SpeechRater <https://www.ets.org/research/topics/as_nlp/speech/>`_ for spoken responses.
+Automated scoring of written and spoken responses is a growing field in educational natural language processing. Automated scoring engines employ machine learning models to predict scores for such responses based on features extracted from the text/audio of these responses. Examples of automated scoring engines include `Project Essay Grade <https://pegwriting.com/>`_ for written responses and `SpeechRater <https://www.ets.org/research/topics/as_nlp/speech/>`_ for spoken responses.
 
 RSMTool is a python package which automates and combines in a *single* :doc:`pipeline <pipeline>` multiple analyses that are commonly conducted when building and evaluating automated scoring models. The output of RSMTool is a comprehensive, customizable HTML statistical report that contains the outputs of these multiple analyses. While RSMTool does make it really simple to run this set of standard analyses using a single command, it is also fully customizable and allows users to easily exclude unneeded analyses, modify the standard analyses, and even include custom analyses in the report.
 

doc/pipeline.rst

@@ -12,7 +12,7 @@ As its primary input, RSMTool takes a :ref:`data file <input_file_format>` conta
 
 This trained model can then be used to generate scores for a held-out evaluation data whose feature values are pre-processed using the same :ref:`Pre-processing Parameters <preprocessing_parameters>`. In addition to the raw scores predicted by the model, the **Prediction Analysis** component of the pipline generates several additional :ref:`post-processed scores <score_postprocessing>` that are commonly used in automated scoring.
 
-The primary output of RSMTool is a comprehensive, customizable HTML statistical report that contains the multiple analyses required for a comprehensive evaluation of an automated scoring model including descriptive analyses for all features, model analyses, subgroup comparisons, as well as several different evaluation measures illustrating model efficacy [#]_. More Details about these analyses can be found in a separate `technical paper <https://github.com/EducationalTestingService/rsmtool/raw/master/doc/rsmtool.pdf>`_.
+The primary output of RSMTool is a comprehensive, customizable HTML statistical report that contains the multiple analyses required for a comprehensive evaluation of an automated scoring model including descriptive analyses for all features, model analyses, subgroup comparisons, as well as several different evaluation measures illustrating model efficacy [#]_. More Details about these analyses can be found in a separate `technical paper <https://raw.githubusercontent.com/EducationalTestingService/rsmtool/master/doc/rsmtool.pdf>`_.
 
 In addition to the HTML report, RSMTool also saves the intermediate outputs of all of the performed analyses as :ref:`CSV files <intermediate_files_rsmtool>`.
 
@@ -90,7 +90,7 @@ The raw_trim predictions rounded to the nearest integer.
 
 .. note::
 
-    The rounding is done using ``rint`` function from ``numpy``. See `numpy documentation <http://docs.scipy.org/doc/numpy/reference/generated/numpy.around.html#numpy.around>`_ for treatment of values such as 1.5.
+    The rounding is done using ``rint`` function from ``numpy``. See `numpy documentation <https://docs.scipy.org/doc/numpy/reference/generated/numpy.around.html#numpy.around>`_ for treatment of values such as 1.5.
 
 scale
 ~~~~~

doc/release_process.rst

@@ -3,23 +3,23 @@ RSMTool Release Process
 
 This process is only meant for the project administrators, not users and developers.
 
-1. Run `tests/update_files.py` to make sure that all test data in the new release have correct experiment ids and filenames. If any (non-model) files need to be changed this should be investigated before the branch is released. 
+1. Run ``tests/update_files.py`` to make sure that all test data in the new release have correct experiment ids and filenames. If any (non-model) files need to be changed this should be investigated before the branch is released. 
 
 2. Create a release branch on GitHub.
 
-3. In that release branch, update the version numbers in ``version.py``, update the conda-recipe, and update the README, if necessary.
+3. In that release branch, update the version numbers in ``version.py``, update the conda-recipe, and update the README, if necessary. You should also run `make linkcheck` on the documentation to fix and update any broken/redirected links.
 
-4. Build the new conda package locally on your mac using the following command::
+4. Upload source and wheel packages to PyPI using ``python setup.py sdist upload`` and ``python setup.py bdist_wheel upload``
 
-    conda build -c defaults -c conda-forge --python=3.6 --numpy=1.13 rsmtool
+5. Build the new conda package locally on your mac using the following command (*Note*: you may have to replace the contents of the ``requirements()`` function in ``setup.py`` with a ``pass`` statement to get ``conda build`` to work)::
 
-5. Convert the package for both linux and windows::
+    conda build -c defaults -c conda-forge --python=3.6 --numpy=1.14 rsmtool
 
-    conda convert -p win-64 -p linux-64 <mac package tarball>
+6. Convert the package for both linux and windows::
 
-6. Upload all packages to anaconda.org using ``anaconda upload``.
+    conda convert -p win-64 -p linux-64 <mac package tarball>
 
-7. Upload source package to PyPI using ``python setup.py sdist upload``.
+7. Upload each of the packages to anaconda.org using ``anaconda upload <package tarball>``.
 
 8. Create pull requests on the `rsmtool-conda-tester <https://github.com/EducationalTestingService/rsmtool-conda-tester/>`_ and `rsmtool-pip-tester <https://github.com/EducationalTestingService/rsmtool-pip-tester/>`_ repositories to test the conda and PyPI packages on Linux and Windows.
 

doc/usage_rsmeval.rst

@@ -5,7 +5,7 @@
 
 RSMTool provides the ``rsmeval`` command-line utility to evaluate existing predictions and generate a report with all the built-in analyses. This can be useful in scenarios where the user wants to use more sophisticated machine learning algorithms not available in RSMTool to build the scoring model but still wants to be able to evaluate that model's predictions using the standard analyses.
 
-For example, say a researcher *has* an existing automated scoring engine for grading short responses that extracts the features and computes the predicted score. This engine uses a large number of binary, sparse features. She cannot use ``rsmtool`` to train her model since it requires numeric features. So, she uses `scikit-learn <http://scikit-learn.org/>`_ to train her model.
+For example, say a researcher *has* an existing automated scoring engine for grading short responses that extracts the features and computes the predicted score. This engine uses a large number of binary, sparse features. She cannot use ``rsmtool`` to train her model since it requires numeric features. So, she uses `scikit-learn <https://scikit-learn.org/>`_ to train her model.
 
 Once the model is trained, the researcher wants to evaluate her engine's performance using the analyses recommended by the educational measurement community as well as conduct additional investigations for specific subgroups of test-takers. However, these kinds of analyses are not available in ``scikit-learn``. She can use ``rsmeval`` to set up a customized report using a combination of existing and custom sections and quickly produce the evaluation that is useful to her.
 

doc/who.rst

@@ -5,7 +5,7 @@ Who is RSMTool for?
 
 We expect the primary users of RSMTool to be researchers working on developing new automated scoring engines or on improving existing ones. Here's the most common scenario.
 
-A researcher already *has* a set of responses such as essays or recorded spoken responses which have already been assigned numeric scores by human graders. He has also processed these responses and extracted a set of (numeric) features using systems such as `Coh-Metrix <http://cohmetrix.com/>`_, `TextEvaluator <https://texteval-pilot.ets.org/TextEvaluator/>`_, `OpenSmile <http://audeering.com/research/opensmile/>`_, or using his own custom text/speech processing pipeline. He wishes to understand how well the set of chosen features can predict the human score.
+A researcher already *has* a set of responses such as essays or recorded spoken responses which have already been assigned numeric scores by human graders. He has also processed these responses and extracted a set of (numeric) features using systems such as `Coh-Metrix <http://cohmetrix.com/>`_, `TextEvaluator <https://textevaluator.ets.org/TextEvaluator/>`_, `OpenSmile <https://audeering.com/research/opensmile/>`_, or using his own custom text/speech processing pipeline. He wishes to understand how well the set of chosen features can predict the human score.
 
 He can then run an RSMTool "experiment" to build a regression-based scoring model (using one of many available regressors) and produce a report. The report includes descriptive statistics for all his features, diagnostic information about the trained regression model, and a comprehensive evaluation of model performance on a held-out set of responses.
 

requirements.txt

@@ -4,7 +4,7 @@ joblib==0.11
 matplotlib==2.1.2
 nose==1.3.7
 notebook==5.7.2
-numpy==1.14.0
+numpy==1.14.*
 pandas==0.23.4
 scikit-learn==0.19.1
 scipy==1.1.0

rsmtool/modeler.py

@@ -372,7 +372,7 @@ def train_rebalanced_lr(self, df_train, feature_columns):
         """
         Train `RebalancedLR` (formerly empWtBalanced) -
         Balanced empirical weights by changing betas
-        [adapted from http://bit.ly/UTP7gS]
+        [adapted from: https://stats.stackexchange.com/q/30876]
 
         Parameters
         ----------
@@ -959,9 +959,8 @@ def train_builtin_model(self,
         Returns
         -------
         learner : `Learner` object
-            SKLL `LinearRegression` `Learner <http://skll.readthedocs.io/en/
-            latest/api/skll.html#skll.Learner>`_ object containing the
-            coefficients learned by training the built-in model.
+            SKLL `LinearRegression` `Learner <https://skll.readthedocs.io/en/latest/api/learner.html#skll.learner.Learner`_ object containing
+            the coefficients learned by training the built-in model.
         """
         # get the columns that actually contain the feature values
         feature_columns = [c for c in df_train.columns if c not in ['spkitemid', 'sc1']]

rsmtool/reporter.py

@@ -192,9 +192,7 @@ def merge_notebooks(notebook_files, output_file):
 
         Note
         ----
-        Adapted from: http://stackoverflow.com/questions/
-        20454668/how-to-merge-two-ipython-notebooks-correctly
-        -without-getting-json-error.
+        Adapted from: https://stackoverflow.com/q/20454668.
         """
 
         # Merging ipython notebooks basically means that we keep the

rsmtool/utils.py

@@ -314,7 +314,7 @@ def parse_json_with_comments(filename):
     Note
     ----
     This code was adapted from:
-    http://www.lifl.fr/~riquetd/parse-a-json-file-with-comments.html.
+    https://web.archive.org/web/20150520154859/http://www.lifl.fr/~riquetd/parse-a-json-file-with-comments.html
     """
 
     # Regular expression to identify comments

rsmtool/version.py

@@ -3,5 +3,5 @@
 in one place. Based on the suggestion `here. <http://bit.ly/16LbuJF>`_
 """
 
-__version__ = '6.0.1'
+__version__ = '6.1.0'
 VERSION = tuple(int(x) for x in __version__.split('.'))

tests/data/experiments/lr-with-continuous-human-scores-in-test/output/lr_with_continuous_human_scores_in_test_rsmtool.json

@@ -16,6 +16,7 @@
     "select_transformations": false,
     "standardize_features": true,
     "use_thumbnails": false,
+    "use_truncation_thresholds": false,
     "predict_expected_scores": false,
     "sign": null,
     "length_column": null,

tests/data/experiments/lr-with-continuous-human-scores/output/lr_with_continuous_human_scores_rsmtool.json

@@ -16,6 +16,7 @@
     "select_transformations": false,
     "standardize_features": true,
     "use_thumbnails": false,
+    "use_truncation_thresholds": false,
     "predict_expected_scores": false,
     "sign": null,
     "length_column": null,

tests/data/experiments/lr-with-length-non-numeric/output/lr_with_length_non_numeric_rsmtool.json

@@ -16,6 +16,7 @@
     "select_transformations": false,
     "standardize_features": true,
     "use_thumbnails": false,
+    "use_truncation_thresholds": false,
     "predict_expected_scores": false,
     "sign": null,
     "second_human_score_column": null,