Merge pull request #325 from EducationalTestingService/master

Merge `master` into `stable`.

.gitignore

@@ -13,3 +13,5 @@
 *test_outputs/
 
 __pycache__
+/rsmtool.sublime-workspace
+/rsmtool.sublime-project

.pep8speaks.yml

@@ -0,0 +1,22 @@
+# File : .pep8speaks.yml
+
+scanner:
+    diff_only: True  # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned.
+    linter: flake8  # Other option is pycodestyle
+
+flake8:  # Valid if scanner.linter is flake8
+    max-line-length: 100
+
+no_blank_comment: False  # If True, no comment is made on PR without any errors.
+descending_issues_order: False  # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file
+
+message:  # Customize the comment made by the bot
+    opened:  # Messages when a new PR is submitted
+        header: "Hello @{name}! Thanks for opening this PR. "
+                # The keyword {name} is converted into the author's username
+        footer: "Do see the [Hitchhiker's guide to code style](https://goo.gl/hqbW4r)"
+                # The messages can be written as they would over GitHub
+    updated:  # Messages when new commits are added to the PR
+        header: "Hello @{name}! Thanks for updating this PR. "
+        footer: ""  # Why to comment the link to the style guide everytime? :)
+    no_errors: "There are currently no PEP 8 issues detected in this Pull Request. Cheers! :tada: "

.travis.yml

@@ -12,7 +12,7 @@ env:
   matrix:
   - TESTFILES="tests/test_experiment_rsmtool_1.py"
   - TESTFILES="tests/test_comparer.py tests/test_configuration_parser.py tests/test_experiment_rsmtool_2.py"
-  - TESTFILES="tests/test_analyzer.py tests/test_experiment_rsmeval.py"
+  - TESTFILES="tests/test_analyzer.py tests/test_experiment_rsmeval.py tests/test_fairness_utils.py tests/test_prmse_utils.py tests/test_container.py"
   - TESTFILES="tests/test_experiment_rsmcompare.py tests/test_experiment_rsmsummarize.py tests/test_modeler.py tests/test_preprocessor.py tests/test_writer.py tests/test_experiment_rsmtool_3.py"
   - TESTFILES="tests/test_experiment_rsmpredict.py tests/test_reader.py tests/test_reporter.py tests/test_transformer.py tests/test_utils.py tests/test_experiment_rsmtool_4.py"
 sudo: false
@@ -21,10 +21,10 @@ before_install:
 - if [ -x ${HOME}/miniconda3/pkgs ]; then pushd ${HOME}/miniconda3/pkgs && find -maxdepth 1 -mindepth 1 -type d | xargs rm -rf && popd; fi
 - chmod +x miniconda.sh
 - ./miniconda.sh -b -f
-- ${HOME}/miniconda3/bin/conda update --yes conda
+- ${HOME}/miniconda3/bin/conda install -c anaconda --yes setuptools
 
 install:
-- ${HOME}/miniconda3/bin/conda create --name rsmenv -c defaults -c conda-forge -c desilinguist --file conda_requirements.txt --yes --quiet
+- ${HOME}/miniconda3/bin/conda create --name rsmenv -c defaults -c conda-forge -c desilinguist --file requirements.txt --yes --quiet
 - ${HOME}/miniconda3/envs/rsmenv/bin/pip install nose-cov python-coveralls
 - ${HOME}/miniconda3/envs/rsmenv/bin/pip install -e .
 

DistributeTests.ps1

@@ -0,0 +1,67 @@
+<#  
+.SYNOPSIS  
+    Distribute the tests in VSTS pipeline across multiple agents 
+.DESCRIPTION  
+    This script divides test files across multiple agents for running on Azure DevOps.
+    It is adapted from the script in this repository: 
+    https://github.com/PBoraMSFT/ParallelTestingSample-Python/blob/master/DistributeTests.ps1
+
+    The distribution is basically identical to the way we do it in .travis.yaml
+#>
+
+$tests = Get-ChildItem .\tests\ -Filter "test*.py" # search for test files with specific pattern.
+$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # standard VSTS variables available using parallel execution; total number of parallel jobs running
+$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE  # current job position
+$testCount = $tests.Count
+
+# below conditions are used if parallel pipeline is not used. i.e. pipeline is running with single agent (no parallel configuration)
+if ($totalAgents -eq 0) {
+    $totalAgents = 1
+}
+if (!$agentNumber -or $agentNumber -eq 0) {
+    $agentNumber = 1
+}
+
+Write-Host "Total agents: $totalAgents"
+Write-Host "Agent number: $agentNumber"
+Write-Host "Total tests: $testCount"
+
+$testsToRun= @()
+
+if ($agentNumber -eq 1) {
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_1.py"
+}
+elseif ($agentNumber -eq 2) {
+    $testsToRun = $testsToRun + "tests/test_comparer.py"
+    $testsToRun = $testsToRun + "tests/test_configuration_parser.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_2.py"
+    $testsToRun = $testsToRun + "tests/test_container.py"
+}
+elseif ($agentNumber -eq 3) {
+    $testsToRun = $testsToRun + "tests/test_analyzer.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmeval.py"
+    $testsToRun = $testsToRun + "tests/test_fairness_utils.py"
+    $testsToRun = $testsToRun + "tests/test_prmse_utils.py"
+}
+elseif ($agentNumber -eq 4) {
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmcompare.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmsummarize.py"
+    $testsToRun = $testsToRun + "tests/test_modeler.py"
+    $testsToRun = $testsToRun + "tests/test_preprocessor.py"
+    $testsToRun = $testsToRun + "tests/test_writer.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_3.py"
+}
+elseif ($agentNumber -eq 5) {
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmpredict.py"
+    $testsToRun = $testsToRun + "tests/test_reader.py"
+    $testsToRun = $testsToRun + "tests/test_reporter.py"
+    $testsToRun = $testsToRun + "tests/test_transformer.py"
+    $testsToRun = $testsToRun + "tests/test_utils.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_4.py"
+}
+
+# join all test files seperated by space. pytest runs multiple test files in following format pytest test1.py test2.py test3.py
+$testFiles = $testsToRun -Join " "
+Write-Host "Test files $testFiles"
+# write these files into variable so that we can run them using pytest in subsequent task. 
+Write-Host "##vso[task.setvariable variable=pytestfiles;]$testFiles" 

MANIFEST.in

@@ -1,8 +1,11 @@
 include LICENSE
 include *.rst
 include requirements.txt
+recursive-exclude doc *
+recursive-exclude tests *
+recursive-exclude examples *
 recursive-include rsmtool/notebooks *
-recursive-include doc *
 prune test_outputs
+prune rsmtool/rsmtool.egg-info
 prune rsmtool/notebooks/.ipynb_checkpoints
 prune rsmtool/notebooks/comparison/.ipynb_checkpoints

README.rst

@@ -46,7 +46,7 @@ For installation and usage, please see the `official documentation <http://rsmto
 Requirements
 ------------
 
-- Python 3.6
+- Python >=3.6, <3.7
 - ``numpy``
 - ``scipy``
 - ``scikit-learn``

azure-pipelines.yml

@@ -0,0 +1,52 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+variables:
+  MPLBACKEND: Agg
+
+trigger:
+  branches:
+    include:
+    - '*'  # must quote since "*" is a YAML reserved character; we want a string
+
+jobs:
+
+- job: 'RSMToolTests'
+  pool:
+    vmImage: 'windows-latest'
+  strategy:
+    parallel: 5
+
+  steps:
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+
+  - script: |
+      conda update --quiet --yes conda
+      conda info -a
+    displayName: "Update conda"
+
+  - script: |
+      conda create --name rsmdev --yes --quiet -c conda-forge -c defaults -c desilinguist python=%PYTHON_VERSION% --file requirements.txt
+      conda init cmd.exe
+      CALL activate rsmdev
+      pip install -e .
+    displayName: 'Install dependencies & code'
+
+  - powershell: ./DistributeTests.ps1 
+    displayName: 'PowerShell Script to distribute tests'
+
+  - script: |
+      CALL activate rsmdev
+      echo $(pytestfiles)
+      nosetests --with-xunit $(pytestfiles)
+    displayName: 'Run tests'
+
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    inputs:
+      testResultsFiles: 'nosetests.xml'
+      testRunTitle: 'RSMTool tests'
+    condition: succeededOrFailed()

conda-recipe/rsmtool/meta.yaml

@@ -1,15 +1,18 @@
+{% set data = load_setup_py_data() %}
+
 package:
   name: rsmtool
-  version: 6.1.0
+  version: {{data.get('version')}}
 
 source:
   path: ../../../rsmtool
 
 build:
   number: 0
+  noarch: python
   script:
     - cd $SRC_DIR
-    - $PYTHON setup.py install
+    - "{{ PYTHON }} -m pip install . --no-deps -vv"
   entry_points:
     - rsmtool = rsmtool.rsmtool:main
     - rsmcompare = rsmtool.rsmcompare:main
@@ -19,55 +22,19 @@ build:
     - render_notebook = rsmtool.reporter:main
     - convert_feature_json = rsmtool.convert_feature_json:main
 
+{% block requirements -%}
 requirements:
   build:
-    - python
-    - numpy>=1.14.0,<1.15
-    - scipy>=1.1.0,<1.2.0
-    - ipython>=6.5.0,<6.5.1
-    - jupyter>=1.0.0,<1.1
-    - joblib>=0.11,<0.12
-    - matplotlib>=2.1.2,<2.2
-    - nose>=1.3.7,<1.4
-    - notebook>=5.7.2,<5.8
-    - pandas>=0.23.4,<0.23.5
-    - scikit-learn>=0.19.1,<0.19.2
-    - seaborn>=0.9.0,<0.10.0
-    - skll>=1.5,<1.6
-    - statsmodels>=0.9.0,<0.9.1
-    - sphinx
-    - sphinx_rtd_theme
-    - zeromq
-    - coverage
-    - parameterized
-    - setuptools
-    - openpyxl
-    - xlrd
-    - xlwt
-
+    - python >=3.6
+    {% for req in data.get('install_requires', []) -%}
+    - {{req.replace(">", " >").replace("<", " <").replace("==", " ")}}
+    {% endfor %}
   run:
-    - python
-    - numpy>=1.14.0,<1.15
-    - scipy>=1.1.0,<1.2.0
-    - ipython>=6.5.0,<6.5.1
-    - jupyter>=1.0.0,<1.1
-    - joblib>=0.11,<0.12
-    - matplotlib>=2.1.2,<2.2
-    - nose>=1.3.7,<1.4
-    - notebook>=5.7.2,<5.8
-    - pandas>=0.23.4,<0.23.5
-    - scikit-learn>=0.19.1,<0.19.2
-    - seaborn>=0.9.0,<0.10.0
-    - skll>=1.5,<1.6
-    - statsmodels>=0.9.0,<0.9.1
-    - sphinx
-    - sphinx_rtd_theme
-    - zeromq
-    - coverage
-    - parameterized
-    - openpyxl
-    - xlrd
-    - xlwt
+    - python >=3.6
+    {% for req in data.get('install_requires', []) -%}
+    - {{req.replace(">", " >").replace("<", " <").replace("==", " ")}}
+    {% endfor %}
+{%- endblock %}
 
 test:
   # Python imports
@@ -99,3 +66,4 @@ test:
 about:
   home: https://github.com/EducationalTestingService/rsmtool
   license: Apache 2.0
+  license_file: LICENSE

doc/api.rst

@@ -73,6 +73,12 @@ From :py:mod:`~rsmtool.convert_feature_json` Module
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autofunction:: rsmtool.convert_feature_json_file
 
+
+From :py:mod:`~rsmtool.fairness_utils` Module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. _fairness_api:
+.. autofunction:: rsmtool.fairness_utils.get_fairness_analyses
+
 From :py:mod:`~rsmtool.modeler` Module
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. automodule:: rsmtool.modeler
@@ -87,6 +93,12 @@ From :py:mod:`~rsmtool.preprocessor` Module
     :undoc-members:
     :show-inheritance:
 
+
+From :py:mod:`~rsmtool.prmse_utils` Module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. _prmse_api:
+.. autofunction:: rsmtool.prmse_utils.compute_prmse
+
 From :py:mod:`~rsmtool.reader` Module
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. automodule:: rsmtool.reader
@@ -110,12 +122,20 @@ From :py:mod:`~rsmtool.transformer` Module
 
 From :py:mod:`~rsmtool.utils` Module
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _agreement_api:
 .. autofunction:: rsmtool.utils.agreement
 .. autofunction:: rsmtool.utils.partial_correlations
 .. autofunction:: rsmtool.utils.get_thumbnail_as_html
 .. autofunction:: rsmtool.utils.show_thumbnail
 .. autofunction:: rsmtool.utils.compute_expected_scores_from_model
 .. autofunction:: rsmtool.utils.parse_json_with_comments
+.. _qwk_api:
+.. autofunction:: rsmtool.utils.quadratic_weighted_kappa
+.. _smd_api:
+.. autofunction:: rsmtool.utils.standardized_mean_difference
+.. _dsm_api:
+.. autofunction:: rsmtool.utils.difference_of_standardized_means
 
 From :py:mod:`~rsmtool.writer` Module
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~