Merge pull request #309 from EducationalTestingService/add-azure-pipe…

…lines

Integrate testing on Windows using Azure Pipelines

DistributeTests.ps1

@@ -0,0 +1,66 @@
+<#  
+.SYNOPSIS  
+    Distribute the tests in VSTS pipeline across multiple agents 
+.DESCRIPTION  
+    This script divides test files across multiple agents for running on Azure DevOps.
+    It is adapted from the script in this repository: 
+    https://github.com/PBoraMSFT/ParallelTestingSample-Python/blob/master/DistributeTests.ps1
+
+    The distribution is basically identical to the way we do it in .travis.yaml
+#>
+
+$tests = Get-ChildItem .\tests\ -Filter "test*.py" # search for test files with specific pattern.
+$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # standard VSTS variables available using parallel execution; total number of parallel jobs running
+$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE  # current job position
+$testCount = $tests.Count
+
+# below conditions are used if parallel pipeline is not used. i.e. pipeline is running with single agent (no parallel configuration)
+if ($totalAgents -eq 0) {
+    $totalAgents = 1
+}
+if (!$agentNumber -or $agentNumber -eq 0) {
+    $agentNumber = 1
+}
+
+Write-Host "Total agents: $totalAgents"
+Write-Host "Agent number: $agentNumber"
+Write-Host "Total tests: $testCount"
+
+$testsToRun= @()
+
+if ($agentNumber -eq 1) {
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_1.py"
+}
+elseif ($agentNumber -eq 2) {
+    $testsToRun = $testsToRun + "tests/test_comparer.py"
+    $testsToRun = $testsToRun + "tests/test_configuration_parser.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_2.py"
+}
+elseif ($agentNumber -eq 3) {
+    $testsToRun = $testsToRun + "tests/test_analyzer.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmeval.py"
+    $testsToRun = $testsToRun + "tests/test_fairness_utils.py"
+    $testsToRun = $testsToRun + "tests/test_prmse_utils.py"
+}
+elseif ($agentNumber -eq 4) {
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmcompare.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmsummarize.py"
+    $testsToRun = $testsToRun + "tests/test_modeler.py"
+    $testsToRun = $testsToRun + "tests/test_preprocessor.py"
+    $testsToRun = $testsToRun + "tests/test_writer.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_3.py"
+}
+elseif ($agentNumber -eq 5) {
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmpredict.py"
+    $testsToRun = $testsToRun + "tests/test_reader.py"
+    $testsToRun = $testsToRun + "tests/test_reporter.py"
+    $testsToRun = $testsToRun + "tests/test_transformer.py"
+    $testsToRun = $testsToRun + "tests/test_utils.py"
+    $testsToRun = $testsToRun + "tests/test_experiment_rsmtool_4.py"
+}
+
+# join all test files seperated by space. pytest runs multiple test files in following format pytest test1.py test2.py test3.py
+$testFiles = $testsToRun -Join " "
+Write-Host "Test files $testFiles"
+# write these files into variable so that we can run them using pytest in subsequent task. 
+Write-Host "##vso[task.setvariable variable=pytestfiles;]$testFiles" 

azure-pipelines.yml

@@ -0,0 +1,52 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+variables:
+  MPLBACKEND: Agg
+
+trigger:
+  branches:
+    include:
+    - '*'  # must quote since "*" is a YAML reserved character; we want a string
+
+jobs:
+
+- job: 'RSMToolTests'
+  pool:
+    vmImage: 'windows-latest'
+  strategy:
+    parallel: 5
+
+  steps:
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+
+  - script: |
+      conda update --quiet --yes conda
+      conda info -a
+    displayName: "Update conda"
+
+  - script: |
+      conda create --name rsmdev --yes --quiet -c conda-forge -c defaults -c desilinguist python=%PYTHON_VERSION% --file requirements.txt
+      conda init cmd.exe
+      CALL activate rsmdev
+      pip install -e .
+    displayName: 'Install dependencies & code'
+
+  - powershell: ./DistributeTests.ps1 
+    displayName: 'PowerShell Script to distribute tests'
+
+  - script: |
+      CALL activate rsmdev
+      echo $(pytestfiles)
+      nosetests --with-xunit $(pytestfiles)
+    displayName: 'Run tests'
+
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    inputs:
+      testResultsFiles: 'nosetests.xml'
+      testRunTitle: 'RSMTool tests'
+    condition: succeededOrFailed()

tests/test_reader.py

@@ -155,7 +155,7 @@ def test_container_train_property(self):
                       [('train', 'csv'), ('test', 'xls'), ('train_metadata', 'tsv')],
                       [('train', 'jsonlines'), ('test', 'jsonlines')]]
 
-        converter = {'id': str, 'feature1': int, 'feature2': int, 'candidate': str}
+        converter = {'id': str, 'feature1': np.int64, 'feature2': np.int64, 'candidate': str}
         converters = [{'train': converter, 'test': converter},
                       {'train': converter},
                       {'train': converter, 'test': converter},
@@ -187,7 +187,7 @@ def test_container_test_property_frame_equal(self):
         test_lists = [('feature_specs', 'csv'),
                       ('test', 'tsv'),
                       ('train_metadata', 'xlsx')]
-        converter = {'id': str, 'feature1': int, 'feature2': int, 'candidate': str}
+        converter = {'id': str, 'feature1': np.int64, 'feature2': np.int64, 'candidate': str}
         converters = {'test': converter}
         container = self.get_container(test_lists, converters)
         frame = container.test
@@ -211,7 +211,7 @@ def test_get_path_default(self):
 
     def test_getitem_test_from_key(self):
         test_lists = [('feature_specs', 'csv'), ('test', 'tsv'), ('train', 'xlsx')]
-        converter = {'id': str, 'feature1': int, 'feature2': int, 'candidate': str}
+        converter = {'id': str, 'feature1': np.int64, 'feature2': np.int64, 'candidate': str}
         converters = {'train': converter, 'test': converter}
         container = self.get_container(test_lists, converters)
         frame = container['test']