Merge dd164ec into df6164a

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://img.shields.io/travis/fitbenchmarking/fitbenchmarking.svg?style=flat-square)](https://travis-ci.org/fitbenchmarking/fitbenchmarking)
+[![Build Status](https://img.shields.io/azure-devops/tests/fitbenchmarking/fitbenchmarking/1.svg?style=flat-square&compactmessage)](https://dev.azure.com/fitbenchmarking/fitbenchmarking/_build/latest?definitionId=1&branchName=master)
 [![Documentation Status](https://img.shields.io/readthedocs/fitbenchmarking?style=flat-square)](https://fitbenchmarking.readthedocs.io/en/latest)
 [![Coverage Status](https://img.shields.io/coveralls/github/fitbenchmarking/fitbenchmarking.svg?style=flat-square)](https://coveralls.io/github/fitbenchmarking/fitbenchmarking)
 ![Windows Supported](https://img.shields.io/badge/win10-support-blue.svg?style=flat-square&logo=windows)

azure-pipelines.yml

@@ -1,28 +1,78 @@
-# CI for Windows
-# Here not repeating LINTING and similar done with Travis (only this may make sense later)
 # For info syntax used in this file see: https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
+schedules:
+- cron: "0 9 * * Mon"
+  displayName: Weekly tests
+  branches:
+    include:
+    - master
+    - releases-*
+  always: true
+
 trigger:
 - master
+- release-*
+
+pr:
+- master
+- release-*
+
+jobs:
+- template: pipelines/docker_template.yml
+  parameters:
+     name: Ubuntu_Default_Unit_Tests
+     vmImage: ubuntu-latest
+     test_type: unit
+     full_install: False
+
+- template: pipelines/docker_template.yml
+  parameters:
+     name: Ubuntu_Full_Unit_Tests
+     vmImage: ubuntu-latest
+     test_type: unit
+     full_install: True
+     coverage: True
+
+- template: pipelines/docker_template.yml
+  parameters:
+     name: Ubuntu_Default_System_Tests
+     vmImage: ubuntu-latest
+     test_type: system
+     full_install: False
+
+- template: pipelines/docker_template.yml
+  parameters:
+     name: Ubuntu_Full_System_Tests
+     vmImage: ubuntu-latest
+     test_type: system
+     full_install: True
+
+- template: pipelines/docker_template.yml
+  parameters:
+     name: Ubuntu_Linting_Tests
+     vmImage: ubuntu-latest
+     test_type: linting
+     full_install: True
+
+- job: 'Windows_Default_Unit_Tests'
+  pool:
+    vmImage: 'windows-latest'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+    displayName: 'Use Python $(python.version)'
+
+  - script: |
+      python -m pip install -r requirements.txt
+      python -m pip install .[bumps,DFO,minuit,SAS]
+    displayName: 'Install FitBenchmarking and requirements'
 
-pool:
-  vmImage: 'windows-latest'
-strategy:
-  matrix:
-    Python36:
-      python.version: '3.6'
-
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-  displayName: 'Use Python $(python.version)'
-
-- script: |
-    pip install -r requirements.txt
-    pip install .[bumps,DFO,minuit,SAS]
-  displayName: 'Install FitBenchmarking and requirements'
-
-- script: |
-    bash travis/unit_tests_default.sh
-  displayName: 'unit testing'
+  - script: |
+      bash pipelines/unit_tests_default.sh
+    displayName: 'unit testing'

docs/source/contributors/structure.rst

@@ -11,7 +11,7 @@ At the root of the repository there are six directories:
  - docs
  - examples
  - fitbenchmarking
- - travis
+ - pipelines
 
 
 #################
@@ -25,7 +25,7 @@ through setuptools.
 Docker (``Docker``)
 ###################
 
-The continuous integration process on travis currently run on a Docker container,
+The continuous integration process on Azure Pipelines currently run on a Docker container,
 and this directory holds the Dockerfiles.  The Docker containers are hosted on
 Dockerhub.
 
@@ -183,11 +183,11 @@ class (see :ref:`options_extend` to extend)
 and :class:`~fitbenchmarking.utils.fitbm_result.FittingResult` class,
 as well as functions for logging and directory creation.
 
-###################
-Travis (``travis``)
-###################
+#########################
+Pipelines (``pipelines``)
+#########################
 
-We use `Travis <https://travis-ci.org/github/fitbenchmarking/fitbenchmarking>`_
+We use `Azure Pipelines <https://azure.microsoft.com/en-gb/services/devops/pipelines/>`__
 to run our Continuous Integration tests.
 The specific tests run are defined in a series of Bash scripts,
 which are stored in this folder.

docs/source/users/tests.rst

@@ -4,36 +4,51 @@
 FitBenchmarking Tests
 #####################
 
-The tests for FitBenchmarking require ``pytest>=3.6``. We have split the tests into two categories:
+The tests for FitBenchmarking require ``pytest>=3.6``. We have split the tests
+into two categories:
 
-* ``default``: denotes tests involving ``pip`` installable :ref:`software packages<getting-started>`,
-* ``all``: in addition to ``default``, also runs tests on :ref:`external packages <external-instructions>`.
+* ``default``: denotes tests involving ``pip`` installable
+  :ref:`software packages<getting-started>`,
+* ``all``: in addition to ``default``, also runs tests on
+  :ref:`external packages <external-instructions>`.
 
 Unit tests
 ----------
 
-Each module directory in FitBenchmarking (e.g. ``controllers``) contains a test folder which has the ``unit`` tests for that module. One can run the tests for a module by:
+Each module directory in FitBenchmarking (e.g. ``controllers``) contains a
+test folder which has the ``unit`` tests for that module.
+One can run the tests for a module by:
 
 .. code-block:: bash
 
    pytest fitbenchmarking/<MODULE_DIR> --test-type <TEST_TYPE>
 
-where <TEST_TYPE> is either ``default`` or ``all``. If ``--test-type`` argument is not given the default is ``all``
+where <TEST_TYPE> is either ``default`` or ``all``.
+If ``--test-type`` argument is not given the default is ``all``
 
 System tests
 ------------
 
-System tests can be found in the ``systests`` directory in FitBenchmarking. As with the unit tests, these can be run via:
+System tests can be found in the ``systests`` directory in FitBenchmarking.
+As with the unit tests, these can be run via:
 
 .. code-block:: bash
 
    pytest fitbenchmarking/systests --test-type <TEST_TYPE>
 
 
 .. warning::
-   The files in the expected results subdirectory of the ``systests`` directory are generated to check consistency in our automated tests via `Travis CI <https://travis-ci.org/>`__.  They might not pass on your local operating system due to, for example, different software package versions being installed.
-
-Travis CI tests
----------------
-
-The scripts that are used our automated tests via `Travis CI <https://travis-ci.org/>`__ are located in the ``travis`` folder. These give an example of how to run both the unit and system tests within FitBenchmarking.
+   The files in the expected results subdirectory of the ``systests``
+   directory are generated to check consistency in our automated tests via
+   `Azure Pipelines <https://azure.microsoft.com/en-gb/services/devops/pipelines/>`__.
+   They might not pass on your local operating system due to, for example,
+   different software package versions being installed.
+
+Azure Pipelines tests
+---------------------
+
+The scripts that are used our automated tests via
+`Azure Pipelines <https://azure.microsoft.com/en-gb/services/devops/pipelines/>`__
+are located in the ``pipelines`` folder.
+These give an example of how to run both the unit and system tests within
+FitBenchmarking.

pipelines/docker_template.yml

@@ -0,0 +1,66 @@
+# Template for using docker
+
+# From https://docs.microsoft.com/en-gb/azure/devops/pipelines/process/templates?view=azure-devops#job-stage-and-step-templates-with-parameters
+
+parameters:
+- name: name
+  default: Ubuntu_Bionic
+- name: vmImage
+  default: ubuntu-18.04
+- name: test_type
+  default: unit
+- name: full_install
+  type: boolean
+  default: False
+- name: coverage
+  type: boolean
+  default: False
+
+jobs:
+- job: ${{ parameters.name }}
+
+  variables:
+  - group: coverage
+
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+
+  ${{ if eq(parameters.full_install, true) }}:
+    container:
+      image: fitbenchmarking/fitbenchmarking-extras:latest
+  ${{ if eq(parameters.full_install, false) }}:
+    container: 
+      image: fitbenchmarking/fitbenchmarking-deps:latest
+
+  steps:
+  - script: |
+      env
+      sudo -E --preserve-env=PATH /opt/venv/bin/python -m pip install -r requirements.txt
+      sudo -E --preserve-env=PATH /opt/venv/bin/python -m pip install .[bumps,DFO,minuit,SAS]
+    displayName: 'Install FitBenchmarking and requirements'
+
+  - ${{ if eq(parameters.full_install, true) }}:
+    - script: |
+        sudo -E mkdir -p $MASTSIF
+        sudo -E chown $(whoami) $MASTSIF
+        sudo -E mkdir -p $PYCUTEST_CACHE
+        sudo -E chown $(whoami) $PYCUTEST_CACHE
+        pipelines/${{ parameters.test_type }}_tests.sh
+      displayName: 'Running ${{ parameters.test_type }} tests - Full Install'
+  - ${{ if eq(parameters.full_install, false) }}:
+    - script: |
+        pipelines/${{ parameters.test_type }}_tests_default.sh
+      displayName: 'Running ${{ parameters.test_type }} tests - Default Install'
+  
+  - ${{ if eq(parameters.coverage, true) }}:
+    - script: |
+        # Hack while coveralls doesn't explicitly support Azure
+        export TRAVIS_JOB_ID=$BUILD_BUILDNUMBER
+        export TRAVIS_BRANCH=$BUILD_SOURCEBRANCH
+        mv .coverage .coverage.docker
+        coverage combine --append
+        coveralls
+      displayName: 'Update Coverage'
+      env:
+        COVERALLS_REPO_TOKEN: $(token)
+        COVERALLS_SERVICE_NAME: azure-pipelines

requirements.txt

@@ -4,3 +4,5 @@ coveralls
 python-coveralls
 urllib3==1.23
 coverage<5.0
+pylint
+flake8