From 64abc9dc115af2695757badcf021dc13f6f64cb8 Mon Sep 17 00:00:00 2001 From: Viktor Gal Date: Mon, 24 Sep 2018 09:10:49 -0400 Subject: [PATCH] add initial azure pipelines definition test macos build --- .ci/archive-deploy.yml | 22 +++++++ .ci/ci.yml | 103 ++++++++++++++++++++++++++++++++ .ci/docker-steps.yml | 123 +++++++++++++++++++++++++++++++++++++++ .ci/download-extract.yml | 27 +++++++++ .ci/macos-steps.yml | 50 ++++++++++++++++ .ci/setup_clcache.cmd | 16 +++++ .ci/windows-steps.yml | 49 ++++++++++++++++ .travis.yml | 79 ------------------------- README.md | 3 +- appveyor.yml | 37 ------------ 10 files changed, 391 insertions(+), 118 deletions(-) create mode 100644 .ci/archive-deploy.yml create mode 100644 .ci/ci.yml create mode 100644 .ci/docker-steps.yml create mode 100644 .ci/download-extract.yml create mode 100644 .ci/macos-steps.yml create mode 100644 .ci/setup_clcache.cmd create mode 100644 .ci/windows-steps.yml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml diff --git a/.ci/archive-deploy.yml b/.ci/archive-deploy.yml new file mode 100644 index 00000000000..b165e9edeb5 --- /dev/null +++ b/.ci/archive-deploy.yml @@ -0,0 +1,22 @@ +parameters: + artifactName: '' + archive: '' + name: '' + dir: '' + +steps: +- task: ArchiveFiles@2 + displayName: Archive ${{ parameters.name }} + inputs: + rootFolderOrFile: ${{ parameters.dir }} + includeRootFolder: false + archiveType: tar + tarCompression: xz + archiveFile: ${{ parameters.archive }} + +- task: PublishBuildArtifacts@1 + displayName: Publish ${{ parameters.name }} archive + inputs: + pathtoPublish: ${{ parameters.archive }} + artifactName: ${{ parameters.artifactName }} + publishLocation: 'Container' diff --git a/.ci/ci.yml b/.ci/ci.yml new file mode 100644 index 00000000000..f470182fc0c --- /dev/null +++ b/.ci/ci.yml @@ -0,0 +1,103 @@ +jobs: +- job: libshogun + displayName: Linux libshogun CI Tests + + pool: + vmImage: ubuntu-16.04 + + strategy: + maxParallel: 2 + matrix: + clang: + CC: "clang" + CXX: "clang++" + cmakeOptions: '' + gcc: + CC: "gcc" + CXX: "g++" + cmakeOptions: '' + + variables: + testRunTitle: '$(build.sourceBranchName)-debian' + testRunPlatform: debian + imageName: 'shogun/shogun-dev' + targetPrefix: '$(build.binariesDirectory)/opt' + libshogunArtifactName: 'libshogun-$(CC)' + libshogunArchive: '$(Build.ArtifactStagingDirectory)/libshogun-$(Build.BuildId).tar.xz' + ccacheDir: '$(build.binariesDirectory)/ccache' + ccacheArtifactName: 'ccache-$(CC)' + ccacheArchive: '$(Build.ArtifactStagingDirectory)/ccache-$(Build.BuildId).tar.xz' + + steps: + - template: ./docker-steps.yml + +- job: swig + displayName: Linux SWIG Interface CI Tests + dependsOn: libshogun + + pool: + vmImage: ubuntu-16.04 + + strategy: + maxParallel: 7 + matrix: + python: + CC: "clang" + CXX: "clang++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_PYTHON=ON' + java: + CC: "clang" + CXX: "clang++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_JAVA=ON' + csharp: + CC: "clang" + CXX: "clang++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_CSHARP=ON' + r: + CC: "clang" + CXX: "clang++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_R=ON' + lua: + CC: "clang" + CXX: "clang++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_LUA=ON' + octave: + CC: "gcc" + CXX: "g++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_OCTAVE=ON' + ruby: + CC: "clang" + CXX: "clang++" + cmakeOptions: '$(commonSWIGCMakeFlags) -DINTERFACE_RUBY=ON' + + variables: + swig: true + testRunTitle: '$(build.sourceBranchName)-debian' + testRunPlatform: debian + imageName: 'shogun/shogun-dev' + targetPrefix: '$(build.binariesDirectory)/opt' + libshogunArtifactName: 'libshogun-$(CC)' + libshogunArchive: '$(Build.ArtifactStagingDirectory)/libshogun-$(Build.BuildId).tar.xz' + commonSWIGCMakeFlags: '-DLIBSHOGUN=OFF -DTRAVIS_DISABLE_UNIT_TESTS=ON -DTRAVIS_DISABLE_META_CPP=ON' + + steps: + - template: ./docker-steps.yml + +#- job: macOS +# displayName: macOS CI Tests + +# variables: +# testRunTitle: '$(build.sourceBranchName)-macos' +# testRunPlatform: macos +# ccacheDir: $(Agent.HomeDirectory)/.ccache +# ccacheArtifactName: 'ccache-macos' + +# pool: +# vmImage: xcode9-macos10.13 + +# steps: +# - template: ./macos-steps.yml + +#- template: ./windows-steps.yml +# parameters: +# buildConfiguration: Release diff --git a/.ci/docker-steps.yml b/.ci/docker-steps.yml new file mode 100644 index 00000000000..3acda32b2a0 --- /dev/null +++ b/.ci/docker-steps.yml @@ -0,0 +1,123 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + submodules: true + +- script: mkdir $(ccacheDir) $(targetPrefix) + condition: + displayName: Create CCache directory + +- template: ./download-extract.yml + parameters: + name: 'ccache' + artifactName: $(ccacheArtifactName) + downloadPath: $(ccacheDir) + destinationFolder: $(ccacheDir) + archiveFilePatterns: '$(ccacheDir)/*.tar.xz' + +- script: | + ls -la $(ccacheDir) + displayName: Debug + +- script: | + echo $(cmakeOptions) + displayName: Debug + +- ${{ if eq(variables['swig'], 'true') }}: + - template: ./download-extract.yml + parameters: + name: 'libshogun' + artifactName: $(libshogunArtifactName) + downloadPath: $(targetPrefix) + destinationFolder: $(targetPrefix) + archiveFilePatterns: '$(targetPrefix)/*.tar.xz' + + - script: | + ls -la $(targetPrefix) + displayName: Debug + +- task: docker@0 + displayName: Setup + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + envVars: | + CCACHE_DIR=/build/ccache + CC=$(CC) + CXX=$(CXX) + workDir: '/build' + containerCommand: 'cmake -DENABLE_TESTING=ON -DCMAKE_INSTALL_PREFIX=/build/opt $(cmakeOptions) /src' + detached: false + +- ${{ if eq(variables['cmakeOptions'], '') }}: + # some of the cpps are just requiring way too much memory + - task: docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + envVars: | + CCACHE_DIR=/build/ccache + workDir: '/build' + containerCommand: 'make -s shogun' + detached: false + +- task: docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + envVars: | + CCACHE_DIR=/build/ccache + workDir: '/build' + containerCommand: 'make -j2 -s all' + detached: false + +- task: docker@0 + displayName: Install + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/build' + containerCommand: 'make install' + detached: false + +- task: docker@0 + displayName: Test + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/build' + containerCommand: 'ctest --output-on-failure -j 2' + detached: false + +- ${{ if eq(variables['cmakeOptions'], '') }}: + - template: ./archive-deploy.yml + parameters: + name: 'LibShogun' + dir: $(targetPrefix) + archive: $(libshogunArchive) + artifactName: $(libshogunArtifactName) + +- template: ./archive-deploy.yml + parameters: + name: 'CCache' + dir: $(ccacheDir) + archive: $(ccacheArchive) + artifactName: $(ccacheArtifactName) diff --git a/.ci/download-extract.yml b/.ci/download-extract.yml new file mode 100644 index 00000000000..fae3a787b05 --- /dev/null +++ b/.ci/download-extract.yml @@ -0,0 +1,27 @@ +parameters: + artifactName: '' + downloadPath: '' + name: '' + archiveFilePatterns: '' + destinationFolder: '' + +steps: +- task: DownloadBuildArtifacts@0 + displayName: Download ${{ parameters.name }} archive + continueOnError: true + inputs: + buildType: 'specific' + project: $(System.TeamProject) + pipeline: $(System.TeamProjectId) + branchName: $(Build.SourceBranch) + buildVersionToDownload: 'latest' + artifactName: ${{ parameters.artifactName }} + downloadPath: ${{ parameters.downloadPath }} + +- task: ExtractFiles@1 + displayName: Extract ${{ parameters.name }} archive + continueOnError: true + inputs: + archiveFilePatterns: ${{ parameters.archiveFilePatterns }} + destinationFolder: ${{ parameters.destinationFolder }} + cleanDestinationFolder: True diff --git a/.ci/macos-steps.yml b/.ci/macos-steps.yml new file mode 100644 index 00000000000..deab097ca24 --- /dev/null +++ b/.ci/macos-steps.yml @@ -0,0 +1,50 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + submodules: true + +- task: DownloadBuildArtifacts@0 + displayName: 'Download ccache archive' + continueOnError: true + inputs: + buildType: 'specific' + project: $(System.TeamProject) + pipeline: $(System.TeamProjectId) + branchName: $(Build.SourceBranch) + buildVersionToDownload: 'latest' + artifactName: $(ccacheArtifactName) + downloadPath: $(build.binariesDirectory) + +- task: ExtractFiles@1 + displayName: 'Extract ccache archive' + continueOnError: true + inputs: + archiveFilePatterns: '$(build.binariesDirectory)/*.tar.xz' + destinationFolder: $(ccacheDir) + cleanDestinationFolder: True + +- bash: | + brew update + brew install cmake ccache pkg-config arpack eigen glpk hdf5 json-c lapack lzo nlopt snappy xz + displayName: Install dependencies + +- task: CMake@1 + displayName: Setup + inputs: + cmakeArgs: -DBUILD_EXAMPLES=OFF -DBUNDLE_JSON=OFF -DBUNDLE_NLOPT=OFF -DENABLE_TESTING=ON -DENABLE_COVERAGE=OFF -DBUILD_META_EXAMPLES=OFF .. + +- script: make -j4 + displayName: Build + workingDirectory: build + +- script: ctest --output-on-failure -j 2 + displayName: Test + workingDirectory: build + +- template: ./archive-deploy.yml + parameters: + name: 'CCache' + dir: $(ccacheDir) + archive: $(ccacheArchive) + artifactName: $(ccacheArtifactName) diff --git a/.ci/setup_clcache.cmd b/.ci/setup_clcache.cmd new file mode 100644 index 00000000000..c8ffedc39c8 --- /dev/null +++ b/.ci/setup_clcache.cmd @@ -0,0 +1,16 @@ +@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd +@rem The cmd /C hack circumvents a regression where conda installs a conda.bat +@rem script in non-root environments. +set CONDA_INSTALL=cmd /C conda install -q -y +set PIP_INSTALL=pip install -q + +@echo on + +@rem Use clcache for faster builds +pip install -q git+https://github.com/frerich/clcache.git +clcache -s +set CLCACHE_SERVER=1 +set CLCACHE_HARDLINK=1 +powershell.exe -Command "Start-Process clcache-server" + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/.ci/windows-steps.yml b/.ci/windows-steps.yml new file mode 100644 index 00000000000..e682dadfadd --- /dev/null +++ b/.ci/windows-steps.yml @@ -0,0 +1,49 @@ +parameters: + buildConfiguration: Release + +jobs: +- job: Windows + displayName: Windows CI Tests + pool: + vmImage: 'vs2017-win2016' + + strategy: + matrix: + py36: + CONDA_PY: "36" + CONDA_ENV: shogun + + steps: + - checkout: self + clean: true + fetchDepth: 5 + submodules: true + + - task: CondaEnvironment@1 + displayName: Install dependencies + inputs: + createCustomEnvironment: True + environmentName: shogun + packageSpecs: 'python=3.6.* setuptools numpy scipy eigen rxcpp cmake snappy zlib ctags ply ninja mkl-devel jinja2' + createOptions: '-c conda-forge' + updateConda: false + + - script: | + .ci\\setup_clcache.cmd + displayName: Setup CLCache + + - task: CMake@1 + displayName: Setup + inputs: + cmakeArgs: '-G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=${{ parameters.buildConfiguration }} -DBUILD_META_EXAMPLES=OFF -DENABLE_TESTING=ON ..' + + - task: VSBuild@1 + displayName: Build + inputs: + solution: 'build/shogun.sln' + platform: x64 + configuration: ${{ parameters.buildConfiguration }} + + - script: ctest -j%NUMBER_OF_PROCESSORS% -C ${{ parameters.buildConfiguration }} -V + displayName: Test + workingDirectory: build diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ac344ad548c..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,79 +0,0 @@ -sudo: required -language: cpp -cache: ccache -notifications: - email: false - irc: - channels: - - "irc.freenode.org#shogun" - on_success: change - on_failure: always - template: - - "it's %{author}'s turn to pay the next round of drinks for the massacre he caused in %{repository}: %{build_url}" -env: - global: - - INTERFACE_TEST_OPTIONS="-DTRAVIS_DISABLE_UNIT_TESTS=ON -DTRAVIS_DISABLE_META_CPP=ON" -matrix: - fast_finish: true - include: - - compiler: gcc - services: docker - - compiler: clang - services: docker - - compiler: gcc - services: docker - env: - - CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON -DENABLE_CCACHE=OFF" - - CODE_COVERAGE=1 - - compiler: clang - services: docker - env: - - CMAKE_OPTIONS="-DINTERFACE_PYTHON=ON ${INTERFACE_TEST_OPTIONS}" - - CC=clang - - CXX=clang++ - - compiler: clang - services: docker - env: - - CMAKE_OPTIONS="-DINTERFACE_RUBY=ON ${INTERFACE_TEST_OPTIONS}" - - CC=clang - - CXX=clang++ - - compiler: clang - services: docker - env: - - CMAKE_OPTIONS="-DINTERFACE_JAVA=ON ${INTERFACE_TEST_OPTIONS}" - - CC=clang - - CXX=clang++ - - compiler: clang - services: docker - env: - - CMAKE_OPTIONS="-DINTERFACE_CSHARP=ON ${INTERFACE_TEST_OPTIONS}" - - CC=clang - - CXX=clang++ - - compiler: gcc - services: docker - env: - - CMAKE_OPTIONS="-DINTERFACE_OCTAVE=ON ${INTERFACE_TEST_OPTIONS}" - - INTERFACE_OCTAVE=true - - compiler: clang - services: docker - env: - - CMAKE_OPTIONS="-DINTERFACE_R=ON ${INTERFACE_TEST_OPTIONS}" - - CC=clang - - CXX=clang++ - allow_failures: - - compiler: gcc - services: docker - env: - - CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON -DENABLE_CCACHE=OFF" - - CODE_COVERAGE=1 -before_install: - - $TRAVIS_BUILD_DIR/scripts/travis_before_install.sh -before_script: - - $TRAVIS_BUILD_DIR/scripts/travis_before_script.sh -script: - - $TRAVIS_BUILD_DIR/scripts/travis_script.sh -after_success: - - | - if [ $CODE_COVERAGE ] ; then - docker exec -t devenv /bin/sh -c "cd /opt/shogun/build; lcov --directory src/shogun/ --capture --output-file all_coverage.info; lcov --remove all_coverage.info '/usr/*' > coverage.info; rm all_coverage.info; codecov -X gcov" - fi diff --git a/README.md b/README.md index 12400432d3f..2d9d7f7bbf8 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,7 @@ Cite Shogun: Develop branch build status: -[![Build Status](https://travis-ci.org/shogun-toolbox/shogun.svg?branch=develop)](https://travis-ci.org/shogun-toolbox/shogun) -[![Build status](https://ci.appveyor.com/api/projects/status/jx095rnr9qhg8dcv/branch/develop?svg=true)](https://ci.appveyor.com/project/vigsterkr/shogun/branch/develop) +[![Build status](https://dev.azure.com/shogunml/shogun/_apis/build/status/shogun-CI)](https://dev.azure.com/shogunml/shogun/_build/latest?definitionId=-1) [![codecov](https://codecov.io/gh/shogun-toolbox/shogun/branch/develop/graph/badge.svg)](https://codecov.io/gh/shogun-toolbox/shogun) Donate to Shogun via NumFocus: diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index e04d6604dc5..00000000000 --- a/appveyor.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: "{build}" - -cache: - - C:\Users\appveyor\clcache - - third_party/GoogleMock -> cmake/external/GoogleTestNMock.cmake - -environment: - matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - GENERATOR: Visual Studio 15 2017 Win64 - PYTHON: "3.6" - APPVEYOR_SAVE_CACHE_ON_ERROR: true - -platform: - - x64 - -configuration: - - Release - -install: - - set MINICONDA=C:\Miniconda36-x64 - - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH% - - call scripts\appveyor-setup.bat - - git submodule -q update --init - -build: - verbosity: minimal - -build_script: - - call scripts\msvc-build.bat - -test_script: - - cd %APPVEYOR_BUILD_FOLDER%\build - - ctest --output-on-failure -C %CONFIGURATION% - -after_build: - - clcache -s