Merge pull request #3 from kartikdutt18/restructure-pt1

Initial Restructuring (Adding Data loader and associated Utility Functions).

.ci/ci.yaml

@@ -4,12 +4,55 @@ trigger:
     - '*'
 pr:
   branches:
-    include:
-    - '*'
+     include:
+     - '*'
+
+# Define machines on which the build will run.
+jobs:
+- job: macOS
+  timeoutInMinutes: 360
+  displayName: macOS-10.14
+  pool:
+    vmImage: macOS-10.14
+
+  strategy:
+    matrix:
+      Plain:
+        CMakeArgs-mlpack: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF'
+        CMakeArgs-models: '-DDEBUG=ON'
+
+  steps:
+  - template: macos-steps.yaml
+
+# Linux build.
+- job: Linux
+  timeoutInMinutes: 360
+  pool:
+    vmImage: ubuntu-16.04
+
+  strategy:
+    matrix:
+      Plain:
+          CMakeArgs-mlpack: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF'
+          CMakeArgs-models: '-DDEBUG=ON'
+
+  steps:
+  - template: linux-steps.yaml
+
+# Windows build.
+- job: WindowsVS15
+  timeoutInMinutes: 360
+  displayName: Windows VS15
+  pool:
+    vmImage: vs2017-win2016
 
-pool:
-  vmImage: 'ubuntu-latest'
+  strategy:
+    matrix:
+      Plain:
+          CMakeArgs-mlpack: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF'
+          CMakeArgs-models: '-DDEBUG=ON'
+          CMakeGenerator: '-G "Visual Studio 15 2017 Win64"'
+          MSBuildVersion: '15.0'
 
-steps:
-- script: echo Hello, checkout mlpack/models!
-  displayName: 'Initial ci config.'
+  steps:
+  - template: windows-steps.yaml

.ci/linux-steps.yaml

@@ -0,0 +1,66 @@
+steps:
+# Checkout repository
+- checkout: self
+  clean: true
+  fetchDepth: 1
+
+# Install Build Dependencies.
+# Install mlpack from master to get new functionalities instantly.
+- script: |
+    git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf
+    git clone --depth 1 https://github.com/mlpack/mlpack.git
+
+    mkdir data
+
+    sudo add-apt-repository ppa:mhier/libboost-latest
+    sudo apt-get update
+
+    # Remove BOOST_ROOT from the environment to prevent attempting to use a
+    # boost which is incompatible with the compiler.
+    unset BOOST_ROOT
+    echo "##vso[task.setvariable variable=BOOST_ROOT]"$BOOST_ROOT
+
+    sudo apt-get install -y --allow-unauthenticated liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils
+
+    curl https://ftp.fau.de/macports/distfiles/armadillo/armadillo-8.400.0.tar.xz | tar -xvJ && cd armadillo*
+    cmake . && make && sudo make install && cd ..
+  displayName: 'Install Build Dependencies'
+
+# Configure CMake Args for mlpack.
+- script: |
+    unset BOOST_ROOT
+    cd mlpack && mkdir build && cd build && cmake $(CMakeArgs-mlpack) ..
+  displayName: 'CMake for mlpack'
+
+# Install mlpack
+- script: cd mlpack/build && sudo make install -j2
+  displayName: 'Install mlpack'
+
+# Configure CMake Args for models.
+- script: mkdir build && cd build && cmake $(CMakeArgs-models) ..
+  displayName: 'CMake for models'
+
+# Build mlpack
+- script: cd build && make -j2
+  displayName: 'Build models'
+
+# Run CTests.
+- script: cd build/tests/ && sudo CTEST_OUTPUT_ON_FAILURE=1 ctest -R UtilsTest
+  displayName: 'Run tests via ctest'
+
+# Publish test results to Azure Pipelines
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: cTest
+    testResultsFiles: build/tests/Testing/*/Test.xml
+    failTaskOnFailedTests: true
+  displayName: 'Publish tests'
+
+# Publish build artifacts to Azure Pipelines
+- task: PublishBuildArtifacts@1
+  inputs:
+    pathtoPublish: 'build/tests/Testing/'
+    artifactName: 'Tests'
+  displayName: 'Publish artifacts test results'
+
+

.ci/macos-steps.yaml

@@ -0,0 +1,58 @@
+steps:
+
+# Checkout repository.
+- checkout: self
+  clean: true
+  fetchDepth: 1
+
+# Install Build Dependencies.
+# Install mlpack from master to get new functionalities instantly.
+- script: |
+    set -e
+    sudo xcode-select --switch /Applications/Xcode_10.1.app/Contents/Developer
+    unset BOOST_ROOT
+    pip install cython numpy pandas zipp configparser
+    brew install openblas armadillo boost
+    git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf
+    git clone --depth 1 https://github.com/mlpack/mlpack.git
+  displayName: 'Install Build Dependencies'
+
+# Configure CMake Args for mlpack.
+- script: |
+    unset BOOST_ROOT
+    cd mlpack && mkdir build && cd build && cmake $(CMakeArgs-mlpack) ..
+  displayName: 'CMake for mlpack'
+
+# Install mlpack
+- script: cd mlpack/build && make install -j2
+  displayName: 'Install mlpack'
+
+# Configure CMake Args for models.
+- script: |
+    mkdir data && mkdir build && cd build && cmake $(CMakeArgs-models) ..
+  displayName: 'CMake for models'
+
+# Build mlpack
+- script: cd build && make -j2
+  displayName: 'Build models'
+
+# Run CTests.
+- script: cd build/tests/ && sudo CTEST_OUTPUT_ON_FAILURE=1 ctest -R UtilsTest
+  displayName: 'Run tests via ctest'
+
+# Publish test results to Azure Pipelines
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: cTest
+    testResultsFiles: build/tests/Testing/*/Test.xml
+    failTaskOnFailedTests: true
+  displayName: 'Publish tests'
+
+# Publish build artifacts to Azure Pipelines
+- task: PublishBuildArtifacts@1
+  inputs:
+    pathtoPublish: 'build/tests/Testing/'
+    artifactName: 'Tests'
+  displayName: 'Publish artifacts test results'
+
+

.ci/windows-steps.yaml

@@ -0,0 +1,146 @@
+steps:
+- checkout: self
+  clean: true
+  fetchDepth: 1
+- task: NuGetToolInstaller@0
+  inputs:
+    versionSpec: '5.3.0'
+
+# Fetch build dependencies
+- powershell: |
+    nuget install OpenBLAS -o $(Agent.ToolsDirectory)
+    nuget install boost -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_unit_test_framework-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_program_options-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_random-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_serialization-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_math_c99-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_filesystem-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_system-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_date_time-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install boost_regex-vc140 -o $(Agent.ToolsDirectory) -Version 1.60.0
+    nuget install OpenBLAS -o $(Agent.ToolsDirectory)
+
+    mkdir -p $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_program_options-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_math_c99-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_random-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_serialization-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_unit_test_framework-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_filesystem-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_system-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_date_time-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+    cp $(Agent.ToolsDirectory)/boost_regex-vc140.1.60.0.0/lib/native/address-model-64/lib/*.* $(Agent.ToolsDirectory)/boost_libs
+  displayName: 'Fetch build dependencies'
+
+# Configure armadillo
+- bash: |
+    git clone --depth 1 https://github.com/mlpack/jenkins-conf.git conf
+
+    curl -O http://www.ratml.org/misc/armadillo-8.400.0.tar.gz -o armadillo-8.400.0.tar.gz
+    tar -xzvf armadillo-8.400.0.tar.gz
+
+    cd armadillo-8.400.0/ && cmake $(CMakeGenerator) \
+    -DBLAS_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \
+    -DLAPACK_LIBRARY:FILEPATH=$(Agent.ToolsDirectory)/OpenBLAS.0.2.14.1/lib/native/lib/x64/libopenblas.dll.a \
+    -DCMAKE_PREFIX:FILEPATH=../../armadillo \
+    -DBUILD_SHARED_LIBS=OFF \
+    -DCMAKE_BUILD_TYPE=Release .
+  displayName: 'Configure armadillo'
+
+# Build armadillo
+- task: MSBuild@1
+  inputs:
+    solution: 'armadillo-8.400.0/*.sln'
+    msbuildLocationMethod: 'location'
+    msbuildVersion: $(MSBuildVersion)
+    configuration: 'Release'
+    msbuildArchitecture: 'x64'
+    msbuildArguments: /m /p:BuildInParallel=true
+    clean: false
+    maximumCpuCount: false
+  displayName: 'Build armadillo'
+
+# Configure mlpack
+- powershell: |
+    git clone --depth 1 https://github.com/mlpack/mlpack.git
+    mkdir mlpack\build
+    cd mlpack\build
+
+    cmake $(CMakeGenerator) `
+    $(CMakeArgs-mlpack) `
+    -DBLAS_LIBRARIES:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a `
+    -DLAPACK_LIBRARIES:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a `
+    -DARMADILLO_INCLUDE_DIR="..\..\armadillo-8.400.0\include" `
+    -DARMADILLO_LIBRARY="..\..\armadillo-8.400.0\Release\armadillo.lib" `
+    -DBOOST_INCLUDEDIR=$(Agent.ToolsDirectory)\boost.1.60.0.0\lib\native\include `
+    -DBOOST_LIBRARYDIR=$(Agent.ToolsDirectory)\boost_libs `
+    -DBUILD_JULIA_BINDINGS=OFF `
+    -DCMAKE_BUILD_TYPE=Release ..
+  displayName: 'Configure mlpack'
+
+# Build mlpack
+- task: MSBuild@1
+  inputs:
+    solution: 'mlpack/build/*.sln'
+    msbuildLocationMethod: 'location'
+    msbuildVersion: $(MSBuildVersion)
+    configuration: 'Release'
+    msbuildArchitecture: 'x64'
+    msbuildArguments: /m /p:BuildInParallel=true
+    maximumCpuCount: false
+    clean: false
+  displayName: 'Build mlpack'
+
+# Configure mlpack
+- powershell: |
+    cp $(Agent.ToolsDirectory)\boost_libs\*.* mlpack\build\
+    cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\*.* mlpack\build\
+    cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* mlpack\build\
+  displayName: 'Configure mlpack'
+
+- powershell: |
+    mkdir build
+    mkdir data
+    cp $(Agent.ToolsDirectory)\boost_libs\*.* build\
+    cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\*.* build\
+    cp $(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.* build\
+    cd build
+    cmake $(CMakeGenerator) `
+    $(CMakeArgs-models) `
+    -DBLAS_LIBRARIES:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a `
+    -DLAPACK_LIBRARIES:FILEPATH=$(Agent.ToolsDirectory)\OpenBLAS.0.2.14.1\lib\native\lib\x64\libopenblas.dll.a `
+    -DARMADILLO_INCLUDE_DIR="..\armadillo-8.400.0\include" `
+    -DARMADILLO_LIBRARY="..\armadillo-8.400.0\Release\armadillo.lib" `
+    -DMLPACK_INCLUDE_DIR="..\mlpack\build\include" `
+    -DMLPACK_LIBRARY="..\mlpack\build\Release\mlpack.lib" `
+    -DBOOST_INCLUDEDIR=$(Agent.ToolsDirectory)\boost.1.60.0.0\lib\native\include `
+    -DBOOST_LIBRARYDIR=$(Agent.ToolsDirectory)\boost_libs ..
+  displayName: 'Configure models'
+
+# Build models
+- task: MSBuild@1
+  inputs:
+    solution: 'build/*.sln'
+    msbuildLocationMethod: 'location'
+    msbuildVersion: $(MSBuildVersion)
+    configuration: 'Release'
+    msbuildArchitecture: 'x64'
+    msbuildArguments: /m /p:BuildInParallel=true
+    maximumCpuCount: false
+    clean: false
+  displayName: 'Build models'
+
+# Run tests via ctest.
+- bash: |
+    cd build/tests
+    CTEST_OUTPUT_ON_FAILURE=1 ctest -T Test -C Release -R UtilsTest
+  displayName: 'Run tests via ctest'
+
+# Publish test results to Azure Pipelines
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: cTest
+    testResultsFiles: build/tests/Testing/*/Test.xml
+    failTaskOnFailedTests: true
+  displayName: 'Publish tests'

.gitignore

@@ -2,4 +2,7 @@ build*
 xcode*
 .DS_Store
 .idea
-cmake-build-*
+cmake-build-*
+*.csv
+.travis/configs.hpp
+Testing/*