azure-pipelines.yml

# Hyperspace Build
trigger:
  batch: true
  branches:
    include:
      - master

jobs:
  - job: Build_Spark2_4_2_11
    displayName: 'Build sources and run unit tests for Spark 2.4 / Scala 2.11'
    pool:
      vmImage: 'ubuntu-18.04'
    steps:
      - template: 'ci/linux_test.yml'
        parameters:
          sparkVersionUnderBar: "2_4"
          sparkVersion: "2.4"
          scalaVersion: "2.11.12"

  - job: Build_Spark2_4_2_12
    displayName: 'Build sources and run unit tests for Spark 2.4 / Scala 2.12'
    pool:
      vmImage: 'ubuntu-18.04'
    steps:
      - template: 'ci/linux_test.yml'
        parameters:
          sparkVersionUnderBar: "2_4"
          sparkVersion: "2.4"
          scalaVersion: "2.12.8"

  - job: Build_Spark3_0_2_12
    displayName: 'Build sources and run unit tests for Spark 3.0 / Scala 2.12'
    pool:
      vmImage: 'ubuntu-18.04'
    steps:
      - template: 'ci/linux_test.yml'
        parameters:
          sparkVersionUnderBar: "3_0"
          sparkVersion: "3.0"
          scalaVersion: "2.12.8"

  - job: Build_Spark3_1_2_12
    displayName: 'Build sources and run unit tests for Spark 3.1 / Scala 2.12'
    pool:
      vmImage: 'ubuntu-18.04'
    steps:
      - template: 'ci/linux_test.yml'
        parameters:
          sparkVersionUnderBar: "3_1"
          sparkVersion: "3.1"
          scalaVersion: "2.12.8"

  - job: Build_Spark2_4_2_11_WIN
    displayName: 'Build sources and run unit tests for Spark 2.4 / Scala 2.11 on Windows'
    pool:
      vmImage: 'windows-2019'
    steps:
      - template: 'ci/windows_test.yml'
        parameters:
          sparkVersionUnderBar: "2_4"
          scalaVersion: "2.11.12"

  - job: Build_Spark2_4_2_12_WIN
    displayName: 'Build sources and run unit tests for Spark 2.4 / Scala 2.12 on Windows'
    pool:
      vmImage: 'windows-2019'
    steps:
      - template: 'ci/windows_test.yml'
        parameters:
          sparkVersionUnderBar: "2_4"
          scalaVersion: "2.12.8"

  - job: Build_Spark3_0_2_12_WIN
    displayName: 'Build sources and run unit tests for Spark 3.0 / Scala 2.12 on Windows'
    pool:
      vmImage: 'windows-2019'
    steps:
      - template: 'ci/windows_test.yml'
        parameters:
          sparkVersionUnderBar: "3_0"
          scalaVersion: "2.12.8"

  - job: PythonTest
    displayName: 'Run Python tests'
    pool:
      vmImage: 'ubuntu-18.04'
    steps:
      - task: UsePythonVersion@0
        displayName: 'Set Python version'
        inputs:
          versionSpec: '2.7'
          addToPath: true
      - task: JavaToolInstaller@0
        displayName: 'Set Java version'
        inputs:
          versionSpec: '8'
          jdkArchitectureOption: 'x64'
          jdkSourceOption: 'PreInstalled'
      - script: sbt ++2.11.12 "project spark2_4" clean update compile
        displayName: 'Running $sbt clean & update & compile'
      - task: Bash@3
        inputs:
          filePath: 'script/download_spark.sh'
        displayName: 'Downloading spark'
      - task: PythonScript@0
        inputs:
          scriptSource: 'filePath'
          scriptPath: 'run-tests.py'
        displayName: 'Running python tests'
        env:
          SPARK_HOME: $(Build.SourcesDirectory)/spark-2.4.2-bin-hadoop2.7