From 16cb709d2f8e59560406fa67220679ecf54f0afd Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 18 May 2020 01:53:53 -0700 Subject: [PATCH 01/39] test --- azure-pipelines.yml | 521 +++++++----------------------- script/download-spark-distros.cmd | 9 - 2 files changed, 113 insertions(+), 417 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 20215a7b2..b7d58a884 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,436 +38,141 @@ resources: name: dotnet/spark ref: refs/tags/v$(forwardCompatibleRelease) -jobs: -- job: Build - displayName: Build and Test Sources - pool: Hosted VS2017 +stages: +- stage: Build + jobs: + - job: Build + displayName: Build and Test Sources + pool: Hosted VS2017 - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - - steps: - - checkout: self - path: s\master - - checkout: forwardCompatibleRelease - path: s\$(forwardCompatibleRelease) - - - task: Maven@3 - displayName: 'Maven build src' - inputs: - mavenPomFile: master/src/scala/pom.xml - - - task: Maven@3 - displayName: 'Maven build benchmark' - inputs: - mavenPomFile: master/benchmark/scala/pom.xml - - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe - inputs: - filename: master\script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) - - - task: BatchScript@1 - displayName: Download backward compatible worker v$(backwardCompatibleRelease) - inputs: - filename: master\script\download-worker-release.cmd - arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' - - - script: master\build.cmd -pack - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=true - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker - displayName: '.NET build' - - - task: DotNetCoreCLI@2 - displayName: '.NET unit tests' - inputs: - command: test - projects: 'master/**/*UnitTest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.1' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.2' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.3' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.4' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.0' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.1' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.3' - inputs: - command: test - projects: 'master/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.4' - inputs: - command: test - projects: 'master/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.5' - inputs: - command: test - projects: 'master/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.1 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.2 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.3 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.4 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.0 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.1 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.3 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.4 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.5 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: 'master/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - - task: Maven@3 - displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' - inputs: - mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml - - - script: $(forwardCompatibleRelease)\build.cmd - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=false - displayName: '.NET build for forward compatible release v$(forwardCompatibleRelease)' + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + steps: + - checkout: self + path: s\master + - checkout: forwardCompatibleRelease + path: s\$(forwardCompatibleRelease) - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.1 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + - script: dir $(Build.BinariesDirectory) - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.2 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + - task: Maven@3 + displayName: 'Maven build src' + inputs: + mavenPomFile: master/src/scala/pom.xml - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.3 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + - task: Maven@3 + displayName: 'Maven build benchmark' + inputs: + mavenPomFile: master/benchmark/scala/pom.xml - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.4 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: master\script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + + - script: master\build.cmd -pack + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=true + /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker + displayName: '.NET build' + + - task: DotNetCoreCLI@2 + displayName: '.NET unit tests' + inputs: + command: test + projects: 'master/**/*UnitTest/*.csproj' + arguments: '--configuration $(buildConfiguration)' - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.0 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0' + inputs: + command: test + projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 + DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.1 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.3 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) +- stage: BackwardCompatibility + dependsOn: Build + jobs: + - job: BackwardCompatibility + # displayName: Build and Test Sources + pool: Hosted VS2017 - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.4 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.5 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) + steps: + - checkout: self + path: s\master + - checkout: forwardCompatibleRelease + path: s\$(forwardCompatibleRelease) - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Stage .NET artifacts - inputs: - sourceFolder: $(Build.SourcesDirectory)/master/artifacts/packages/$(buildConfiguration)/Shipping - contents: | - **/*.nupkg - **/*.snupkg - targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping + - script: dir $(Build.BinariesDirectory) - - task: CopyFiles@2 - displayName: Stage build logs + - task: BatchScript@1 + displayName: Download backward compatible worker v$(backwardCompatibleRelease) inputs: - sourceFolder: $(Build.SourcesDirectory)/master/artifacts/log - targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log + filename: master\script\download-worker-release.cmd + arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' - - task: PublishBuildArtifacts@1 + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: Microsoft.Spark.Binaries + command: test + projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 + DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - job: SignPublish - dependsOn: - - Build - displayName: Sign and Publish Artifacts - pool: - name: NetCoreInternal-Pool - queue: buildpool.windows.10.amd64.vs2017 +- stage: ForwardCompatibility + dependsOn: Build + jobs: + - job: ForwardCompatibility + # displayName: Build and Test Sources + pool: Hosted VS2017 variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts + - checkout: self + path: s\master + - checkout: forwardCompatibleRelease + path: s\$(forwardCompatibleRelease) + + - script: dir $(Build.BinariesDirectory) + + - task: Maven@3 + displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' inputs: - artifactName: Microsoft.Spark.Binaries - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin + mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml + + - script: $(forwardCompatibleRelease)\build.cmd + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=false + displayName: '.NET build for forward compatible release v$(forwardCompatibleRelease)' + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0 from forward compatible release v$(forwardCompatibleRelease)' inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' env: - TeamName: $(_TeamName) - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - task: PowerShell@2 - displayName: Sign artifacts and Package Microsoft.Spark.Worker - inputs: - filePath: eng\common\build.ps1 - arguments: -restore -sign -publish - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:DotNetSignType=$(_SignType) - /p:SparkPackagesDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\BuildArtifacts\artifacts\packages - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker - /p:SparkWorkerPackageOutputDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries' - artifactName: Microsoft.Spark.Binaries + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 + DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) \ No newline at end of file diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index d02bb49a7..913641739 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -14,14 +14,5 @@ cp hadoop-2.8.1\winutils.exe hadoop\bin echo "Downloading Spark distros." curl -k -L -o spark-2.3.0.tgz https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz && tar xzvf spark-2.3.0.tgz -curl -k -L -o spark-2.3.1.tgz https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzvf spark-2.3.1.tgz -curl -k -L -o spark-2.3.2.tgz https://archive.apache.org/dist/spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz && tar xzvf spark-2.3.2.tgz -curl -k -L -o spark-2.3.3.tgz https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz && tar xzvf spark-2.3.3.tgz -curl -k -L -o spark-2.3.4.tgz https://archive.apache.org/dist/spark/spark-2.3.4/spark-2.3.4-bin-hadoop2.7.tgz && tar xzvf spark-2.3.4.tgz -curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz && tar xzvf spark-2.4.0.tgz -curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz -curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz -curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz -curl -k -L -o spark-2.4.5.tgz https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz && tar xzvf spark-2.4.5.tgz endlocal \ No newline at end of file From dadd272e88bd13208c933b9dccb7f81c190c99c8 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 18 May 2020 02:20:38 -0700 Subject: [PATCH 02/39] test --- azure-pipelines.yml | 51 +++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b7d58a884..f9f69b23b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -51,30 +51,23 @@ stages: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop steps: - - checkout: self - path: s\master - - checkout: forwardCompatibleRelease - path: s\$(forwardCompatibleRelease) - - - script: dir $(Build.BinariesDirectory) - - task: Maven@3 displayName: 'Maven build src' inputs: - mavenPomFile: master/src/scala/pom.xml + mavenPomFile: src/scala/pom.xml - task: Maven@3 displayName: 'Maven build benchmark' inputs: - mavenPomFile: master/benchmark/scala/pom.xml + mavenPomFile: benchmark/scala/pom.xml - task: BatchScript@1 displayName: Download Spark Distros & Winutils.exe inputs: - filename: master\script\download-spark-distros.cmd + filename: script\download-spark-distros.cmd arguments: $(Build.BinariesDirectory) - - script: master\build.cmd -pack + - script: build.cmd -pack -c $(buildConfiguration) -ci $(_OfficialBuildIdArgs) @@ -86,14 +79,14 @@ stages: displayName: '.NET unit tests' inputs: command: test - projects: 'master/**/*UnitTest/*.csproj' + projects: '**/*UnitTest/*.csproj' arguments: '--configuration $(buildConfiguration)' - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' inputs: command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 @@ -112,24 +105,23 @@ stages: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop steps: - - checkout: self - path: s\master - - checkout: forwardCompatibleRelease - path: s\$(forwardCompatibleRelease) - - - script: dir $(Build.BinariesDirectory) + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) - task: BatchScript@1 displayName: Download backward compatible worker v$(backwardCompatibleRelease) inputs: - filename: master\script\download-worker-release.cmd + filename: script\download-worker-release.cmd arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' inputs: command: test - projects: 'master/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 @@ -148,19 +140,20 @@ stages: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop steps: - - checkout: self - path: s\master - checkout: forwardCompatibleRelease - path: s\$(forwardCompatibleRelease) - - - script: dir $(Build.BinariesDirectory) - task: Maven@3 displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' inputs: - mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml + mavenPomFile: src/scala/pom.xml - - script: $(forwardCompatibleRelease)\build.cmd + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + + - script: build.cmd -c $(buildConfiguration) -ci $(_OfficialBuildIdArgs) @@ -171,7 +164,7 @@ stages: displayName: 'E2E tests for Spark 2.3.0 from forward compatible release v$(forwardCompatibleRelease)' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 From 9bb78b8f29f56ac080dcb5e2a90521d869833fe3 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 19 May 2020 00:26:40 -0700 Subject: [PATCH 03/39] test --- azure-pipelines.yml | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f9f69b23b..8cda1c399 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,4 @@ -# Spark .NET build + # Spark .NET build trigger: batch: true @@ -105,6 +105,11 @@ stages: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop steps: + - task: Maven@3 + displayName: 'Maven build src' + inputs: + mavenPomFile: src/scala/pom.xml + - task: BatchScript@1 displayName: Download Spark Distros & Winutils.exe inputs: @@ -117,6 +122,13 @@ stages: filename: script\download-worker-release.cmd arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' + - script: build.cmd + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=false + displayName: '.NET build for backward compatible release v$(backwardCompatibleRelease)' + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' inputs: @@ -140,26 +152,37 @@ stages: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop steps: + - checkout: self + path: s\master - checkout: forwardCompatibleRelease - + path: s\$(forwardCompatibleRelease) + - task: Maven@3 displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' inputs: - mavenPomFile: src/scala/pom.xml + mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml - task: BatchScript@1 displayName: Download Spark Distros & Winutils.exe inputs: - filename: script\download-spark-distros.cmd + filename: master\script\download-spark-distros.cmd arguments: $(Build.BinariesDirectory) - - script: build.cmd + - script: $(forwardCompatibleRelease)\build.cmd -c $(buildConfiguration) -ci $(_OfficialBuildIdArgs) /p:PublishSparkWorker=false displayName: '.NET build for forward compatible release v$(forwardCompatibleRelease)' + - script: master\build.cmd -pack + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=true + /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker + displayName: '.NET build' + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0 from forward compatible release v$(forwardCompatibleRelease)' inputs: From d1df44fb4b6cf8e2c4aab072fcd63b38e920f979 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 19 May 2020 00:55:20 -0700 Subject: [PATCH 04/39] test --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8cda1c399..08e0d5234 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -187,7 +187,7 @@ stages: displayName: 'E2E tests for Spark 2.3.0 from forward compatible release v$(forwardCompatibleRelease)' inputs: command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 From 1856063db085112bf08bad050a5261eb3557d04b Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 19 May 2020 03:25:48 -0700 Subject: [PATCH 05/39] init --- azure-pipelines.yml | 396 +++++++++++++++++++++++++++--- script/download-spark-distros.cmd | 9 + 2 files changed, 365 insertions(+), 40 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 08e0d5234..b3e51119f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,4 @@ - # Spark .NET build +# Spark .NET build trigger: batch: true @@ -22,7 +22,7 @@ variables: (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.SparkSessionTests.TestCreateDataFrameWithTimestamp)" - LatestDotnetWorkerDir: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp3.1\win-x64' + CurrentDotnetWorkerDir: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp3.1\win-x64' BackwardCompatibleDotnetWorkerDir: $(Build.BinariesDirectory)\Microsoft.Spark.Worker-$(backwardCompatibleRelease) # Azure DevOps variables are transformed into environment variables, with these variables we @@ -40,6 +40,7 @@ resources: stages: - stage: Build + displayName: Build and Test Sources jobs: - job: Build displayName: Build and Test Sources @@ -49,6 +50,7 @@ stages: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: - task: Maven@3 @@ -90,66 +92,170 @@ stages: arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) -- stage: BackwardCompatibility - dependsOn: Build - jobs: - - job: BackwardCompatibility - # displayName: Build and Test Sources - pool: Hosted VS2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.1' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - steps: - - task: Maven@3 - displayName: 'Maven build src' + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.2' inputs: - mavenPomFile: src/scala/pom.xml + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.3' inputs: - filename: script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - - task: BatchScript@1 - displayName: Download backward compatible worker v$(backwardCompatibleRelease) + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.4' inputs: - filename: script\download-worker-release.cmd - arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - - script: build.cmd - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=false - displayName: '.NET build for backward compatible release v$(backwardCompatibleRelease)' + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.0' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' + displayName: 'E2E tests for Spark 2.4.1' inputs: command: test projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.4' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.5' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Stage .NET artifacts + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping + contents: | + **/*.nupkg + **/*.snupkg + targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping + + - task: CopyFiles@2 + displayName: Stage build logs + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/log + targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: Microsoft.Spark.Binaries + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - job: SignPublish + dependsOn: + - Build + displayName: Sign and Publish Artifacts + pool: + name: NetCoreInternal-Pool + queue: buildpool.windows.10.amd64.vs2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - task: PowerShell@2 + displayName: Sign artifacts and Package Microsoft.Spark.Worker + inputs: + filePath: eng\common\build.ps1 + arguments: -restore -sign -publish + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:DotNetSignType=$(_SignType) + /p:SparkPackagesDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\BuildArtifacts\artifacts\packages + /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker + /p:SparkWorkerPackageOutputDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries' + artifactName: Microsoft.Spark.Binaries - stage: ForwardCompatibility + displayName: Forward Compatibility Test dependsOn: Build jobs: - job: ForwardCompatibility - # displayName: Build and Test Sources + displayName: Forward Compatibility Test pool: Hosted VS2017 variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: - checkout: self @@ -165,7 +271,7 @@ stages: - task: BatchScript@1 displayName: Download Spark Distros & Winutils.exe inputs: - filename: master\script\download-spark-distros.cmd + filename: $(forwardCompatibleRelease)\script\download-spark-distros.cmd arguments: $(Build.BinariesDirectory) - script: $(forwardCompatibleRelease)\build.cmd @@ -175,7 +281,7 @@ stages: /p:PublishSparkWorker=false displayName: '.NET build for forward compatible release v$(forwardCompatibleRelease)' - - script: master\build.cmd -pack + - script: master\build.cmd -c $(buildConfiguration) -ci $(_OfficialBuildIdArgs) @@ -191,4 +297,214 @@ stages: arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - DOTNET_WORKER_DIR: $(LatestDotnetWorkerDir) \ No newline at end of file + +- task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.1 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.2 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.3 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.4 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.0 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.1 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.4 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.5 from forward compatible release v$(forwardCompatibleRelease)' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 + +- stage: BackwardCompatibility + displayName: Backward Compatibility Test + dependsOn: Build + jobs: + - job: BackwardCompatibility + displayName: Backward Compatibility Test + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) + + steps: + - task: Maven@3 + displayName: 'Maven build src' + inputs: + mavenPomFile: src/scala/pom.xml + + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + + - task: BatchScript@1 + displayName: Download backward compatible worker v$(backwardCompatibleRelease) + inputs: + filename: script\download-worker-release.cmd + arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' + + - script: build.cmd + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=false + displayName: '.NET build' + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.1 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.2 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration --filter $(TestsToFilterOut) + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.3 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.4 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.0 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.1 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.4 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.5 backward compatible worker v$(backwardCompatibleRelease)' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 + diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index 913641739..d02bb49a7 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -14,5 +14,14 @@ cp hadoop-2.8.1\winutils.exe hadoop\bin echo "Downloading Spark distros." curl -k -L -o spark-2.3.0.tgz https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz && tar xzvf spark-2.3.0.tgz +curl -k -L -o spark-2.3.1.tgz https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzvf spark-2.3.1.tgz +curl -k -L -o spark-2.3.2.tgz https://archive.apache.org/dist/spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz && tar xzvf spark-2.3.2.tgz +curl -k -L -o spark-2.3.3.tgz https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz && tar xzvf spark-2.3.3.tgz +curl -k -L -o spark-2.3.4.tgz https://archive.apache.org/dist/spark/spark-2.3.4/spark-2.3.4-bin-hadoop2.7.tgz && tar xzvf spark-2.3.4.tgz +curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz && tar xzvf spark-2.4.0.tgz +curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz +curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz +curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz +curl -k -L -o spark-2.4.5.tgz https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz && tar xzvf spark-2.4.5.tgz endlocal \ No newline at end of file From 0eb90561e6d45437f34fa1ae55b2774d02c50d19 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 19 May 2020 03:31:19 -0700 Subject: [PATCH 06/39] indentation --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b3e51119f..5815f7d68 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -298,7 +298,7 @@ stages: env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 -- task: DotNetCoreCLI@2 + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.1 from forward compatible release v$(forwardCompatibleRelease)' inputs: command: test From 86a8685544409ca23dfd4aee04d8b50e367e9ac2 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 19 May 2020 03:41:44 -0700 Subject: [PATCH 07/39] fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5815f7d68..0ade8dc34 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -441,7 +441,7 @@ stages: inputs: command: test projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration --filter $(TestsToFilterOut) + arguments: '--configuration $(buildConfiguration --filter $(TestsToFilterOut)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 From 37553c85d38ac45a1ee57cfd46450c84347b0f21 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Thu, 21 May 2020 18:00:17 -0700 Subject: [PATCH 08/39] test --- azure-pipelines.yml | 427 ++---------------------------- script/download-spark-distros.cmd | 9 - 2 files changed, 24 insertions(+), 412 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0ade8dc34..39376d2c5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,427 +84,48 @@ stages: projects: '**/*UnitTest/*.csproj' arguments: '--configuration $(buildConfiguration)' - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0' + - task: CopyFiles@2 + displayName: Stage .NET artifacts inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.1' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.2' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.3' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.4' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.0' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.1' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.3' - inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.4' - inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 + sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping + contents: | + **/*.nupkg + **/*.snupkg + targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.5' + - task: CopyFiles@2 + displayName: Copy Spark Distros inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Stage .NET artifacts - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping - contents: | - **/*.nupkg - **/*.snupkg - targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping - - - task: CopyFiles@2 - displayName: Stage build logs - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/log - targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log + sourceFolder: $(Build.BinariesDirectory) + targetFolder: $(Build.ArtifactStagingDirectory)/Binaries - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: Microsoft.Spark.Binaries + - upload: $(Build.ArtifactStagingDirectory) + artifact: Microsoft.Spark.Binaries - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - job: SignPublish - dependsOn: - - Build - displayName: Sign and Publish Artifacts - pool: - name: NetCoreInternal-Pool - queue: buildpool.windows.10.amd64.vs2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - artifactName: Microsoft.Spark.Binaries - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - task: PowerShell@2 - displayName: Sign artifacts and Package Microsoft.Spark.Worker - inputs: - filePath: eng\common\build.ps1 - arguments: -restore -sign -publish - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:DotNetSignType=$(_SignType) - /p:SparkPackagesDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\BuildArtifacts\artifacts\packages - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker - /p:SparkWorkerPackageOutputDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries' - artifactName: Microsoft.Spark.Binaries - -- stage: ForwardCompatibility - displayName: Forward Compatibility Test +- stage: Test + displayName: Test Sources dependsOn: Build jobs: - - job: ForwardCompatibility - displayName: Forward Compatibility Test + - job: Test + displayName: Test Sources pool: Hosted VS2017 variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Binaries\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: - - checkout: self - path: s\master - - checkout: forwardCompatibleRelease - path: s\$(forwardCompatibleRelease) - - - task: Maven@3 - displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' - inputs: - mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml - - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe - inputs: - filename: $(forwardCompatibleRelease)\script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) - - - script: $(forwardCompatibleRelease)\build.cmd - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=false - displayName: '.NET build for forward compatible release v$(forwardCompatibleRelease)' - - - script: master\build.cmd - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=true - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker - displayName: '.NET build' - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.1 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.2 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.3 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.4 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.0 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.1 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.3 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.4 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.5 from forward compatible release v$(forwardCompatibleRelease)' - inputs: - command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - -- stage: BackwardCompatibility - displayName: Backward Compatibility Test - dependsOn: Build - jobs: - - job: BackwardCompatibility - displayName: Backward Compatibility Test - pool: Hosted VS2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) - - steps: - - task: Maven@3 - displayName: 'Maven build src' - inputs: - mavenPomFile: src/scala/pom.xml - - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe - inputs: - filename: script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) - - - task: BatchScript@1 - displayName: Download backward compatible worker v$(backwardCompatibleRelease) - inputs: - filename: script\download-worker-release.cmd - arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' - - - script: build.cmd - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=false - displayName: '.NET build' + - download: current + artifact: Microsoft.Spark.Binaries + path: $(Build.ArtifactStagingDirectory) - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0 with backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.1 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.2 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.3 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.4 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.0 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.1 backward compatible worker v$(backwardCompatibleRelease)' + displayName: 'E2E tests for Spark 2.3.0' inputs: command: test projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.3 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.4 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' - env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.4.5 backward compatible worker v$(backwardCompatibleRelease)' - inputs: - command: test - projects: '**/Microsoft.Spark*.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Binaries\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index d02bb49a7..913641739 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -14,14 +14,5 @@ cp hadoop-2.8.1\winutils.exe hadoop\bin echo "Downloading Spark distros." curl -k -L -o spark-2.3.0.tgz https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz && tar xzvf spark-2.3.0.tgz -curl -k -L -o spark-2.3.1.tgz https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzvf spark-2.3.1.tgz -curl -k -L -o spark-2.3.2.tgz https://archive.apache.org/dist/spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz && tar xzvf spark-2.3.2.tgz -curl -k -L -o spark-2.3.3.tgz https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz && tar xzvf spark-2.3.3.tgz -curl -k -L -o spark-2.3.4.tgz https://archive.apache.org/dist/spark/spark-2.3.4/spark-2.3.4-bin-hadoop2.7.tgz && tar xzvf spark-2.3.4.tgz -curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz && tar xzvf spark-2.4.0.tgz -curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz -curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz -curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz -curl -k -L -o spark-2.4.5.tgz https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz && tar xzvf spark-2.4.5.tgz endlocal \ No newline at end of file From b9ee68dc8f3b56674ed6e4be93d09a949a84b885 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Thu, 21 May 2020 18:03:39 -0700 Subject: [PATCH 09/39] test --- azure-pipelines.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 39376d2c5..5bb9561a0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -117,9 +117,11 @@ stages: DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: - - download: current - artifact: Microsoft.Spark.Binaries - path: $(Build.ArtifactStagingDirectory) + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts and Binaries + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' From c5a0539108094d589502af69647d04742e303b52 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 17:01:39 -0700 Subject: [PATCH 10/39] test --- azure-pipelines.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5bb9561a0..de296cb9f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -85,22 +85,15 @@ stages: arguments: '--configuration $(buildConfiguration)' - task: CopyFiles@2 - displayName: Stage .NET artifacts - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping - contents: | - **/*.nupkg - **/*.snupkg - targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping - - - task: CopyFiles@2 - displayName: Copy Spark Distros + displayName: Publish Spark Distros inputs: sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/Binaries + targetFolder: $(Build.ArtifactStagingDirectory)/Distros - - upload: $(Build.ArtifactStagingDirectory) - artifact: Microsoft.Spark.Binaries + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: Microsoft.Spark.Binaries - stage: Test displayName: Test Sources @@ -113,7 +106,7 @@ stages: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Binaries\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Distros\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -130,4 +123,4 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Binaries\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file From c696ed4c84980efc4acf1211db51771443078d9f Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 17:36:31 -0700 Subject: [PATCH 11/39] test: stage to job --- azure-pipelines.yml | 160 +++++++++++++++++++++----------------------- 1 file changed, 77 insertions(+), 83 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index de296cb9f..391d18fba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,89 +38,83 @@ resources: name: dotnet/spark ref: refs/tags/v$(forwardCompatibleRelease) -stages: -- stage: Build +jobs: +- job: Build displayName: Build and Test Sources - jobs: - - job: Build - displayName: Build and Test Sources - pool: Hosted VS2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) - - steps: - - task: Maven@3 - displayName: 'Maven build src' - inputs: - mavenPomFile: src/scala/pom.xml - - - task: Maven@3 - displayName: 'Maven build benchmark' - inputs: - mavenPomFile: benchmark/scala/pom.xml - - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe - inputs: - filename: script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) - - - script: build.cmd -pack - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=true - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker - displayName: '.NET build' - - - task: DotNetCoreCLI@2 - displayName: '.NET unit tests' - inputs: - command: test - projects: '**/*UnitTest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: CopyFiles@2 - displayName: Publish Spark Distros - inputs: - sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/Distros - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: Microsoft.Spark.Binaries - -- stage: Test + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) + + steps: + - task: Maven@3 + displayName: 'Maven build src' + inputs: + mavenPomFile: src/scala/pom.xml + + - task: Maven@3 + displayName: 'Maven build benchmark' + inputs: + mavenPomFile: benchmark/scala/pom.xml + + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + + - script: build.cmd -pack + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=true + /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker + displayName: '.NET build' + + - task: DotNetCoreCLI@2 + displayName: '.NET unit tests' + inputs: + command: test + projects: '**/*UnitTest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: CopyFiles@2 + displayName: Publish Spark Distros + inputs: + sourceFolder: $(Build.BinariesDirectory) + targetFolder: $(Build.ArtifactStagingDirectory)/Distros + + - task: PublishBuildArtifacts@1git + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: Microsoft.Spark.Binaries + +- job: Test displayName: Test Sources dependsOn: Build - jobs: - - job: Test - displayName: Test Sources - pool: Hosted VS2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Distros\hadoop - DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) - - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts and Binaries - inputs: - artifactName: Microsoft.Spark.Binaries - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Distros\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) + + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts and Binaries + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file From 5c2498a07c7ee0c3e43dd7c7518b5788b24deecc Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 18:09:34 -0700 Subject: [PATCH 12/39] test: fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 391d18fba..151563649 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,7 +87,7 @@ jobs: sourceFolder: $(Build.BinariesDirectory) targetFolder: $(Build.ArtifactStagingDirectory)/Distros - - task: PublishBuildArtifacts@1git + - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: Microsoft.Spark.Binaries From a2adfd5bc08206e2ea2acf73116c5386e283b087 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 18:34:23 -0700 Subject: [PATCH 13/39] test --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 151563649..f9e0c822c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -85,7 +85,7 @@ jobs: displayName: Publish Spark Distros inputs: sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/Distros + targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/distros - task: PublishBuildArtifacts@1 inputs: @@ -100,7 +100,7 @@ jobs: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Distros\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\BuildArtifacts\artifacts\distros\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -117,4 +117,4 @@ jobs: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file + SPARK_HOME: $(Build.ArtifactStagingDirectory)\BuildArtifacts\artifacts\distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file From 6bff7465f7aa7f4836b432ffff2a63f5aa2770b6 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 18:55:49 -0700 Subject: [PATCH 14/39] test: publish and download --- azure-pipelines.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f9e0c822c..0b2bdd886 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,12 +81,6 @@ jobs: projects: '**/*UnitTest/*.csproj' arguments: '--configuration $(buildConfiguration)' - - task: CopyFiles@2 - displayName: Publish Spark Distros - inputs: - sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/distros - - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' @@ -99,9 +93,7 @@ jobs: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\BuildArtifacts\artifacts\distros\hadoop - DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) steps: - task: DownloadBuildArtifacts@0 @@ -110,11 +102,4 @@ jobs: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) - - task: DotNetCoreCLI@2 - displayName: 'E2E tests for Spark 2.3.0' - inputs: - command: test - projects: '**/Microsoft.Spark.E2ETest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\BuildArtifacts\artifacts\distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file + - script: dir $(Build.ArtifactStagingDirectory) \ No newline at end of file From a7e76b00b31de2ab3eee4e21c1e510be67035767 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 22:10:19 -0700 Subject: [PATCH 15/39] test: revert to stage --- azure-pipelines.yml | 145 +++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 62 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0b2bdd886..05e84231d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,68 +38,89 @@ resources: name: dotnet/spark ref: refs/tags/v$(forwardCompatibleRelease) -jobs: -- job: Build +stages: +- stage: Build displayName: Build and Test Sources - pool: Hosted VS2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) - - steps: - - task: Maven@3 - displayName: 'Maven build src' - inputs: - mavenPomFile: src/scala/pom.xml - - - task: Maven@3 - displayName: 'Maven build benchmark' - inputs: - mavenPomFile: benchmark/scala/pom.xml - - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe - inputs: - filename: script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) - - - script: build.cmd -pack - -c $(buildConfiguration) - -ci - $(_OfficialBuildIdArgs) - /p:PublishSparkWorker=true - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker - displayName: '.NET build' - - - task: DotNetCoreCLI@2 - displayName: '.NET unit tests' - inputs: - command: test - projects: '**/*UnitTest/*.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: Microsoft.Spark.Binaries - -- job: Test + jobs: + - job: Build + displayName: Build and Test Sources + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) + + steps: + - task: Maven@3 + displayName: 'Maven build src' + inputs: + mavenPomFile: src/scala/pom.xml + + - task: Maven@3 + displayName: 'Maven build benchmark' + inputs: + mavenPomFile: benchmark/scala/pom.xml + + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + + - script: build.cmd -pack + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:PublishSparkWorker=true + /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker + displayName: '.NET build' + + - task: DotNetCoreCLI@2 + displayName: '.NET unit tests' + inputs: + command: test + projects: '**/*UnitTest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: CopyFiles@2 + displayName: Publish Spark Distros + inputs: + sourceFolder: $(Build.BinariesDirectory) + targetFolder: $(Build.ArtifactStagingDirectory)/Distros + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: Microsoft.Spark.Binaries + +- stage: Test displayName: Test Sources dependsOn: Build - pool: Hosted VS2017 - - variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts and Binaries - inputs: - artifactName: Microsoft.Spark.Binaries - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: dir $(Build.ArtifactStagingDirectory) \ No newline at end of file + jobs: + - job: Test + displayName: Test Sources + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Distros\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) + + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts and Binaries + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file From 072a4472a18c790b9009244ef957d593c535e125 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 22:57:19 -0700 Subject: [PATCH 16/39] test: copy jars --- azure-pipelines.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 05e84231d..d7f2d7993 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -85,10 +85,17 @@ stages: arguments: '--configuration $(buildConfiguration)' - task: CopyFiles@2 - displayName: Publish Spark Distros + displayName: Publish Spark distros inputs: sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/Distros + targetFolder: $(Build.ArtifactStagingDirectory)/distros + + - task: CopyFiles@2 + displayName: Publish maven build + inputs: + sourceFolder: $(Build.SourcesDirectory) + contents: **/microsoft-spark-2.3.x**/*.jar + targetFolder: $(Build.ArtifactStagingDirectory)/jars - task: PublishBuildArtifacts@1 inputs: @@ -106,7 +113,7 @@ stages: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Distros\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -116,6 +123,12 @@ stages: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) + - task: CopyFiles@2 + displayName: Copy jar files + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/jars + targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' inputs: @@ -123,4 +136,4 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file From 999449fe921144971788586c4bc0d72828aea22c Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 23:00:58 -0700 Subject: [PATCH 17/39] test: fix indentation --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d7f2d7993..ddc2fa6be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,7 +94,7 @@ stages: displayName: Publish maven build inputs: sourceFolder: $(Build.SourcesDirectory) - contents: **/microsoft-spark-2.3.x**/*.jar + contents: **/microsoft-spark-2.3.x**/*.jar targetFolder: $(Build.ArtifactStagingDirectory)/jars - task: PublishBuildArtifacts@1 From f41fdb42706c3ddcff95579c075efcf3aca67999 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 23:06:39 -0700 Subject: [PATCH 18/39] test: fix contents --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ddc2fa6be..48522f9c0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -93,8 +93,7 @@ stages: - task: CopyFiles@2 displayName: Publish maven build inputs: - sourceFolder: $(Build.SourcesDirectory) - contents: **/microsoft-spark-2.3.x**/*.jar + sourceFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target targetFolder: $(Build.ArtifactStagingDirectory)/jars - task: PublishBuildArtifacts@1 From d75efd1e3a1af7321494d49cd211fd24e8e583c2 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 23:16:58 -0700 Subject: [PATCH 19/39] test: add jar contents --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 48522f9c0..0358d71fe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,6 +94,7 @@ stages: displayName: Publish maven build inputs: sourceFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + contents: **/*.jar targetFolder: $(Build.ArtifactStagingDirectory)/jars - task: PublishBuildArtifacts@1 From 6c0cd1c99a0783725b27b14dcf96af225110a4eb Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 23:20:10 -0700 Subject: [PATCH 20/39] test: jar --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0358d71fe..ac5594be4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,7 +94,7 @@ stages: displayName: Publish maven build inputs: sourceFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target - contents: **/*.jar + contents: *.jar targetFolder: $(Build.ArtifactStagingDirectory)/jars - task: PublishBuildArtifacts@1 From 3b01efd61c8b7e71629b661abff8db6a9f439af2 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 23:25:52 -0700 Subject: [PATCH 21/39] test: add quotes --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ac5594be4..4aa1b85d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,7 +94,7 @@ stages: displayName: Publish maven build inputs: sourceFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target - contents: *.jar + contents: "**/*.jar" targetFolder: $(Build.ArtifactStagingDirectory)/jars - task: PublishBuildArtifacts@1 From 98e04f7f222b1836d6931a426528738a1f022877 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Mon, 1 Jun 2020 23:53:11 -0700 Subject: [PATCH 22/39] test: worker dir --- azure-pipelines.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4aa1b85d4..25ab54032 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ variables: (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.SparkSessionTests.TestCreateDataFrameWithTimestamp)" - CurrentDotnetWorkerDir: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp3.1\win-x64' + CurrentDotnetWorkerDir: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker\netcoreapp3.1\win-x64' BackwardCompatibleDotnetWorkerDir: $(Build.BinariesDirectory)\Microsoft.Spark.Worker-$(backwardCompatibleRelease) # Azure DevOps variables are transformed into environment variables, with these variables we @@ -49,8 +49,6 @@ stages: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.BinariesDirectory)\hadoop - DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: - task: Maven@3 From 199a538b4db389daaa9846b5addbe2dbd41508eb Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 00:36:55 -0700 Subject: [PATCH 23/39] test: add fc bc --- azure-pipelines.yml | 90 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 25ab54032..abe86e26e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,10 +40,9 @@ resources: stages: - stage: Build - displayName: Build and Test Sources + displayName: Build Sources jobs: - job: Build - displayName: Build and Test Sources pool: Hosted VS2017 variables: @@ -83,13 +82,13 @@ stages: arguments: '--configuration $(buildConfiguration)' - task: CopyFiles@2 - displayName: Publish Spark distros + displayName: Stage Spark distros inputs: sourceFolder: $(Build.BinariesDirectory) targetFolder: $(Build.ArtifactStagingDirectory)/distros - task: CopyFiles@2 - displayName: Publish maven build + displayName: Stage Maven build jars inputs: sourceFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target contents: "**/*.jar" @@ -105,7 +104,6 @@ stages: dependsOn: Build jobs: - job: Test - displayName: Test Sources pool: Hosted VS2017 variables: @@ -133,5 +131,87 @@ stages: command: test projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 + +- stage: ForwardCompatibility + displayName: Test Forward Compatibility + dependsOn: Build + jobs: + - job: ForwardCompatibility + displayName: 'E2E tests from forward compatible release v$(forwardCompatibleRelease)' + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\hadoop + DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) + + steps: + - checkout: self + path: s\master + - checkout: forwardCompatibleRelease + path: s\$(forwardCompatibleRelease) + + - task: Maven@3 + displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' + inputs: + mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml + + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts and Binaries + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 + +- stage: BackwardCompatibility + displayName: Test Backward Compatibility + dependsOn: Build + jobs: + - job: BackwardCompatibility + displayName: 'E2E tests with backward compatible worker v$(backwardCompatibleRelease)' + pool: Hosted VS2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\hadoop + DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) + + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts and Binaries + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: CopyFiles@2 + displayName: Copy jar files + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/jars + targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + + - task: BatchScript@1 + displayName: Download backward compatible worker v$(backwardCompatibleRelease) + inputs: + filename: script\download-worker-release.cmd + arguments: '$(Build.BinariesDirectory) $(backwardCompatibleRelease)' + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.0' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file From 30ac6d5a1075968f2c08f460bdb08855d1fd14aa Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 01:49:49 -0700 Subject: [PATCH 24/39] test: final --- azure-pipelines.yml | 356 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 340 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index abe86e26e..ef68d88dc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -85,31 +85,96 @@ stages: displayName: Stage Spark distros inputs: sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/distros + targetFolder: $(Build.ArtifactStagingDirectory)/SparkDistros - task: CopyFiles@2 displayName: Stage Maven build jars inputs: - sourceFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target - contents: "**/*.jar" - targetFolder: $(Build.ArtifactStagingDirectory)/jars + sourceFolder: $(Build.SourcesDirectory) + contents: '**/*.jar' + targetFolder: $(Build.ArtifactStagingDirectory)/Jars + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Stage .NET artifacts + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping + contents: | + **/*.nupkg + **/*.snupkg + targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping + + - task: CopyFiles@2 + displayName: Stage build logs + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/log + targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: Microsoft.Spark.Binaries + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - job: SignPublish + dependsOn: + - Build + displayName: Sign and Publish Artifacts + pool: + name: NetCoreInternal-Pool + queue: buildpool.windows.10.amd64.vs2017 + + variables: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + artifactName: Microsoft.Spark.Binaries + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - task: PowerShell@2 + displayName: Sign artifacts and Package Microsoft.Spark.Worker + inputs: + filePath: eng\common\build.ps1 + arguments: -restore -sign -publish + -c $(buildConfiguration) + -ci + $(_OfficialBuildIdArgs) + /p:DotNetSignType=$(_SignType) + /p:SparkPackagesDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\BuildArtifacts\artifacts\packages + /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker + /p:SparkWorkerPackageOutputDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries' + artifactName: Microsoft.Spark.Binaries + - stage: Test displayName: Test Sources dependsOn: Build jobs: - job: Test + displayName: Test Sources pool: Hosted VS2017 variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -120,11 +185,19 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 - displayName: Copy jar files + displayName: Copy jar file of Spark 2.3.x inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/jars + sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars + contents: 'microsoft-spark-2.3.x*.jar' targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + - task: CopyFiles@2 + displayName: Copy jar file of Spark 2.4.x + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars + contents: 'microsoft-spark-2.4.x*.jar' + targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.4.x/target + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' inputs: @@ -132,20 +205,101 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.1' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.2' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.2-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.3' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.4' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.0' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.1' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.4' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.5' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.5-bin-hadoop2.7 - stage: ForwardCompatibility displayName: Test Forward Compatibility dependsOn: Build jobs: - job: ForwardCompatibility - displayName: 'E2E tests from forward compatible release v$(forwardCompatibleRelease)' + displayName: E2E tests from forward compatible release v$(forwardCompatibleRelease) pool: Hosted VS2017 variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -172,20 +326,101 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.1' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.2' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.2-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.3' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.4' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.0' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.1' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.4' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.5' + inputs: + command: test + projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.5-bin-hadoop2.7 - stage: BackwardCompatibility displayName: Test Backward Compatibility dependsOn: Build jobs: - job: BackwardCompatibility - displayName: 'E2E tests with backward compatible worker v$(backwardCompatibleRelease)' + displayName: E2E tests with backward compatible worker v$(backwardCompatibleRelease) pool: Hosted VS2017 variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\hadoop + HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) steps: @@ -196,11 +431,19 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 - displayName: Copy jar files + displayName: Copy jar file of Spark 2.3.x inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/jars + sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars + contents: 'microsoft-spark-2.3.x*.jar' targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + - task: CopyFiles@2 + displayName: Copy jar file of Spark 2.4.x + inputs: + sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars + contents: 'microsoft-spark-2.4.x*.jar' + targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.4.x/target + - task: BatchScript@1 displayName: Download backward compatible worker v$(backwardCompatibleRelease) inputs: @@ -214,4 +457,85 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\distros\spark-2.3.0-bin-hadoop2.7 \ No newline at end of file + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.1' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.2' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.2-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.3' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.3.4' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.0' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.0-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.1' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.1-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.3-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.4' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.4-bin-hadoop2.7 + + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.5' + inputs: + command: test + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' + env: + SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.5-bin-hadoop2.7 \ No newline at end of file From f5b3888a952a46482f8fd12b952d61b5515d1913 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 02:21:01 -0700 Subject: [PATCH 25/39] test: clean up --- azure-pipelines.yml | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ef68d88dc..61c740ee2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,10 +10,6 @@ variables: buildConfiguration: 'Release' _SignType: real _TeamName: DotNetSpark - MSBUILDSINGLELOADCONTEXT: 1 - # backwardCompatibleRelease/forwardCompatibleRelease is the "oldest" releases that work with the current release - backwardCompatibleRelease: '0.9.0' - forwardCompatibleRelease: '0.9.0' TestsToFilterOut: "(FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameVectorUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.BroadcastTests.TestDestroy)&\ @@ -64,7 +60,7 @@ stages: displayName: Download Spark Distros & Winutils.exe inputs: filename: script\download-spark-distros.cmd - arguments: $(Build.BinariesDirectory) + arguments: $(Build.ArtifactStagingDirectory)/SparkDistros - script: build.cmd -pack -c $(buildConfiguration) @@ -81,16 +77,10 @@ stages: projects: '**/*UnitTest/*.csproj' arguments: '--configuration $(buildConfiguration)' - - task: CopyFiles@2 - displayName: Stage Spark distros - inputs: - sourceFolder: $(Build.BinariesDirectory) - targetFolder: $(Build.ArtifactStagingDirectory)/SparkDistros - - task: CopyFiles@2 displayName: Stage Maven build jars inputs: - sourceFolder: $(Build.SourcesDirectory) + sourceFolder: $(Build.SourcesDirectory)/src/scala contents: '**/*.jar' targetFolder: $(Build.ArtifactStagingDirectory)/Jars @@ -188,14 +178,14 @@ stages: displayName: Copy jar file of Spark 2.3.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: 'microsoft-spark-2.3.x*.jar' + contents: '**/microsoft-spark-2.3.x**' targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target - task: CopyFiles@2 displayName: Copy jar file of Spark 2.4.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: 'microsoft-spark-2.4.x*.jar' + contents: '**/microsoft-spark-2.4.x**' targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.4.x/target - task: DotNetCoreCLI@2 @@ -291,6 +281,8 @@ stages: - stage: ForwardCompatibility displayName: Test Forward Compatibility dependsOn: Build + # forwardCompatibleRelease is the "oldest" releases that work with the current release + variables: forwardCompatibleRelease: '0.9.0' jobs: - job: ForwardCompatibility displayName: E2E tests from forward compatible release v$(forwardCompatibleRelease) @@ -412,6 +404,8 @@ stages: - stage: BackwardCompatibility displayName: Test Backward Compatibility dependsOn: Build + # backwardCompatibleRelease is the "oldest" releases that work with the current release + variables: backwardCompatibleRelease: '0.9.0' jobs: - job: BackwardCompatibility displayName: E2E tests with backward compatible worker v$(backwardCompatibleRelease) @@ -430,18 +424,18 @@ stages: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) - - task: CopyFiles@2 + - task: CopyFiles@2 displayName: Copy jar file of Spark 2.3.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: 'microsoft-spark-2.3.x*.jar' + contents: '**/microsoft-spark-2.3.x**' targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target - task: CopyFiles@2 displayName: Copy jar file of Spark 2.4.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: 'microsoft-spark-2.4.x*.jar' + contents: '**/microsoft-spark-2.4.x**' targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.4.x/target - task: BatchScript@1 From c7e2afad6964a3e4ad25328189edb172f024d592 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 02:31:49 -0700 Subject: [PATCH 26/39] clean up --- azure-pipelines.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 61c740ee2..730254c20 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -282,7 +282,8 @@ stages: displayName: Test Forward Compatibility dependsOn: Build # forwardCompatibleRelease is the "oldest" releases that work with the current release - variables: forwardCompatibleRelease: '0.9.0' + variables: + forwardCompatibleRelease: '0.9.0' jobs: - job: ForwardCompatibility displayName: E2E tests from forward compatible release v$(forwardCompatibleRelease) @@ -405,7 +406,8 @@ stages: displayName: Test Backward Compatibility dependsOn: Build # backwardCompatibleRelease is the "oldest" releases that work with the current release - variables: backwardCompatibleRelease: '0.9.0' + variables: + backwardCompatibleRelease: '0.9.0' jobs: - job: BackwardCompatibility displayName: E2E tests with backward compatible worker v$(backwardCompatibleRelease) From ee21dee545eec59c3e984e11f48d81a3e3d88d5c Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 02:33:23 -0700 Subject: [PATCH 27/39] indentation --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 730254c20..d7b7d455f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -426,7 +426,7 @@ stages: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) - - task: CopyFiles@2 + - task: CopyFiles@2 displayName: Copy jar file of Spark 2.3.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars From 18c8e8c35dad654727a54b4d036a5d229bdb0f8f Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 02:38:26 -0700 Subject: [PATCH 28/39] test: release variable --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d7b7d455f..1ae88eecf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,6 +10,8 @@ variables: buildConfiguration: 'Release' _SignType: real _TeamName: DotNetSpark + forwardCompatibleRelease: '0.9.0' + backwardCompatibleRelease: '0.9.0' TestsToFilterOut: "(FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameVectorUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.BroadcastTests.TestDestroy)&\ From b53e33c704aba638c812d89815948d493551d527 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 02:46:08 -0700 Subject: [PATCH 29/39] test job display name --- azure-pipelines.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1ae88eecf..8b6924125 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,6 +10,7 @@ variables: buildConfiguration: 'Release' _SignType: real _TeamName: DotNetSpark + # forwardCompatibleRelease/backwardCompatibleRelease is the "oldest" releases that work with the current release forwardCompatibleRelease: '0.9.0' backwardCompatibleRelease: '0.9.0' TestsToFilterOut: "(FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ @@ -283,15 +284,13 @@ stages: - stage: ForwardCompatibility displayName: Test Forward Compatibility dependsOn: Build - # forwardCompatibleRelease is the "oldest" releases that work with the current release - variables: - forwardCompatibleRelease: '0.9.0' jobs: - job: ForwardCompatibility displayName: E2E tests from forward compatible release v$(forwardCompatibleRelease) pool: Hosted VS2017 variables: + forwardCompatibleRelease: '0.9.0' ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop @@ -407,15 +406,13 @@ stages: - stage: BackwardCompatibility displayName: Test Backward Compatibility dependsOn: Build - # backwardCompatibleRelease is the "oldest" releases that work with the current release - variables: - backwardCompatibleRelease: '0.9.0' jobs: - job: BackwardCompatibility displayName: E2E tests with backward compatible worker v$(backwardCompatibleRelease) pool: Hosted VS2017 variables: + backwardCompatibleRelease: '0.9.0' ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop From cba2cb9c55a5feb863ad11b72d672e38e7b53064 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 03:08:39 -0700 Subject: [PATCH 30/39] clean up --- azure-pipelines.yml | 15 +++++---------- script/download-spark-distros.cmd | 9 +++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8b6924125..51f0f751a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -63,7 +63,7 @@ stages: displayName: Download Spark Distros & Winutils.exe inputs: filename: script\download-spark-distros.cmd - arguments: $(Build.ArtifactStagingDirectory)/SparkDistros + arguments: $(Build.ArtifactStagingDirectory)\SparkDistros - script: build.cmd -pack -c $(buildConfiguration) @@ -161,7 +161,6 @@ stages: dependsOn: Build jobs: - job: Test - displayName: Test Sources pool: Hosted VS2017 variables: @@ -182,14 +181,14 @@ stages: inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.3.x**' - targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + targetFolder: $(Build.SourcesDirectory)/src/scala - task: CopyFiles@2 displayName: Copy jar file of Spark 2.4.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.4.x**' - targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.4.x/target + targetFolder: $(Build.SourcesDirectory)/src/scala - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' @@ -286,11 +285,9 @@ stages: dependsOn: Build jobs: - job: ForwardCompatibility - displayName: E2E tests from forward compatible release v$(forwardCompatibleRelease) pool: Hosted VS2017 variables: - forwardCompatibleRelease: '0.9.0' ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop @@ -408,11 +405,9 @@ stages: dependsOn: Build jobs: - job: BackwardCompatibility - displayName: E2E tests with backward compatible worker v$(backwardCompatibleRelease) pool: Hosted VS2017 variables: - backwardCompatibleRelease: '0.9.0' ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop @@ -430,14 +425,14 @@ stages: inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.3.x**' - targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.3.x/target + targetFolder: $(Build.SourcesDirectory)/src/scala - task: CopyFiles@2 displayName: Copy jar file of Spark 2.4.x inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.4.x**' - targetFolder: $(Build.SourcesDirectory)/src/scala/microsoft-spark-2.4.x/target + targetFolder: $(Build.SourcesDirectory)/src/scala - task: BatchScript@1 displayName: Download backward compatible worker v$(backwardCompatibleRelease) diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index 913641739..d02bb49a7 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -14,5 +14,14 @@ cp hadoop-2.8.1\winutils.exe hadoop\bin echo "Downloading Spark distros." curl -k -L -o spark-2.3.0.tgz https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz && tar xzvf spark-2.3.0.tgz +curl -k -L -o spark-2.3.1.tgz https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzvf spark-2.3.1.tgz +curl -k -L -o spark-2.3.2.tgz https://archive.apache.org/dist/spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz && tar xzvf spark-2.3.2.tgz +curl -k -L -o spark-2.3.3.tgz https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz && tar xzvf spark-2.3.3.tgz +curl -k -L -o spark-2.3.4.tgz https://archive.apache.org/dist/spark/spark-2.3.4/spark-2.3.4-bin-hadoop2.7.tgz && tar xzvf spark-2.3.4.tgz +curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz && tar xzvf spark-2.4.0.tgz +curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz +curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz +curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz +curl -k -L -o spark-2.4.5.tgz https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz && tar xzvf spark-2.4.5.tgz endlocal \ No newline at end of file From 09bf383927d616fdf3163c560c31324ed7d851a0 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 03:40:04 -0700 Subject: [PATCH 31/39] test: spark distro dest --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 51f0f751a..c11be13e9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -65,6 +65,8 @@ stages: filename: script\download-spark-distros.cmd arguments: $(Build.ArtifactStagingDirectory)\SparkDistros + - script: dir $(Build.ArtifactStagingDirectory) + - script: build.cmd -pack -c $(buildConfiguration) -ci @@ -103,6 +105,8 @@ stages: sourceFolder: $(Build.SourcesDirectory)/artifacts/log targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log + - script: dir $(Build.ArtifactStagingDirectory) + - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' From 750a34789c5f137df35925be93e1e102a9d89722 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 04:02:07 -0700 Subject: [PATCH 32/39] add dir --- azure-pipelines.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c11be13e9..8e5c3cd9c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,14 +59,14 @@ stages: inputs: mavenPomFile: benchmark/scala/pom.xml + - script: mkdir $(Build.ArtifactStagingDirectory)\SparkDistros + - task: BatchScript@1 displayName: Download Spark Distros & Winutils.exe inputs: filename: script\download-spark-distros.cmd arguments: $(Build.ArtifactStagingDirectory)\SparkDistros - - script: dir $(Build.ArtifactStagingDirectory) - - script: build.cmd -pack -c $(buildConfiguration) -ci @@ -105,8 +105,6 @@ stages: sourceFolder: $(Build.SourcesDirectory)/artifacts/log targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log - - script: dir $(Build.ArtifactStagingDirectory) - - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' From 3a806d7ddb98f9fe6dea3c47b3f4e2cbd2dd3271 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 04:37:16 -0700 Subject: [PATCH 33/39] download spark distros --- azure-pipelines.yml | 92 +++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8e5c3cd9c..93947d943 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,14 +59,6 @@ stages: inputs: mavenPomFile: benchmark/scala/pom.xml - - script: mkdir $(Build.ArtifactStagingDirectory)\SparkDistros - - - task: BatchScript@1 - displayName: Download Spark Distros & Winutils.exe - inputs: - filename: script\download-spark-distros.cmd - arguments: $(Build.ArtifactStagingDirectory)\SparkDistros - - script: build.cmd -pack -c $(buildConfiguration) -ci @@ -168,7 +160,7 @@ stages: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -192,6 +184,12 @@ stages: contents: '**/microsoft-spark-2.4.x**' targetFolder: $(Build.SourcesDirectory)/src/scala + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' inputs: @@ -199,7 +197,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.0-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.1' @@ -208,7 +206,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.1-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.2' @@ -217,7 +215,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.2-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.3' @@ -226,7 +224,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.3-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.4' @@ -235,7 +233,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.4-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.0' @@ -244,7 +242,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.0-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.1' @@ -253,7 +251,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.1-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.3' @@ -262,7 +260,7 @@ stages: projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.3-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.4' @@ -271,7 +269,7 @@ stages: projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.4-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.5' @@ -280,7 +278,7 @@ stages: projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.5-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - stage: ForwardCompatibility displayName: Test Forward Compatibility @@ -292,7 +290,7 @@ stages: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: @@ -312,6 +310,12 @@ stages: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.0' inputs: @@ -319,7 +323,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.0-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.1' @@ -328,7 +332,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.1-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.2' @@ -337,7 +341,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.2-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.3' @@ -346,7 +350,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.3-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.4' @@ -355,7 +359,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.4-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.0' @@ -364,7 +368,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.0-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.1' @@ -373,7 +377,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.1-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.3' @@ -382,7 +386,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.3-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.4' @@ -391,7 +395,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.4-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.5' @@ -400,7 +404,7 @@ stages: projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.5-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - stage: BackwardCompatibility displayName: Test Backward Compatibility @@ -412,7 +416,7 @@ stages: variables: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - HADOOP_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\hadoop + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop DOTNET_WORKER_DIR: $(BackwardCompatibleDotnetWorkerDir) steps: @@ -436,6 +440,12 @@ stages: contents: '**/microsoft-spark-2.4.x**' targetFolder: $(Build.SourcesDirectory)/src/scala + - task: BatchScript@1 + displayName: Download Spark Distros & Winutils.exe + inputs: + filename: script\download-spark-distros.cmd + arguments: $(Build.BinariesDirectory) + - task: BatchScript@1 displayName: Download backward compatible worker v$(backwardCompatibleRelease) inputs: @@ -449,7 +459,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.0-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.1' @@ -458,7 +468,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.1-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.2' @@ -467,7 +477,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.2-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.3' @@ -476,7 +486,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.3-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.3.4' @@ -485,7 +495,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.3.4-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.0' @@ -494,7 +504,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.0-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.1' @@ -503,7 +513,7 @@ stages: projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.1-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.3' @@ -512,7 +522,7 @@ stages: projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.3-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.4' @@ -521,7 +531,7 @@ stages: projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.4-bin-hadoop2.7 + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 - task: DotNetCoreCLI@2 displayName: 'E2E tests for Spark 2.4.5' @@ -530,4 +540,4 @@ stages: projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)' env: - SPARK_HOME: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\SparkDistros\spark-2.4.5-bin-hadoop2.7 \ No newline at end of file + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 \ No newline at end of file From 18efda62df6939184fc727f67c12f0af6637bda7 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 05:04:01 -0700 Subject: [PATCH 34/39] clean up for fc --- azure-pipelines.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 93947d943..6115a0d3a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -294,15 +294,13 @@ stages: DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir) steps: - - checkout: self - path: s\master - checkout: forwardCompatibleRelease path: s\$(forwardCompatibleRelease) - task: Maven@3 displayName: 'Maven build src for forward compatible release v$(forwardCompatibleRelease)' inputs: - mavenPomFile: $(forwardCompatibleRelease)/src/scala/pom.xml + mavenPomFile: src/scala/pom.xml - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts and Binaries @@ -320,7 +318,7 @@ stages: displayName: 'E2E tests for Spark 2.3.0' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7 @@ -329,7 +327,7 @@ stages: displayName: 'E2E tests for Spark 2.3.1' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7 @@ -338,7 +336,7 @@ stages: displayName: 'E2E tests for Spark 2.3.2' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7 @@ -347,7 +345,7 @@ stages: displayName: 'E2E tests for Spark 2.3.3' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7 @@ -356,7 +354,7 @@ stages: displayName: 'E2E tests for Spark 2.3.4' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.4-bin-hadoop2.7 @@ -365,7 +363,7 @@ stages: displayName: 'E2E tests for Spark 2.4.0' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7 @@ -374,7 +372,7 @@ stages: displayName: 'E2E tests for Spark 2.4.1' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark.E2ETest/*.csproj' + projects: '**/Microsoft.Spark.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7 @@ -383,7 +381,7 @@ stages: displayName: 'E2E tests for Spark 2.4.3' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 @@ -392,7 +390,7 @@ stages: displayName: 'E2E tests for Spark 2.4.4' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.4-bin-hadoop2.7 @@ -401,7 +399,7 @@ stages: displayName: 'E2E tests for Spark 2.4.5' inputs: command: test - projects: '$(forwardCompatibleRelease)/**/Microsoft.Spark*.E2ETest/*.csproj' + projects: '**/Microsoft.Spark*.E2ETest/*.csproj' arguments: '--configuration $(buildConfiguration)' env: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 From 5459d25c83ddfae97507fcf185440eb77261a568 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 11:32:37 -0700 Subject: [PATCH 35/39] add fc build var --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6115a0d3a..ed18c2d8d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,6 +10,7 @@ variables: buildConfiguration: 'Release' _SignType: real _TeamName: DotNetSpark + MSBUILDSINGLELOADCONTEXT: 1 # forwardCompatibleRelease/backwardCompatibleRelease is the "oldest" releases that work with the current release forwardCompatibleRelease: '0.9.0' backwardCompatibleRelease: '0.9.0' From e3ff65032e51dfd4012595212fd0d167d8fd2b3e Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Tue, 2 Jun 2020 12:14:05 -0700 Subject: [PATCH 36/39] clean up display name --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ed18c2d8d..1c4303744 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -166,20 +166,20 @@ stages: steps: - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts and Binaries + displayName: Download Build Artifacts inputs: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 - displayName: Copy jar file of Spark 2.3.x + displayName: Copy microsoft-spark-2.3.x.jar inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.3.x**' targetFolder: $(Build.SourcesDirectory)/src/scala - task: CopyFiles@2 - displayName: Copy jar file of Spark 2.4.x + displayName: Copy microsoft-spark-2.4.x.jar inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.4.x**' @@ -304,7 +304,7 @@ stages: mavenPomFile: src/scala/pom.xml - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts and Binaries + displayName: Download Build Artifacts inputs: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) @@ -420,20 +420,20 @@ stages: steps: - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts and Binaries + displayName: Download Build Artifacts inputs: artifactName: Microsoft.Spark.Binaries downloadPath: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 - displayName: Copy jar file of Spark 2.3.x + displayName: Copy microsoft-spark-2.3.x.jar inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.3.x**' targetFolder: $(Build.SourcesDirectory)/src/scala - task: CopyFiles@2 - displayName: Copy jar file of Spark 2.4.x + displayName: Copy microsoft-spark-2.4.x.jar inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars contents: '**/microsoft-spark-2.4.x**' From b476ab583dc8bba943eb67417af0ef5158fe8e66 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Sat, 13 Jun 2020 01:31:11 -0700 Subject: [PATCH 37/39] resolve comments --- azure-pipelines.yml | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1c4303744..df2cb174d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -152,10 +152,10 @@ stages: artifactName: Microsoft.Spark.Binaries - stage: Test - displayName: Test Sources + displayName: E2E Tests dependsOn: Build jobs: - - job: Test + - job: Run pool: Hosted VS2017 variables: @@ -172,17 +172,10 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 - displayName: Copy microsoft-spark-2.3.x.jar - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: '**/microsoft-spark-2.3.x**' - targetFolder: $(Build.SourcesDirectory)/src/scala - - - task: CopyFiles@2 - displayName: Copy microsoft-spark-2.4.x.jar + displayName: Copy jars inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: '**/microsoft-spark-2.4.x**' + contents: '**/*.jar' targetFolder: $(Build.SourcesDirectory)/src/scala - task: BatchScript@1 @@ -282,10 +275,10 @@ stages: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - stage: ForwardCompatibility - displayName: Test Forward Compatibility + displayName: E2E Forward Compatibility Tests dependsOn: Build jobs: - - job: ForwardCompatibility + - job: Run pool: Hosted VS2017 variables: @@ -406,10 +399,10 @@ stages: SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.5-bin-hadoop2.7 - stage: BackwardCompatibility - displayName: Test Backward Compatibility + displayName: E2E Backward Compatibility Tests dependsOn: Build jobs: - - job: BackwardCompatibility + - job: Run pool: Hosted VS2017 variables: From 5b5d326deb99974af847f3f65b73c5c254873d13 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Sat, 13 Jun 2020 02:05:40 -0700 Subject: [PATCH 38/39] combine copy jars for bc --- azure-pipelines.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index df2cb174d..a97b826f2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -419,17 +419,10 @@ stages: downloadPath: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 - displayName: Copy microsoft-spark-2.3.x.jar - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: '**/microsoft-spark-2.3.x**' - targetFolder: $(Build.SourcesDirectory)/src/scala - - - task: CopyFiles@2 - displayName: Copy microsoft-spark-2.4.x.jar + displayName: Copy jars inputs: sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars - contents: '**/microsoft-spark-2.4.x**' + contents: '**/*.jar' targetFolder: $(Build.SourcesDirectory)/src/scala - task: BatchScript@1 From 8e60ba008ab97e656a5bc6429c9abd1a6203bf92 Mon Sep 17 00:00:00 2001 From: elvaliuliuliu Date: Sat, 13 Jun 2020 13:07:22 -0700 Subject: [PATCH 39/39] add ArtifactPath var --- azure-pipelines.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a97b826f2..14b6e689a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,8 @@ variables: (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.SparkSessionTests.TestCreateDataFrameWithTimestamp)" - CurrentDotnetWorkerDir: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker\netcoreapp3.1\win-x64' + ArtifactPath: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries' + CurrentDotnetWorkerDir: '$(ArtifactPath)\Microsoft.Spark.Worker\netcoreapp3.1\win-x64' BackwardCompatibleDotnetWorkerDir: $(Build.BinariesDirectory)\Microsoft.Spark.Worker-$(backwardCompatibleRelease) # Azure DevOps variables are transformed into environment variables, with these variables we @@ -142,13 +143,13 @@ stages: -ci $(_OfficialBuildIdArgs) /p:DotNetSignType=$(_SignType) - /p:SparkPackagesDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\BuildArtifacts\artifacts\packages - /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\Microsoft.Spark.Worker - /p:SparkWorkerPackageOutputDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries + /p:SparkPackagesDir=$(ArtifactPath)\BuildArtifacts\artifacts\packages + /p:SparkWorkerPublishDir=$(ArtifactPath)\Microsoft.Spark.Worker + /p:SparkWorkerPackageOutputDir=$(ArtifactPath) - task: PublishBuildArtifacts@1 inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries' + pathtoPublish: '$(ArtifactPath)' artifactName: Microsoft.Spark.Binaries - stage: Test @@ -174,7 +175,7 @@ stages: - task: CopyFiles@2 displayName: Copy jars inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars + sourceFolder: $(ArtifactPath)/Jars contents: '**/*.jar' targetFolder: $(Build.SourcesDirectory)/src/scala @@ -421,7 +422,7 @@ stages: - task: CopyFiles@2 displayName: Copy jars inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/Jars + sourceFolder: $(ArtifactPath)/Jars contents: '**/*.jar' targetFolder: $(Build.SourcesDirectory)/src/scala