diff --git a/azure-pipelines-e2e-tests-template.yml b/azure-pipelines-e2e-tests-template.yml index f7f841ede..2f3f5e685 100644 --- a/azure-pipelines-e2e-tests-template.yml +++ b/azure-pipelines-e2e-tests-template.yml @@ -18,9 +18,21 @@ stages: jobs: - ${{ each option in test.jobOptions }}: - job: Run_${{ replace(option.pool, ' ', '_') }} - pool: ${{ option.pool }} + ${{ if eq(lower(option.pool), 'windows') }}: + pool: Hosted VS2017 + ${{ else }}: + pool: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + packageType: sdk + version: 3.1.x + installationPath: $(Agent.ToolsDirectory)/dotnet + - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: @@ -72,7 +84,7 @@ stages: script: | echo "Download Hadoop utils for Windows." curl -k -L -o hadoop.zip https://github.com/steveloughran/winutils/releases/download/tag_2017-08-29-hadoop-2.8.1-native/hadoop-2.8.1.zip - unzip hadoop.zip + Expand-Archive -Path hadoop.zip -Destination . New-Item -ItemType Directory -Force -Path hadoop\bin cp hadoop-2.8.1\winutils.exe hadoop\bin diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ba1242e4a..79e70a117 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -203,131 +203,131 @@ stages: tests: - version: '2.4.0' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.1' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.3' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.4' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.5' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.6' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.7' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '2.4.8' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '3.0.0' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0) - version: '3.0.1' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0) - version: '3.0.2' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0) - version: '3.1.1' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1) - version: '3.1.2' jobOptions: - - pool: 'Hosted VS2017' + - pool: 'Windows' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) - - pool: 'Hosted Ubuntu 1604' + - pool: 'Linux' testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1) diff --git a/src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs b/src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs index 2182c5dcc..f91a2139b 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs @@ -84,6 +84,7 @@ public SparkFixture() }; _process.Start(); + _process.BeginErrorReadLine(); _process.BeginOutputReadLine(); bool processExited = false; @@ -192,6 +193,12 @@ private void BuildSparkCmd(out string filename, out string args) Path.Combine(_tempDirectory.Path, "spark-warehouse")).AbsoluteUri; string warehouseDir = $"--conf spark.sql.warehouse.dir={warehouseUri}"; + // Spark24 < 2.4.8, Spark30 < 3.0.3 and Spark31 < 3.1.2 use bintray as the repository + // service for spark-packages. As of May 1st, 2021 bintray has been sunset and is no + // longer available. Specify additional remote repositories to search for the maven + // coordinates given with --packages. + string repositories = "--repositories https://repos.spark-packages.org/"; + string extraArgs = Environment.GetEnvironmentVariable( EnvironmentVariableNames.ExtraSparkSubmitArgs) ?? ""; @@ -205,7 +212,8 @@ private void BuildSparkCmd(out string filename, out string args) string logOption = "--conf spark.driver.extraJavaOptions=-Dlog4j.configuration=" + $"{resourceUri}/log4j.properties"; - args = $"{logOption} {warehouseDir} {AddPackages(extraArgs)} {classArg} --master local {jar} debug"; + args = $"{logOption} {warehouseDir} {AddPackages(extraArgs)} {repositories} {classArg} " + + $"--master local {jar} debug"; } private string GetJarPrefix()