[FLINK-13978][build system] Add experimental support for building on Azure Pipelines

azure-pipelines.yml

@@ -13,23 +13,47 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
+# This file defines an Azure Pipeline build for testing Flink. It is intended to be used
+# with a free Azure Pipelines account.
+# It has the following features:
+#  - default builds for pushes / pull requests
+#  - end-to-end tests
+#
+#
+# For the "apache/flink" repository, we are using the pipeline definition located in
+#   tools/azure-pipelines/build-apache-repo.yml
+# That file points to custom, self-hosted build agents for faster pull request build processing and 
+# integration with Flinkbot.
+# The custom pipeline definition file is configured in the "Pipeline settings" screen
+# of the Azure Pipelines web ui.
+#
 
-trigger:
-  branches:
-    include:
-    - '*' 
 
 resources:
   containers:
-  # Container with Maven 3.2.5 to have the same environment everywhere.
+  # Container with Maven 3.2.5, SSL to have the same environment everywhere.
   - container: flink-build-container
-    image: rmetzger/flink-ci:3
-  repositories:
-    - repository: templates
-      type: github
-      name: flink-ci/flink-azure-builds
-      endpoint: flink-ci
+    image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
+
+# See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
+variables:
+  MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
+  MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+  CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
+  CACHE_FALLBACK_KEY: maven | $(Agent.OS)
+  CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
+
 
 jobs:
-- template: flink-build-jobs.yml@templates
+  - template: tools/azure-pipelines/jobs-template.yml
+    parameters: # see template file for a definition of the parameters.
+      stage_name: ci_build
+      test_pool_definition:
+        vmImage: 'ubuntu-latest'
+      e2e_pool_definition:
+        vmImage: 'ubuntu-16.04'
+      environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+
+
 

flink-end-to-end-tests/run-nightly-tests.sh

@@ -88,8 +88,11 @@ run_test "Resuming Externalized Checkpoint after terminal failure (rocks, increm
 # Docker
 ################################################################################
 
-run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
-run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
+# Ignore these tests on Azure: In these tests, the TaskManagers are not starting on YARN, probably due to memory constraints.
+if [ -z "$TF_BUILD" ] ; then
+	run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
+	run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
+fi
 
 ################################################################################
 # High Availability

flink-end-to-end-tests/test-scripts/common_kubernetes.sh

@@ -30,12 +30,14 @@ RESULT_HASH="e682ec6622b5e83f2eb614617d5ab2cf"
 function setup_kubernetes_for_linux {
     # Download kubectl, which is a requirement for using minikube.
     if ! [ -x "$(command -v kubectl)" ]; then
+        echo "Installing kubectl ..."
         local version=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
         curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/amd64/kubectl && \
             chmod +x kubectl && sudo mv kubectl /usr/local/bin/
     fi
     # Download minikube.
     if ! [ -x "$(command -v minikube)" ]; then
+        echo "Installing minikube ..."
         curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && \
             chmod +x minikube && sudo mv minikube /usr/local/bin/
     fi
@@ -48,9 +50,10 @@ function check_kubernetes_status {
 
 function start_kubernetes_if_not_running {
     if ! check_kubernetes_status; then
+        echo "Starting minikube ..."
         start_command="minikube start"
         # We need sudo permission to set vm-driver to none in linux os.
-        [[ "${OS_TYPE}" = "linux" ]] && start_command="sudo ${start_command} --vm-driver=none"
+        [[ "${OS_TYPE}" = "linux" ]] && start_command="sudo CHANGE_MINIKUBE_NONE_USER=true ${start_command} --vm-driver=none"
         ${start_command}
         # Fix the kubectl context, as it's often stale.
         minikube update-context
@@ -70,6 +73,7 @@ function start_kubernetes {
 }
 
 function stop_kubernetes {
+    echo "Stopping minikube ..."
     stop_command="minikube stop"
     [[ "${OS_TYPE}" = "linux" ]] && stop_command="sudo ${stop_command}"
     if ! retry_times ${MINIKUBE_START_RETRIES} ${MINIKUBE_START_BACKOFF} "${stop_command}"; then

flink-end-to-end-tests/test-scripts/test-runner-common.sh

@@ -19,7 +19,7 @@
 
 source "${END_TO_END_DIR}"/test-scripts/common.sh
 
-FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
+export FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
 
 #######################################
 # Prints the given description, runs the given test and prints how long the execution took.

flink-end-to-end-tests/test-scripts/test_mesos_multiple_submissions.sh

@@ -29,7 +29,7 @@ TEST_PROGRAM_JAR=$END_TO_END_DIR/flink-cli-test/target/PeriodicStreamingJob.jar
 
 function submit_job {
     local output_path=$1
-    docker exec -it mesos-master bash -c "${FLINK_DIR}/bin/flink run -d -p 1 ${TEST_PROGRAM_JAR} --durationInSecond ${DURATION} --outputPath ${output_path}" \
+    docker exec mesos-master bash -c "${FLINK_DIR}/bin/flink run -d -p 1 ${TEST_PROGRAM_JAR} --durationInSecond ${DURATION} --outputPath ${output_path}" \
         | grep "Job has been submitted with JobID" | sed 's/.* //g' | tr -d '\r'
 }
 

flink-end-to-end-tests/test-scripts/test_mesos_wordcount.sh

@@ -31,6 +31,6 @@ mkdir -p "${TEST_DATA_DIR}"
 
 start_flink_cluster_with_mesos
 
-docker exec -it mesos-master nohup bash -c "${FLINK_DIR}/bin/flink run -p 1 ${TEST_PROGRAM_JAR} ${INPUT_ARGS} --output ${OUTPUT_LOCATION}"
+docker exec mesos-master nohup bash -c "${FLINK_DIR}/bin/flink run -p 1 ${TEST_PROGRAM_JAR} ${INPUT_ARGS} --output ${OUTPUT_LOCATION}"
 
 check_result_hash "Mesos WordCount test" "${OUTPUT_LOCATION}" "${RESULT_HASH}"

tools/azure-pipelines/build-apache-repo.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# This file defines the Flink build for the "apache/flink" repository, including
+# the following:
+#  - PR builds
+#  - custom triggered e2e tests
+#  - nightly builds
+
+resources:
+  containers:
+  # Container with Maven 3.2.5, SSL to have the same environment everywhere.
+  - container: flink-build-container
+    image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
+
+variables:
+  MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
+  MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
+  CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
+  CACHE_FALLBACK_KEY: maven | $(Agent.OS)
+  CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
+
+stages:
+  # CI / PR triggered stage:
+  - stage: ci_build
+    displayName: "CI Build (custom builders)"
+    condition: not(eq(variables['Build.Reason'], in('Schedule', 'Manual')))
+    jobs:
+      - template: jobs-template.yml
+        parameters:
+          stage_name: ci_build
+          test_pool_definition:
+            name: Default
+          e2e_pool_definition:
+            vmImage: 'ubuntu-16.04'
+          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"

tools/azure-pipelines/google-mirror-settings.xml

@@ -0,0 +1,28 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<settings>
+  <mirrors>
+    <mirror>
+      <id>google-maven-central</id>
+      <name>GCS Maven Central mirror</name>
+      <url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
+      <mirrorOf>central</mirrorOf>
+    </mirror>
+  </mirrors>
+</settings>

tools/azure-pipelines/jobs-template.yml

@@ -0,0 +1,161 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+parameters:
+  test_pool_definition: # defines the hardware pool for compilation and unit test execution.
+  e2e_pool_definion: # defines the hardware pool for end-to-end test execution
+  stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage)
+  environment: # defines environment variables for downstream scripts
+
+jobs:
+- job: compile_${{parameters.stage_name}}
+  condition: not(eq(variables['MODE'], 'e2e'))
+  pool: ${{parameters.test_pool_definition}}
+  container: flink-build-container
+  timeoutInMinutes: 240
+  cancelTimeoutInMinutes: 1
+  workspace:
+    clean: all # this cleans the entire workspace directory before running a new job
+    # It is necessary because the custom build machines are reused for tests.
+    # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace 
+
+  steps:
+  # The cache task is persisting the .m2 directory between builds, so that
+  # we do not have to re-download all dependencies from maven central for 
+  # each build. The hope is that downloading the cache is faster than
+  # all dependencies individually.
+  # In this configuration, we use a hash over all committed (not generated) .pom files 
+  # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash
+  # (usually because a pom file has changed), we'll fall back to a key without
+  # the pom files (CACHE_FALLBACK_KEY).
+  # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops
+  - task: Cache@2
+    inputs:
+      key: $(CACHE_KEY)
+      restoreKeys: $(CACHE_FALLBACK_KEY)
+      path: $(MAVEN_CACHE_FOLDER)
+    continueOnError: true # continue the build even if the cache fails.
+    displayName: Cache Maven local repo
+
+  # Compile
+  - script: STAGE=compile ${{parameters.environment}} ./tools/azure_controller.sh compile
+    displayName: Build
+
+  # upload artifacts for next stage
+  - task: PublishPipelineArtifact@1
+    inputs:
+      path: $(CACHE_FLINK_DIR)
+      artifact: FlinkCompileCacheDir-${{parameters.stage_name}}
+
+- job: test_${{parameters.stage_name}}
+  dependsOn: compile_${{parameters.stage_name}}
+  condition: not(eq(variables['MODE'], 'e2e'))
+  pool: ${{parameters.test_pool_definition}}
+  container: flink-build-container
+  timeoutInMinutes: 240
+  cancelTimeoutInMinutes: 1
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      core:
+        module: core
+      python:
+        module: python
+      libraries:
+        module: libraries
+      blink_planner:
+        module: blink_planner
+      connectors:
+        module: connectors
+      kafka_gelly:
+        module: kafka/gelly
+      tests:
+        module: tests
+      legacy_scheduler_core:
+        module: legacy_scheduler_core
+      legacy_scheduler_tests:
+        module: legacy_scheduler_tests
+      misc:
+        module: misc
+  steps:
+
+  # download artifacts
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      path: $(CACHE_FLINK_DIR)
+      artifact: FlinkCompileCacheDir-${{parameters.stage_name}}
+
+  # recreate "build-target" symlink for python tests
+  - script: |
+      ln -snf $(CACHE_FLINK_DIR)/flink-dist/target/flink-*-SNAPSHOT-bin/flink-*-SNAPSHOT $(CACHE_FLINK_DIR)/build-target
+    displayName: Recreate 'build-target' symlink
+  # Test
+  - script: STAGE=test ${{parameters.environment}} ./tools/azure_controller.sh $(module)
+    displayName: Test - $(module)
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: 'JUnit'
+
+- job: precommit_${{parameters.stage_name}}
+  dependsOn: compile_${{parameters.stage_name}}
+  # We are not running this job on a container, but in a VM.
+  pool: ${{parameters.e2e_pool_definition}}
+  timeoutInMinutes: 240
+  cancelTimeoutInMinutes: 1
+  workspace:
+    clean: all
+  steps:
+    - task: Cache@2
+      inputs:
+        key: $(CACHE_KEY)
+        restoreKeys: $(CACHE_FALLBACK_KEY)
+        path: $(MAVEN_CACHE_FOLDER)
+      displayName: Cache Maven local repo
+
+    # download artifacts
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        path: $(CACHE_FLINK_DIR)
+        artifact: FlinkCompileCacheDir-${{parameters.stage_name}}
+    - script: ./tools/travis/setup_maven.sh
+    - script: ./tools/azure-pipelines/prepare_precommit.sh
+      displayName: prepare and build Flink
+    - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh
+      displayName: Test - precommit 
+
+- job: e2e_${{parameters.stage_name}}
+  condition: eq(variables['MODE'], 'e2e')
+  # We are not running this job on a container, but in a VM.
+  pool: ${{parameters.e2e_pool_definition}}
+  timeoutInMinutes: 240
+  cancelTimeoutInMinutes: 1
+  workspace:
+    clean: all
+  steps:
+    - task: Cache@2
+      inputs:
+        key: $(CACHE_KEY)
+        restoreKeys: $(CACHE_FALLBACK_KEY)
+        path: $(MAVEN_CACHE_FOLDER)
+      displayName: Cache Maven local repo
+    - script: ./tools/travis/setup_maven.sh
+    - script: ./tools/azure-pipelines/setup_kubernetes.sh
+    - script: M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -De2e-metrics -Dmaven.wagon.http.pool=false" STAGE=compile ./tools/azure_controller.sh compile
+      displayName: Build
+    - script: FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh
+      displayName: Run nightly e2e tests
+