From 84fd23d82c2908192d58186d6e061c89b018cda5 Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Sun, 16 Feb 2020 14:12:35 +0100 Subject: [PATCH] [FLINK-13978][build system] Add experimental support for building on Azure Pipelines This closes #10976 --- azure-pipelines.yml | 48 +++-- flink-end-to-end-tests/run-nightly-tests.sh | 7 +- flink-end-to-end-tests/test-scripts/common.sh | 47 +---- .../test-scripts/test-runner-common.sh | 2 +- .../test_streaming_elasticsearch.sh | 25 +-- tools/azure-pipelines/build-apache-repo.yml | 50 +++++ .../google-mirror-settings.xml | 28 +++ tools/azure-pipelines/jobs-template.yml | 161 +++++++++++++++ tools/azure-pipelines/prepare_precommit.sh | 39 ++++ tools/azure-pipelines/setup_kubernetes.sh | 26 +++ tools/azure_controller.sh | 190 ++++++++++++++++++ tools/travis_watchdog.sh | 33 +-- 12 files changed, 564 insertions(+), 92 deletions(-) create mode 100644 tools/azure-pipelines/build-apache-repo.yml create mode 100644 tools/azure-pipelines/google-mirror-settings.xml create mode 100644 tools/azure-pipelines/jobs-template.yml create mode 100755 tools/azure-pipelines/prepare_precommit.sh create mode 100755 tools/azure-pipelines/setup_kubernetes.sh create mode 100755 tools/azure_controller.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cc41f084b2df3..7ffc7d0a5897f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,23 +13,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +# +# This file defines an Azure Pipeline build for testing Flink. It is intended to be used +# with a free Azure Pipelines account. +# It has the following features: +# - default builds for pushes / pull requests +# - end-to-end tests +# +# +# For the "apache/flink" repository, we are using the pipeline definition located in +# tools/azure-pipelines/build-apache-repo.yml +# That file points to custom, self-hosted build agents for faster pull request build processing and +# integration with Flinkbot. +# The custom pipeline definition file is configured in the "Pipeline settings" screen +# of the Azure Pipelines web ui. +# -trigger: - branches: - include: - - '*' resources: containers: - # Container with Maven 3.2.5 to have the same environment everywhere. + # Container with Maven 3.2.5, SSL to have the same environment everywhere. - container: flink-build-container - image: rmetzger/flink-ci:3 - repositories: - - repository: templates - type: github - name: flink-ci/flink-azure-builds - endpoint: flink-ci + image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab + +# See tools/azure-pipelines/jobs-template.yml for a short summary of the caching +variables: + MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository + MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/** + CACHE_FALLBACK_KEY: maven | $(Agent.OS) + CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache + jobs: -- template: flink-build-jobs.yml@templates + - template: tools/azure-pipelines/jobs-template.yml + parameters: # see template file for a definition of the parameters. + stage_name: ci_build + test_pool_definition: + vmImage: 'ubuntu-latest' + e2e_pool_definition: + vmImage: 'ubuntu-16.04' + environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" + + diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh index aa51819d4c38c..5907b42fb9b45 100755 --- a/flink-end-to-end-tests/run-nightly-tests.sh +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -88,8 +88,11 @@ run_test "Resuming Externalized Checkpoint after terminal failure (rocks, increm # Docker ################################################################################ -run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh" -run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs" +# Ignore these tests on Azure: In these tests, the TaskManagers are not starting on YARN, probably due to memory constraints. +if [ -z "$TF_BUILD" ] ; then + run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh" + run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs" +fi ################################################################################ # High Availability diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index 0b9d61140d76f..ed56b3bce2d35 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -247,9 +247,7 @@ function wait_rest_endpoint_up { echo "Waiting for ${endpoint_name} REST endpoint to come up..." sleep 1 done - echo "${endpoint_name} REST endpoint has not started on query url '${query_url}' within a timeout of ${TIMEOUT} sec. curl output:" - curl ${CURL_SSL_ARGS} "$query_url" - echo "Exiting ..." + echo "${endpoint_name} REST endpoint has not started within a timeout of ${TIMEOUT} sec" exit 1 } @@ -439,51 +437,16 @@ function wait_for_job_state_transition { done } -function is_job_submitted { - JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -a | grep "$1") - if [[ "$JOB_LIST_RESULT" == "" ]]; then - echo "false" - else - echo "true" - fi -} - -function wait_job_submitted { - local TIMEOUT=10 - for i in $(seq 1 ${TIMEOUT}); do - local IS_SUBMITTED=`is_job_submitted $1` - - if [[ "$IS_SUBMITTED" == "true" ]]; then - echo "Job ($1) is submitted." - return - else - echo "Job ($1) is not yet submitted." - fi - sleep 1 - done - echo "Job ($1) has not been submitted within a timeout of ${TIMEOUT} sec" - exit 1 -} - -function is_job_running { - JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -r | grep "$1") - if [[ "$JOB_LIST_RESULT" == "" ]]; then - echo "false" - else - echo "true" - fi -} - function wait_job_running { local TIMEOUT=10 for i in $(seq 1 ${TIMEOUT}); do - local IS_RUNNING=`is_job_running $1` + JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -r | grep "$1") - if [[ "$IS_RUNNING" == "true" ]]; then + if [[ "$JOB_LIST_RESULT" == "" ]]; then + echo "Job ($1) is not yet running." + else echo "Job ($1) is running." return - else - echo "Job ($1) is not yet running." fi sleep 1 done diff --git a/flink-end-to-end-tests/test-scripts/test-runner-common.sh b/flink-end-to-end-tests/test-scripts/test-runner-common.sh index 6a0840fd596a7..ecddbd9fe8ff3 100644 --- a/flink-end-to-end-tests/test-scripts/test-runner-common.sh +++ b/flink-end-to-end-tests/test-scripts/test-runner-common.sh @@ -19,7 +19,7 @@ source "${END_TO_END_DIR}"/test-scripts/common.sh -FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout) +export FLINK_VERSION=$(mvn --file ${END_TO_END_DIR}/pom.xml org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout) ####################################### # Prints the given description, runs the given test and prints how long the execution took. diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh b/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh index adabb470af79a..e2ee273d43a32 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_elasticsearch.sh @@ -39,31 +39,10 @@ on_exit test_cleanup TEST_ES_JAR=${END_TO_END_DIR}/flink-elasticsearch${ELASTICSEARCH_VERSION}-test/target/Elasticsearch${ELASTICSEARCH_VERSION}SinkExample.jar # run the Flink job -JOB_ID=$($FLINK_DIR/bin/flink run -d -p 1 $TEST_ES_JAR \ +$FLINK_DIR/bin/flink run -p 1 $TEST_ES_JAR \ --numRecords 20 \ --index index \ - --type type | awk '{print $NF}' | tail -n 1) + --type type - -# wait for 10 seconds -wait_job_submitted ${JOB_ID} - -# Wait for 60 seconds for the job to finish -MAX_RETRY_SECONDS=60 - -start_time=$(date +%s) - -RUNNING=`is_job_running ${JOB_ID}` -while [[ "$RUNNING" == "true" ]]; do - RUNNING=`is_job_running ${JOB_ID}` - current_time=$(date +%s) - time_diff=$((current_time - start_time)) - if [ $time_diff -ge $MAX_RETRY_SECONDS ]; then - echo "Job did not finish after $MAX_RETRY_SECONDS seconds. Printing logs and failing test: " - cat $FLINK_DIR/log/* - exit 1 - fi -done - # 40 index requests and 20 final update requests verify_result_line_number 60 index diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml new file mode 100644 index 0000000000000..87919f3157f01 --- /dev/null +++ b/tools/azure-pipelines/build-apache-repo.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# This file defines the Flink build for the "apache/flink" repository, including +# the following: +# - PR builds +# - custom triggered e2e tests +# - nightly builds + +resources: + containers: + # Container with Maven 3.2.5, SSL to have the same environment everywhere. + - container: flink-build-container + image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab + +variables: + MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository + MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/** + CACHE_FALLBACK_KEY: maven | $(Agent.OS) + CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache + +stages: + # CI / PR triggered stage: + - stage: ci_build + displayName: "CI Build (custom builders)" + condition: not(eq(variables['Build.Reason'], in('Schedule', 'Manual'))) + jobs: + - template: jobs-template.yml + parameters: + stage_name: ci_build + test_pool_definition: + name: Default + e2e_pool_definition: + vmImage: 'ubuntu-16.04' + environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" diff --git a/tools/azure-pipelines/google-mirror-settings.xml b/tools/azure-pipelines/google-mirror-settings.xml new file mode 100644 index 0000000000000..49a3b7133e9a0 --- /dev/null +++ b/tools/azure-pipelines/google-mirror-settings.xml @@ -0,0 +1,28 @@ + + + + + google-maven-central + GCS Maven Central mirror + https://maven-central-eu.storage-download.googleapis.com/maven2/ + central + + + diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml new file mode 100644 index 0000000000000..6a65d6ff36134 --- /dev/null +++ b/tools/azure-pipelines/jobs-template.yml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +parameters: + test_pool_definition: # defines the hardware pool for compilation and unit test execution. + e2e_pool_definion: # defines the hardware pool for end-to-end test execution + stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage) + environment: # defines environment variables for downstream scripts + +jobs: +- job: compile_${{parameters.stage_name}} + condition: not(eq(variables['MODE'], 'e2e')) + pool: ${{parameters.test_pool_definition}} + container: flink-build-container + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all # this cleans the entire workspace directory before running a new job + # It is necessary because the custom build machines are reused for tests. + # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace + + steps: + # The cache task is persisting the .m2 directory between builds, so that + # we do not have to re-download all dependencies from maven central for + # each build. The hope is that downloading the cache is faster than + # all dependencies individually. + # In this configuration, we use a hash over all committed (not generated) .pom files + # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash + # (usually because a pom file has changed), we'll fall back to a key without + # the pom files (CACHE_FALLBACK_KEY). + # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops + - task: Cache@2 + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + continueOnError: true # continue the build even if the cache fails. + displayName: Cache Maven local repo + + # Compile + - script: STAGE=compile ${{parameters.environment}} ./tools/azure_controller.sh compile + displayName: Build + + # upload artifacts for next stage + - task: PublishPipelineArtifact@1 + inputs: + path: $(CACHE_FLINK_DIR) + artifact: FlinkCompileCacheDir-${{parameters.stage_name}} + +- job: test_${{parameters.stage_name}} + dependsOn: compile_${{parameters.stage_name}} + condition: not(eq(variables['MODE'], 'e2e')) + pool: ${{parameters.test_pool_definition}} + container: flink-build-container + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all + strategy: + matrix: + core: + module: core + python: + module: python + libraries: + module: libraries + blink_planner: + module: blink_planner + connectors: + module: connectors + kafka_gelly: + module: kafka/gelly + tests: + module: tests + legacy_scheduler_core: + module: legacy_scheduler_core + legacy_scheduler_tests: + module: legacy_scheduler_tests + misc: + module: misc + steps: + + # download artifacts + - task: DownloadPipelineArtifact@2 + inputs: + path: $(CACHE_FLINK_DIR) + artifact: FlinkCompileCacheDir-${{parameters.stage_name}} + + # recreate "build-target" symlink for python tests + - script: | + ln -snf $(CACHE_FLINK_DIR)/flink-dist/target/flink-*-SNAPSHOT-bin/flink-*-SNAPSHOT $(CACHE_FLINK_DIR)/build-target + displayName: Recreate 'build-target' symlink + # Test + - script: STAGE=test ${{parameters.environment}} ./tools/azure_controller.sh $(module) + displayName: Test - $(module) + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + +- job: precommit_${{parameters.stage_name}} + dependsOn: compile_${{parameters.stage_name}} + # We are not running this job on a container, but in a VM. + pool: ${{parameters.e2e_pool_definition}} + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all + steps: + - task: Cache@2 + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + displayName: Cache Maven local repo + + # download artifacts + - task: DownloadPipelineArtifact@2 + inputs: + path: $(CACHE_FLINK_DIR) + artifact: FlinkCompileCacheDir-${{parameters.stage_name}} + - script: ./tools/travis/setup_maven.sh + - script: ./tools/azure-pipelines/prepare_precommit.sh + displayName: prepare and build Flink + - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh + displayName: Test - precommit + +- job: e2e_${{parameters.stage_name}} + condition: eq(variables['MODE'], 'e2e') + # We are not running this job on a container, but in a VM. + pool: ${{parameters.e2e_pool_definition}} + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all + steps: + - task: Cache@2 + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + displayName: Cache Maven local repo + - script: ./tools/travis/setup_maven.sh + - script: ./tools/azure-pipelines/setup_kubernetes.sh + - script: M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -De2e-metrics -Dmaven.wagon.http.pool=false" STAGE=compile ./tools/azure_controller.sh compile + displayName: Build + - script: FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh + displayName: Run nightly e2e tests + diff --git a/tools/azure-pipelines/prepare_precommit.sh b/tools/azure-pipelines/prepare_precommit.sh new file mode 100755 index 0000000000000..71a231cc90a48 --- /dev/null +++ b/tools/azure-pipelines/prepare_precommit.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +echo "Merging cache" +cp -RT "$CACHE_FLINK_DIR" "." +echo "Adjusting timestamps" +# adjust timestamps to prevent recompilation +find . -type f -name '*.java' | xargs touch +find . -type f -name '*.scala' | xargs touch +# wait a bit for better odds of different timestamps +sleep 5 +find . -type f -name '*.class' | xargs touch +find . -type f -name '*.timestamp' | xargs touch + + +export M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ +export PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH +mvn -version +mvn install --settings ./tools/azure-pipelines/google-mirror-settings.xml -DskipTests -Drat.skip + + +chmod -R +x build-target +chmod -R +x flink-end-to-end-tests diff --git a/tools/azure-pipelines/setup_kubernetes.sh b/tools/azure-pipelines/setup_kubernetes.sh new file mode 100755 index 0000000000000..19eb50ae16d0f --- /dev/null +++ b/tools/azure-pipelines/setup_kubernetes.sh @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Replace moby by docker" +docker version +sudo apt-get remove -y moby-engine +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io +docker version diff --git a/tools/azure_controller.sh b/tools/azure_controller.sh new file mode 100755 index 0000000000000..2682fcb18b6b1 --- /dev/null +++ b/tools/azure_controller.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +echo $M2_HOME +echo $PATH +echo $MAVEN_OPTS + +mvn -version +echo "Commit: $(git rev-parse HEAD)" + + + +HERE="`dirname \"$0\"`" # relative +HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized +if [ -z "$HERE" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +source "${HERE}/travis/stage.sh" +source "${HERE}/travis/shade.sh" + +print_system_info() { + echo "CPU information" + lscpu + + echo "Memory information" + cat /proc/meminfo + + echo "Disk information" + df -hH + + echo "Running build as" + whoami +} + +print_system_info + + +STAGE=$1 +echo "Current stage: \"$STAGE\"" + +EXIT_CODE=0 + +# Set up a custom Maven settings file, configuring an Google-hosted maven central +# mirror. We use a different mirror because the official maven central mirrors +# often lead to connection timeouts (probably due to rate-limiting) + +# adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html) +MVN="mvn clean install --settings ./tools/azure-pipelines/google-mirror-settings.xml $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -U -DskipTests $PROFILE" + +# Run actual compile&test steps +if [ $STAGE == "$STAGE_COMPILE" ]; then + # run mvn clean install: + $MVN + EXIT_CODE=$? + + if [ $EXIT_CODE == 0 ]; then + echo "\n\n==============================================================================\n" + echo "Checking scala suffixes\n" + echo "==============================================================================\n" + + ./tools/verify_scala_suffixes.sh "${PROFILE}" + EXIT_CODE=$? + else + echo "\n==============================================================================\n" + echo "Previous build failure detected, skipping scala-suffixes check.\n" + echo "==============================================================================\n" + fi + + if [ $EXIT_CODE == 0 ]; then + check_shaded_artifacts + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_s3_fs hadoop + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_s3_fs presto + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_connector_elasticsearch 2 + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_connector_elasticsearch 5 + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_connector_elasticsearch 6 + EXIT_CODE=$(($EXIT_CODE+$?)) + else + echo "==============================================================================" + echo "Previous build failure detected, skipping shaded dependency check." + echo "==============================================================================" + fi + + if [ $EXIT_CODE == 0 ]; then + echo "Creating cache build directory $CACHE_FLINK_DIR" + + cp -r . "$CACHE_FLINK_DIR" + + function minimizeCachedFiles() { + # reduces the size of the cached directory to speed up + # the packing&upload / download&unpacking process + # by removing files not required for subsequent stages + + # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided) + find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' \ + ! -path "$CACHE_FLINK_DIR/flink-formats/flink-csv/target/flink-csv*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-formats/flink-json/target/flink-json*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-formats/flink-avro/target/flink-avro*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-runtime/target/flink-runtime*tests.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-streaming-java/target/flink-streaming-java*tests.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-dist*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table_*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table-blink*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/opt/flink-python*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-elasticsearch-base/target/flink-*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-kafka-base/target/flink-*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-table/flink-table-planner/target/flink-table-planner*tests.jar" | xargs rm -rf + + # .git directory + # not deleting this can cause build stability issues + # merging the cached version sometimes fails + rm -rf "$CACHE_FLINK_DIR/.git" + + # AZ Pipelines has a problem with links. + rm "$CACHE_FLINK_DIR/build-target" + } + + echo "Minimizing cache" + minimizeCachedFiles + else + echo "==============================================================================" + echo "Previous build failure detected, skipping cache setup." + echo "==============================================================================" + fi +elif [ $STAGE != "$STAGE_CLEANUP" ]; then + if ! [ -e $CACHE_FLINK_DIR ]; then + echo "Cached flink dir $CACHE_FLINK_DIR does not exist. Exiting build." + exit 1 + fi + # merged compiled flink into local clone + # this prevents the cache from being re-uploaded + echo "Merging cache" + cp -RT "$CACHE_FLINK_DIR" "." + + echo "Adjusting timestamps" + # adjust timestamps to prevent recompilation + find . -type f -name '*.java' | xargs touch + find . -type f -name '*.scala' | xargs touch + # wait a bit for better odds of different timestamps + sleep 5 + find . -type f -name '*.class' | xargs touch + find . -type f -name '*.timestamp' | xargs touch + + if [ $STAGE == $STAGE_PYTHON ]; then + echo "==============================================================================" + echo "Python stage found. Re-compiling (this is required on Azure for the python tests to pass)" + echo "==============================================================================" + # run mvn install (w/o "clean"): + PY_MVN="${MVN// clean/}" + PY_MVN="$PY_MVN -Drat.skip=true" + ${PY_MVN} + echo "Done compiling ... " + fi + + + TEST="$STAGE" "./tools/travis_watchdog.sh" 300 + EXIT_CODE=$? +elif [ $STAGE == "$STAGE_CLEANUP" ]; then + echo "Cleaning up $CACHE_BUILD_DIR" + rm -rf "$CACHE_BUILD_DIR" +else + echo "Invalid Stage specified: $STAGE" + exit 1 +fi + +# Exit code for Azure build success/failure +exit $EXIT_CODE diff --git a/tools/travis_watchdog.sh b/tools/travis_watchdog.sh index 5e3e4e39e1df4..e96934c7a8791 100755 --- a/tools/travis_watchdog.sh +++ b/tools/travis_watchdog.sh @@ -95,6 +95,11 @@ UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz +if [ ! -z "$TF_BUILD" ] ; then + # set proper artifacts file name on Azure Pipelines + ARTIFACTS_FILE=${BUILD_BUILDNUMBER}.tar.gz +fi + if [ $TEST == $STAGE_PYTHON ]; then CMD=$PYTHON_TEST CMD_PID=$PYTHON_PID @@ -273,18 +278,22 @@ cd ../../ # only run end-to-end tests in misc because we only have flink-dist here case $TEST in (misc) - if [ $EXIT_CODE == 0 ]; then - echo "\n\n==============================================================================\n" - echo "Running bash end-to-end tests\n" - echo "==============================================================================\n" - - FLINK_DIR=build-target flink-end-to-end-tests/run-pre-commit-tests.sh - - EXIT_CODE=$? - else - echo "\n==============================================================================\n" - echo "Previous build failure detected, skipping bash end-to-end tests.\n" - echo "==============================================================================\n" + # If we are not on Azure (we are on Travis) run precommit tests in misc stage. + # On Azure, we run them in a separate job + if [ -z "$TF_BUILD" ] ; then + if [ $EXIT_CODE == 0 ]; then + echo "\n\n==============================================================================\n" + echo "Running bash end-to-end tests\n" + echo "==============================================================================\n" + + FLINK_DIR=build-target flink-end-to-end-tests/run-pre-commit-tests.sh + + EXIT_CODE=$? + else + echo "\n==============================================================================\n" + echo "Previous build failure detected, skipping bash end-to-end tests.\n" + echo "==============================================================================\n" + fi fi if [ $EXIT_CODE == 0 ]; then echo "\n\n==============================================================================\n"