From 5a81004ac5c7461921fa679bb1a428c155ae4a25 Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Tue, 11 Feb 2020 17:00:01 +0100 Subject: [PATCH] [FLINK-13978][build system] Make end to end tests pass --- azure-pipelines.yml | 4 +- .../test-scripts/common_s3_minio.sh | 26 ++++-- .../test-scripts/common_s3_operations.sh | 14 +++- .../test-scripts/test_streaming_kinesis.sh | 11 ++- tools/azure-pipelines/build-apache-repo.yml | 83 ++++++++++--------- tools/azure-pipelines/jobs-template.yml | 6 +- tools/azure_controller.sh | 32 ++++++- 7 files changed, 118 insertions(+), 58 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 81833459ce1a08..4781a4e828b2e6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,7 +32,7 @@ resources: containers: # Container with Maven 3.2.5, SSL to have the same environment everywhere. - container: flink-build-container - image: rmetzger/flink-ci:ubuntu-jdk8-amd64-e005e00 + image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab variables: MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository @@ -49,7 +49,7 @@ jobs: test_pool_definition: vmImage: 'ubuntu-latest' e2e_pool_definition: - vmImage: 'ubuntu-latest' + vmImage: 'ubuntu-16.04' environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" diff --git a/flink-end-to-end-tests/test-scripts/common_s3_minio.sh b/flink-end-to-end-tests/test-scripts/common_s3_minio.sh index a56ce4e9410d3d..81219a8ceb03be 100644 --- a/flink-end-to-end-tests/test-scripts/common_s3_minio.sh +++ b/flink-end-to-end-tests/test-scripts/common_s3_minio.sh @@ -31,6 +31,19 @@ IT_CASE_S3_BUCKET=test-data S3_TEST_DATA_WORDS_URI="s3://$IT_CASE_S3_BUCKET/words" +# allow injecting a custom data dir location. +DATA_DIR=$TEST_INFRA_DIR +if [ ! -z "$DOCKER_TEST_INFRA_DIR" ] ; then + DATA_DIR=$DOCKER_TEST_INFRA_DIR +fi + +_DOCKER_NETWORK="" +_MINIO_HOST="localhost" +if [ ! -z "$AGENT_CONTAINERNETWORK" ] ; then + _DOCKER_NETWORK="--network $AGENT_CONTAINERNETWORK" + _MINIO_HOST="minio" +fi + ################################### # Starts a docker container for s3 minio. # @@ -43,18 +56,18 @@ S3_TEST_DATA_WORDS_URI="s3://$IT_CASE_S3_BUCKET/words" # S3_ENDPOINT ################################### function s3_start { - echo "Spawning minio for s3 tests" + echo "Spawning minio for s3 tests with DATA_DIR=$DATA_DIR" export MINIO_CONTAINER_ID=$(docker run -d \ - -P \ - --mount type=bind,source="$TEST_INFRA_DIR",target=/data \ - -e "MINIO_ACCESS_KEY=$AWS_ACCESS_KEY_ID" -e "MINIO_SECRET_KEY=$AWS_SECRET_ACCESS_KEY" -e "MINIO_DOMAIN=localhost" \ + -P ${_DOCKER_NETWORK} --name minio -p 9000:9000 \ + --mount type=bind,source="$DATA_DIR",target=/data \ + -e "MINIO_ACCESS_KEY=$AWS_ACCESS_KEY_ID" -e "MINIO_SECRET_KEY=$AWS_SECRET_ACCESS_KEY" -e "MINIO_DOMAIN=${_MINIO_HOST}" \ minio/minio \ server \ /data) while [[ "$(docker inspect -f {{.State.Running}} "$MINIO_CONTAINER_ID")" -ne "true" ]]; do sleep 0.1 done - export S3_ENDPOINT="http://$(docker port "$MINIO_CONTAINER_ID" 9000 | sed s'/0\.0\.0\.0/localhost/')" + export S3_ENDPOINT="http://${_MINIO_HOST}:9000" echo "Started minio @ $S3_ENDPOINT" on_exit s3_stop } @@ -66,8 +79,11 @@ function s3_start { # MINIO_CONTAINER_ID ################################### function s3_stop { + echo "Stopping minio ..." docker kill "$MINIO_CONTAINER_ID" docker rm "$MINIO_CONTAINER_ID" + # remove .minio.sys folder + docker run --mount type=bind,source="$DATA_DIR",target=/data alpine rm -rf /data/.minio.sys export S3_ENDPOINT= export MINIO_CONTAINER_ID= } diff --git a/flink-end-to-end-tests/test-scripts/common_s3_operations.sh b/flink-end-to-end-tests/test-scripts/common_s3_operations.sh index ad4f9964378b28..3a76558bd22c3e 100644 --- a/flink-end-to-end-tests/test-scripts/common_s3_operations.sh +++ b/flink-end-to-end-tests/test-scripts/common_s3_operations.sh @@ -29,9 +29,19 @@ # AWSCLI_CONTAINER_ID ################################### function aws_cli_start() { + # allow injecting a custom data dir location. + DATA_DIR=$TEST_INFRA_DIR + if [ ! -z "$DOCKER_TEST_INFRA_DIR" ] ; then + DATA_DIR=$DOCKER_TEST_INFRA_DIR + fi + # allow injecting a custom network + _DOCKER_NETWORK="host" + if [ ! -z "$AGENT_CONTAINERNETWORK" ] ; then + _DOCKER_NETWORK="$AGENT_CONTAINERNETWORK" + fi export AWSCLI_CONTAINER_ID=$(docker run -d \ - --network host \ - --mount type=bind,source="$TEST_INFRA_DIR",target=/hostdir \ + --network ${_DOCKER_NETWORK} \ + --mount type=bind,source="$DATA_DIR",target=/hostdir \ -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ --entrypoint python \ -it banst/awscli) diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kinesis.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kinesis.sh index 08416a03117402..1fd8f4c891962c 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_kinesis.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_kinesis.sh @@ -27,13 +27,20 @@ export AWS_ACCESS_KEY_ID=flinkKinesisTestFakeAccessKeyId export AWS_SECRET_KEY=flinkKinesisTestFakeAccessKey KINESALITE_PORT=4567 +KINESALITE_HOST=localhost +_DOCKER_NETWORK= +# Allow passing a custom network for docker-in-docker scenarios +if [ ! -z "$AGENT_CONTAINERNETWORK" ] ; then + _DOCKER_NETWORK="--network $AGENT_CONTAINERNETWORK" + KINESALITE_HOST="flink-test-kinesis" +fi function start_kinesalite { #docker run -d --rm --name flink-test-kinesis -p ${KINESALITE_PORT}:${KINESALITE_PORT} instructure/kinesalite # override entrypoint to enable SSL docker run -d --rm --entrypoint "/tini" \ --name flink-test-kinesis \ - -p ${KINESALITE_PORT}:${KINESALITE_PORT} \ + -p ${KINESALITE_PORT}:${KINESALITE_PORT} ${_DOCKER_NETWORK} \ instructure/kinesalite -- \ /usr/src/app/node_modules/kinesalite/cli.js --path /var/lib/kinesalite --ssl } @@ -65,6 +72,6 @@ TEST_JAR="${END_TO_END_DIR}/flink-streaming-kinesis-test/target/KinesisExample.j JVM_ARGS=${DISABLE_CERT_CHECKING_JAVA_OPTS} \ $FLINK_DIR/bin/flink run -p 1 -c org.apache.flink.streaming.kinesis.test.KinesisExampleTest $TEST_JAR \ --input-stream test-input --output-stream test-output \ - --aws.endpoint https://localhost:${KINESALITE_PORT} --aws.credentials.provider.basic.secretkey fakekey --aws.credentials.provider.basic.accesskeyid fakeid \ + --aws.endpoint https://${KINESALITE_HOST}:${KINESALITE_PORT} --aws.credentials.provider.basic.secretkey fakekey --aws.credentials.provider.basic.accesskeyid fakeid \ --flink.stream.initpos TRIM_HORIZON \ --flink.partition-discovery.interval-millis 1000 diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml index 699a88ae7d9cc9..02f41386462a96 100644 --- a/tools/azure-pipelines/build-apache-repo.yml +++ b/tools/azure-pipelines/build-apache-repo.yml @@ -23,26 +23,27 @@ -schedules: -- cron: "0 0 * * *" - displayName: Daily midnight build - branches: - include: - - master - always: true # run even if there were no changes to the mentioned branches +# to be enabled with FLINK-15834 +#schedules: +#- cron: "0 0 * * *" +# displayName: Daily midnight build +# branches: +# include: +# - master +# always: true # run even if there were no changes to the mentioned branches resources: containers: # Container with Maven 3.2.5, SSL to have the same environment everywhere. - container: flink-build-container - image: rmetzger/flink-ci:ubuntu-jdk8-amd64-e005e00 + image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab variables: MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' - CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/** - CACHE_FALLBACK_KEY: maven | $(Agent.OS) + CACHE_KEY: AAmaven | $(Agent.OS) | **/pom.xml, !**/target/** + CACHE_FALLBACK_KEY: AAmaven | $(Agent.OS) CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache stages: @@ -57,38 +58,38 @@ stages: test_pool_definition: name: Default e2e_pool_definition: - vmImage: 'ubuntu-latest' + vmImage: 'ubuntu-16.04' environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" - + # to be enabled with FLINK-15834 # Special stage for midnight builds: - - stage: cron_build_on_azure_os_free_pool - displayName: "Cron build on free Azure Resource Pool" - dependsOn: [] # depending on an empty array makes the stages run in parallel - condition: or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['MODE'], 'nightly')) - jobs: - - template: jobs-template.yml - parameters: - stage_name: cron_build_default - test_pool_definition: - vmImage: 'ubuntu-latest' - e2e_pool_definition: - vmImage: 'ubuntu-latest' - environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" - - template: jobs-template.yml - parameters: - stage_name: cron_build_scala2_12 - test_pool_definition: - vmImage: 'ubuntu-latest' - e2e_pool_definition: - vmImage: 'ubuntu-latest' - environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12 -Phive-1.2.1" - - template: jobs-template.yml - parameters: - stage_name: cron_build_jdk11 - test_pool_definition: - vmImage: 'ubuntu-latest' - e2e_pool_definition: - vmImage: 'ubuntu-latest' - environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11 -Djdk11" + # - stage: cron_build_on_azure_os_free_pool + # displayName: "Cron build on free Azure Resource Pool" + # dependsOn: [] # depending on an empty array makes the stages run in parallel + # condition: or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['MODE'], 'nightly')) + # jobs: + # - template: jobs-template.yml + # parameters: + # stage_name: cron_build_default + # test_pool_definition: + # vmImage: 'ubuntu-latest' + # e2e_pool_definition: + # vmImage: 'ubuntu-latest' + # environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" + # - template: jobs-template.yml + # parameters: + # stage_name: cron_build_scala2_12 + # test_pool_definition: + # vmImage: 'ubuntu-latest' + # e2e_pool_definition: + # vmImage: 'ubuntu-latest' + # environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12 -Phive-1.2.1" + # - template: jobs-template.yml + # parameters: + # stage_name: cron_build_jdk11 + # test_pool_definition: + # vmImage: 'ubuntu-latest' + # e2e_pool_definition: + # vmImage: 'ubuntu-latest' + # environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11 -Djdk11" diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index 51e076d6d5b536..fe5621787d2ea5 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -31,7 +31,7 @@ jobs: steps: # Preparation - - task: CacheBeta@1 + - task: Cache@2 inputs: key: $(CACHE_KEY) restoreKeys: $(CACHE_FALLBACK_KEY) @@ -105,7 +105,7 @@ jobs: - job: e2e_${{parameters.stage_name}} - condition: eq(variables['MODE'], 'e2e') + #STUPID DEBUGGING condition: eq(variables['MODE'], 'e2e') # We are not running this job on a container, but in a VM. pool: ${{parameters.e2e_pool_definition}} timeoutInMinutes: 240 @@ -113,7 +113,7 @@ jobs: workspace: clean: all steps: - - task: CacheBeta@1 + - task: Cache@2 inputs: key: $(CACHE_KEY) restoreKeys: $(CACHE_FALLBACK_KEY) diff --git a/tools/azure_controller.sh b/tools/azure_controller.sh index 2badb6f5863d38..0984b8a5a7a5f0 100755 --- a/tools/azure_controller.sh +++ b/tools/azure_controller.sh @@ -72,10 +72,13 @@ echo "Current stage: \"$STAGE\"" EXIT_CODE=0 +#adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html) +# --settings /tmp/az_settings.xml +MVN="mvn clean install $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -U -DskipTests $PROFILE" + # Run actual compile&test steps if [ $STAGE == "$STAGE_COMPILE" ]; then - #adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html) - MVN="mvn clean install --settings /tmp/az_settings.xml $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -DskipTests $PROFILE" + # run mvn clean install: $MVN EXIT_CODE=$? @@ -175,10 +178,33 @@ elif [ $STAGE != "$STAGE_CLEANUP" ]; then echo "==============================================================================" echo "Python stage found. Re-compiling (this is required on Azure for the python tests to pass)" echo "==============================================================================" - mvn install -DskipTests -Drat.skip + # run mvn install (w/o "clean"): + PY_MVN="${MVN// clean/}" + PY_MVN="$PY_MVN -Drat.skip=true" + ${PY_MVN} echo "Done compiling ... " fi + + echo "===== Set DOCKER_TEST_INFRA_DIR ===== " + # + # Some tests in the "run-pre-commit-tests.sh" collection launch Docker containers. + # Since the regular build is executed in Docker (on Azure), we'll be launching those + # containers outside of the current container (on the host, alongside the build&test container). + # Some of these containers mount a path. Currently, these scripts mount relative to the build container, + # thus this path is not available on the host (where the test container is launched). + # + # Here, we figure out the path on the host machine, and set it. + # + DOCKER_THIS_ID=$AGENT_CONTAINERID + + # get volume mount source + DOCKER_VOLUME_MOUNT_SOURCE=`docker inspect -f '{{json .Mounts }}' $DOCKER_THIS_ID | jq -r '.[] | .Source | match("(.*_work/[0-9]+)") | .string'` + export DOCKER_TEST_INFRA_DIR=${DOCKER_VOLUME_MOUNT_SOURCE}/s/flink-end-to-end-tests/test-scripts/ + + echo "DOCKER_TEST_INFRA_DIR determined as '$DOCKER_TEST_INFRA_DIR'" + + TEST="$STAGE" "./tools/travis_watchdog.sh" 300 EXIT_CODE=$? elif [ $STAGE == "$STAGE_CLEANUP" ]; then