[FLINK-13978][build system] Make end to end tests pass

apache · Feb 12, 2020 · 5a81004 · 5a81004
1 parent 719e8c3
commit 5a81004
Show file tree

Hide file tree

Showing 7 changed files with 118 additions and 58 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -32,7 +32,7 @@ resources:
   containers:
   # Container with Maven 3.2.5, SSL to have the same environment everywhere.
   - container: flink-build-container
-    image: rmetzger/flink-ci:ubuntu-jdk8-amd64-e005e00
+    image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
 
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
@@ -49,7 +49,7 @@ jobs:
       test_pool_definition:
         vmImage: 'ubuntu-latest'
       e2e_pool_definition:
-        vmImage: 'ubuntu-latest'
+        vmImage: 'ubuntu-16.04'
       environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
 
 

diff --git a/flink-end-to-end-tests/test-scripts/common_s3_minio.sh b/flink-end-to-end-tests/test-scripts/common_s3_minio.sh
@@ -31,6 +31,19 @@ IT_CASE_S3_BUCKET=test-data
 
 S3_TEST_DATA_WORDS_URI="s3://$IT_CASE_S3_BUCKET/words"
 
+# allow injecting a custom data dir location.
+DATA_DIR=$TEST_INFRA_DIR
+if [ ! -z "$DOCKER_TEST_INFRA_DIR" ] ; then
+  DATA_DIR=$DOCKER_TEST_INFRA_DIR
+fi
+
+_DOCKER_NETWORK=""
+_MINIO_HOST="localhost"
+if [ ! -z "$AGENT_CONTAINERNETWORK" ] ; then
+  _DOCKER_NETWORK="--network $AGENT_CONTAINERNETWORK"
+  _MINIO_HOST="minio"
+fi
+
 ###################################
 # Starts a docker container for s3 minio.
 #
@@ -43,18 +56,18 @@ S3_TEST_DATA_WORDS_URI="s3://$IT_CASE_S3_BUCKET/words"
 #   S3_ENDPOINT
 ###################################
 function s3_start {
-  echo "Spawning minio for s3 tests"
+  echo "Spawning minio for s3 tests with DATA_DIR=$DATA_DIR"
   export MINIO_CONTAINER_ID=$(docker run -d \
-    -P \
-    --mount type=bind,source="$TEST_INFRA_DIR",target=/data \
-    -e "MINIO_ACCESS_KEY=$AWS_ACCESS_KEY_ID" -e "MINIO_SECRET_KEY=$AWS_SECRET_ACCESS_KEY" -e "MINIO_DOMAIN=localhost" \
+    -P ${_DOCKER_NETWORK} --name minio -p 9000:9000 \
+    --mount type=bind,source="$DATA_DIR",target=/data \
+    -e "MINIO_ACCESS_KEY=$AWS_ACCESS_KEY_ID" -e "MINIO_SECRET_KEY=$AWS_SECRET_ACCESS_KEY" -e "MINIO_DOMAIN=${_MINIO_HOST}" \
     minio/minio \
     server \
     /data)
   while [[ "$(docker inspect -f {{.State.Running}} "$MINIO_CONTAINER_ID")" -ne "true" ]]; do
     sleep 0.1
   done
-  export S3_ENDPOINT="http://$(docker port "$MINIO_CONTAINER_ID" 9000 | sed s'/0\.0\.0\.0/localhost/')"
+  export S3_ENDPOINT="http://${_MINIO_HOST}:9000"
   echo "Started minio @ $S3_ENDPOINT"
   on_exit s3_stop
 }
@@ -66,8 +79,11 @@ function s3_start {
 #   MINIO_CONTAINER_ID
 ###################################
 function s3_stop {
+  echo "Stopping minio ..."
   docker kill "$MINIO_CONTAINER_ID"
   docker rm "$MINIO_CONTAINER_ID"
+  # remove .minio.sys folder
+  docker run --mount type=bind,source="$DATA_DIR",target=/data alpine rm -rf /data/.minio.sys
   export S3_ENDPOINT=
   export MINIO_CONTAINER_ID=
 }

diff --git a/flink-end-to-end-tests/test-scripts/common_s3_operations.sh b/flink-end-to-end-tests/test-scripts/common_s3_operations.sh
@@ -29,9 +29,19 @@
 #   AWSCLI_CONTAINER_ID
 ###################################
 function aws_cli_start() {
+  # allow injecting a custom data dir location.
+  DATA_DIR=$TEST_INFRA_DIR
+  if [ ! -z "$DOCKER_TEST_INFRA_DIR" ] ; then
+    DATA_DIR=$DOCKER_TEST_INFRA_DIR
+  fi
+  # allow injecting a custom network
+  _DOCKER_NETWORK="host"
+  if [ ! -z "$AGENT_CONTAINERNETWORK" ] ; then
+    _DOCKER_NETWORK="$AGENT_CONTAINERNETWORK"
+  fi
   export AWSCLI_CONTAINER_ID=$(docker run -d \
-    --network host \
-    --mount type=bind,source="$TEST_INFRA_DIR",target=/hostdir \
+    --network ${_DOCKER_NETWORK} \
+    --mount type=bind,source="$DATA_DIR",target=/hostdir \
     -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
     --entrypoint python \
     -it banst/awscli)

diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_kinesis.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kinesis.sh
@@ -27,13 +27,20 @@ export AWS_ACCESS_KEY_ID=flinkKinesisTestFakeAccessKeyId
 export AWS_SECRET_KEY=flinkKinesisTestFakeAccessKey
 
 KINESALITE_PORT=4567
+KINESALITE_HOST=localhost
+_DOCKER_NETWORK=
+# Allow passing a custom network for docker-in-docker scenarios
+if [ ! -z "$AGENT_CONTAINERNETWORK" ] ; then
+  _DOCKER_NETWORK="--network $AGENT_CONTAINERNETWORK"
+  KINESALITE_HOST="flink-test-kinesis"
+fi
 
 function start_kinesalite {
     #docker run -d --rm --name flink-test-kinesis -p ${KINESALITE_PORT}:${KINESALITE_PORT} instructure/kinesalite
     # override entrypoint to enable SSL
     docker run -d --rm --entrypoint "/tini" \
         --name flink-test-kinesis \
-        -p ${KINESALITE_PORT}:${KINESALITE_PORT} \
+        -p ${KINESALITE_PORT}:${KINESALITE_PORT} ${_DOCKER_NETWORK} \
         instructure/kinesalite -- \
         /usr/src/app/node_modules/kinesalite/cli.js --path /var/lib/kinesalite --ssl
 }
@@ -65,6 +72,6 @@ TEST_JAR="${END_TO_END_DIR}/flink-streaming-kinesis-test/target/KinesisExample.j
 JVM_ARGS=${DISABLE_CERT_CHECKING_JAVA_OPTS} \
 $FLINK_DIR/bin/flink run -p 1 -c org.apache.flink.streaming.kinesis.test.KinesisExampleTest $TEST_JAR \
   --input-stream test-input --output-stream test-output \
-  --aws.endpoint https://localhost:${KINESALITE_PORT} --aws.credentials.provider.basic.secretkey fakekey --aws.credentials.provider.basic.accesskeyid fakeid \
+  --aws.endpoint https://${KINESALITE_HOST}:${KINESALITE_PORT} --aws.credentials.provider.basic.secretkey fakekey --aws.credentials.provider.basic.accesskeyid fakeid \
   --flink.stream.initpos TRIM_HORIZON \
   --flink.partition-discovery.interval-millis 1000
diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml
@@ -23,26 +23,27 @@
 
 
 
-schedules:
-- cron: "0 0 * * *"
-  displayName: Daily midnight build
-  branches:
-    include:
-    - master
-  always: true # run even if there were no changes to the mentioned branches
+# to be enabled with FLINK-15834
+#schedules:
+#- cron: "0 0 * * *"
+#  displayName: Daily midnight build
+#  branches:
+#    include:
+#    - master
+#  always: true # run even if there were no changes to the mentioned branches
 
 resources:
   containers:
   # Container with Maven 3.2.5, SSL to have the same environment everywhere.
   - container: flink-build-container
-    image: rmetzger/flink-ci:ubuntu-jdk8-amd64-e005e00
+    image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
 
 
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
   MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
-  CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
-  CACHE_FALLBACK_KEY: maven | $(Agent.OS)
+  CACHE_KEY: AAmaven | $(Agent.OS) | **/pom.xml, !**/target/**
+  CACHE_FALLBACK_KEY: AAmaven | $(Agent.OS)
   CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
 
 stages:
@@ -57,38 +58,38 @@ stages:
           test_pool_definition:
             name: Default
           e2e_pool_definition:
-            vmImage: 'ubuntu-latest'
+            vmImage: 'ubuntu-16.04'
           environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
-
+  # to be enabled with FLINK-15834
   # Special stage for midnight builds:
-  - stage: cron_build_on_azure_os_free_pool
-    displayName: "Cron build on free Azure Resource Pool"
-    dependsOn: [] # depending on an empty array makes the stages run in parallel
-    condition: or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['MODE'], 'nightly'))
-    jobs:
-      - template: jobs-template.yml
-        parameters:
-          stage_name: cron_build_default
-          test_pool_definition:
-            vmImage: 'ubuntu-latest'
-          e2e_pool_definition:
-            vmImage: 'ubuntu-latest'
-          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
-      - template: jobs-template.yml
-        parameters:
-          stage_name: cron_build_scala2_12
-          test_pool_definition:
-            vmImage: 'ubuntu-latest'
-          e2e_pool_definition:
-            vmImage: 'ubuntu-latest'
-          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12 -Phive-1.2.1"
-      - template: jobs-template.yml
-        parameters:
-          stage_name: cron_build_jdk11
-          test_pool_definition:
-            vmImage: 'ubuntu-latest'
-          e2e_pool_definition:
-            vmImage: 'ubuntu-latest'
-          environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11 -Djdk11"
+  # - stage: cron_build_on_azure_os_free_pool
+  #   displayName: "Cron build on free Azure Resource Pool"
+  #   dependsOn: [] # depending on an empty array makes the stages run in parallel
+  #   condition: or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['MODE'], 'nightly'))
+  #   jobs:
+  #     - template: jobs-template.yml
+  #       parameters:
+  #         stage_name: cron_build_default
+  #         test_pool_definition:
+  #           vmImage: 'ubuntu-latest'
+  #         e2e_pool_definition:
+  #           vmImage: 'ubuntu-latest'
+  #         environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
+  #     - template: jobs-template.yml
+  #       parameters:
+  #         stage_name: cron_build_scala2_12
+  #         test_pool_definition:
+  #           vmImage: 'ubuntu-latest'
+  #         e2e_pool_definition:
+  #           vmImage: 'ubuntu-latest'
+  #         environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.12 -Phive-1.2.1"
+  #     - template: jobs-template.yml
+  #       parameters:
+  #         stage_name: cron_build_jdk11
+  #         test_pool_definition:
+  #           vmImage: 'ubuntu-latest'
+  #         e2e_pool_definition:
+  #           vmImage: 'ubuntu-latest'
+  #         environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11 -Djdk11"
 
 
diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
@@ -31,7 +31,7 @@ jobs:
   steps:
 
   # Preparation
-  - task: CacheBeta@1
+  - task: Cache@2
     inputs:
       key: $(CACHE_KEY)
       restoreKeys: $(CACHE_FALLBACK_KEY)
@@ -105,15 +105,15 @@ jobs:
 
 
 - job: e2e_${{parameters.stage_name}}
-  condition: eq(variables['MODE'], 'e2e')
+  #STUPID DEBUGGING condition: eq(variables['MODE'], 'e2e')
   # We are not running this job on a container, but in a VM.
   pool: ${{parameters.e2e_pool_definition}}
   timeoutInMinutes: 240
   cancelTimeoutInMinutes: 1
   workspace:
     clean: all
   steps:
-    - task: CacheBeta@1
+    - task: Cache@2
       inputs:
         key: $(CACHE_KEY)
         restoreKeys: $(CACHE_FALLBACK_KEY)

diff --git a/tools/azure_controller.sh b/tools/azure_controller.sh
@@ -72,10 +72,13 @@ echo "Current stage: \"$STAGE\""
 
 EXIT_CODE=0
 
+#adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html)
+# --settings /tmp/az_settings.xml 
+MVN="mvn clean install $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -U -DskipTests $PROFILE"
+
 # Run actual compile&test steps
 if [ $STAGE == "$STAGE_COMPILE" ]; then
-    #adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html)
-    MVN="mvn clean install --settings /tmp/az_settings.xml $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -DskipTests $PROFILE"
+    # run mvn clean install:
     $MVN
     EXIT_CODE=$?
 
@@ -175,10 +178,33 @@ elif [ $STAGE != "$STAGE_CLEANUP" ]; then
         echo "=============================================================================="
         echo "Python stage found. Re-compiling (this is required on Azure for the python tests to pass)"
         echo "=============================================================================="
-        mvn install -DskipTests -Drat.skip
+        # run mvn install (w/o "clean"):
+        PY_MVN="${MVN// clean/}"
+        PY_MVN="$PY_MVN -Drat.skip=true"
+        ${PY_MVN}
         echo "Done compiling ... "
     fi
+
+    echo "===== Set DOCKER_TEST_INFRA_DIR ===== "
+    #
+    # Some tests in the "run-pre-commit-tests.sh" collection launch Docker containers.
+    # Since the regular build is executed in Docker (on Azure), we'll be launching those 
+    # containers outside of the current container (on the host, alongside the build&test container).
+    # Some of these containers mount a path. Currently, these scripts mount relative to the build container,
+    # thus this path is not available on the host (where the test container is launched).
+    # 
+    # Here, we figure out the path on the host machine, and set it.
+    #
 
+    DOCKER_THIS_ID=$AGENT_CONTAINERID
+
+    # get volume mount source
+    DOCKER_VOLUME_MOUNT_SOURCE=`docker inspect  -f '{{json .Mounts }}' $DOCKER_THIS_ID | jq -r '.[] | .Source | match("(.*_work/[0-9]+)") | .string'`
+    export DOCKER_TEST_INFRA_DIR=${DOCKER_VOLUME_MOUNT_SOURCE}/s/flink-end-to-end-tests/test-scripts/
+
+    echo "DOCKER_TEST_INFRA_DIR determined as '$DOCKER_TEST_INFRA_DIR'"
+
+
     TEST="$STAGE" "./tools/travis_watchdog.sh" 300
     EXIT_CODE=$?
 elif [ $STAGE == "$STAGE_CLEANUP" ]; then