address PR comments

azure-pipelines.yml

@@ -17,14 +17,16 @@
 # This file defines an Azure Pipeline build for testing Flink. It is intended to be used
 # with a free Azure Pipelines account.
 # It has the following features:
-#  - default builds for pushes / pull requests to a Flink fork and custom AZP account
-#  - end2end tests
+#  - default builds for pushes / pull requests
+#  - end-to-end tests
 #
 #
 # For the "apache/flink" repository, we are using the pipeline definition located in
 #   tools/azure-pipelines/build-apache-repo.yml
 # That file points to custom, self-hosted build agents for faster pull request build processing and 
 # integration with Flinkbot.
+# The custom pipeline definition file is configured in the "Pipeline settings" screen
+# of the Azure Pipelines web ui.
 #
 
 
@@ -34,6 +36,7 @@ resources:
   - container: flink-build-container
     image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
 
+# See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
   MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
@@ -44,7 +47,7 @@ variables:
 
 jobs:
   - template: tools/azure-pipelines/jobs-template.yml
-    parameters:
+    parameters: # see template file for a definition of the parameters.
       stage_name: ci_build
       test_pool_definition:
         vmImage: 'ubuntu-latest'

tools/azure-pipelines/build-apache-repo.yml

@@ -26,10 +26,11 @@
 # to be enabled with FLINK-15834
 #schedules:
 #- cron: "0 0 * * *"
-#  displayName: Daily midnight build
+#  displayName: nightly build
 #  branches:
 #    include:
 #    - master
+#    - release-*
 #  always: true # run even if there were no changes to the mentioned branches
 
 resources:
@@ -38,12 +39,11 @@ resources:
   - container: flink-build-container
     image: rmetzger/flink-ci:ubuntu-jdk8-amd64-2a765ab
 
-
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
   MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
-  CACHE_KEY: AAmaven | $(Agent.OS) | **/pom.xml, !**/target/**
-  CACHE_FALLBACK_KEY: AAmaven | $(Agent.OS)
+  CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
+  CACHE_FALLBACK_KEY: maven | $(Agent.OS)
   CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
 
 stages:
@@ -61,7 +61,7 @@ stages:
             vmImage: 'ubuntu-16.04'
           environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11"
   # to be enabled with FLINK-15834
-  # Special stage for midnight builds:
+  # Special stage for nightly builds:
   # - stage: cron_build_on_azure_os_free_pool
   #   displayName: "Cron build on free Azure Resource Pool"
   #   dependsOn: [] # depending on an empty array makes the stages run in parallel

tools/azure-pipelines/jobs-template.yml

@@ -14,10 +14,10 @@
 # limitations under the License.
 
 parameters:
-  test_pool_definition: # where is compiliation and unit test execution happening?
-  e2e_pool_definion: # where is e2e test execution happening?
-  stage_name: # needed to make job names unique if they are included multiple times
-  environment: # used to pass environment variables into the downstream scripts
+  test_pool_definition: # defines the hardware pool for compilation and unit test execution.
+  e2e_pool_definion: # defines the hardware pool for end-to-end test execution
+  stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage)
+  environment: # defines environment variables for downstream scripts
 
 jobs:
 - job: compile_${{parameters.stage_name}}
@@ -27,16 +27,25 @@ jobs:
   timeoutInMinutes: 240
   cancelTimeoutInMinutes: 1
   workspace:
-    clean: all
-  steps:
+    clean: all # this cleans the entire workspace directory before running a new job
+    # It is necessary because the custom build machines are reused for tests.
+    # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace 
 
-  # Preparation
+  steps:
+  # The cache task is persisting the .m2 directory between builds, so that
+  # we do not have to re-download all dependencies from maven central for 
+  # each build. The hope is that downloading the cache is faster than
+  # all dependencies individually.
+  # In this configuration, we a hash over all committed (not generated) .pom files 
+  # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash
+  # (usually because a pom file has changed), we'll fall back to a key without
+  # the pom files (CACHE_FALLBACK_KEY).
+  # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops
   - task: Cache@2
     inputs:
       key: $(CACHE_KEY)
       restoreKeys: $(CACHE_FALLBACK_KEY)
       path: $(MAVEN_CACHE_FOLDER)
-      cacheHitVar: CACHE_RESTORED
     continueOnError: true # continue the build even if the cache fails.
     displayName: Cache Maven local repo
 
@@ -118,7 +127,6 @@ jobs:
         key: $(CACHE_KEY)
         restoreKeys: $(CACHE_FALLBACK_KEY)
         path: $(MAVEN_CACHE_FOLDER)
-        cacheHitVar: CACHE_RESTORED
       displayName: Cache Maven local repo
     - script: ./tools/travis/setup_maven.sh
     - script: ./tools/azure-pipelines/setup_kubernetes.sh

tools/azure_controller.sh

@@ -24,7 +24,8 @@ echo $MAVEN_OPTS
 mvn -version
 echo "Commit: $(git rev-parse HEAD)"
 
-
+# Set up a custom Maven settings file, configuring an Google-hosted maven central
+# mirror.
 cat << EOF > /tmp/az_settings.xml
 <settings>
   <mirrors>