ci: use azp for api and go-control-plane sync (envoyproxy#13550)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

.azure-pipelines/pipelines.yml

@@ -118,6 +118,8 @@ jobs:
   - job: filter_example
     displayName: "filter-example sync"
     dependsOn: []
+    pool:
+      vmImage: "ubuntu-18.04"
     condition: and(succeeded(), eq(variables['PostSubmit'], true))
     steps:
       - task: InstallSSHKey@0
@@ -133,6 +135,50 @@ jobs:
         env:
           AZP_BRANCH: $(Build.SourceBranch)
 
+  - job: api
+    displayName: "data-plane-api sync"
+    dependsOn: []
+    condition: and(succeeded(), eq(variables['PostSubmit'], true))
+    pool:
+      vmImage: "ubuntu-18.04"
+    steps:
+      - task: InstallSSHKey@0
+        inputs:
+          hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+          sshPublicKey: "$(DataPlaneApiPublicKey)"
+          sshPassphrase: "$(SshDeployKeyPassphrase)"
+          sshKeySecureFile: "$(DataPlaneApiPrivateKey)"
+
+      - bash: ci/api_mirror.sh
+        displayName: "Sync data-plane-api"
+        workingDirectory: $(Build.SourcesDirectory)
+        env:
+          AZP_BRANCH: $(Build.SourceBranch)
+
+  - job: go_control_plane
+    displayName: "go-control-plane sync"
+    dependsOn: []
+    condition: and(succeeded(), eq(variables['PostSubmit'], true))
+    steps:
+      - task: InstallSSHKey@0
+        inputs:
+          hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+          sshPublicKey: "$(GoControlPlanePublicKey)"
+          sshPassphrase: "$(SshDeployKeyPassphrase)"
+          sshKeySecureFile: "$(GoControlPlanePrivateKey)"
+
+      - bash: |
+          cp -a ~/.ssh $(Build.StagingDirectory)/
+          ci/run_envoy_docker.sh 'ci/go_mirror.sh'
+        displayName: "Sync go-control-plane"
+        workingDirectory: $(Build.SourcesDirectory)
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+          BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          AZP_BRANCH: $(Build.SourceBranch)
+
   - job: bazel
     displayName: "Linux-x64"
     dependsOn: ["release"]
@@ -141,6 +187,8 @@ jobs:
     strategy:
       maxParallel: 3
       matrix:
+        api:
+          CI_TARGET: "bazel.api"
         gcc:
           CI_TARGET: "bazel.gcc"
         clang_tidy:

ci/api_mirror.sh

@@ -3,16 +3,15 @@
 set -e
 
 CHECKOUT_DIR=../data-plane-api
+MAIN_BRANCH="refs/heads/master"
+API_MAIN_BRANCH="master"
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
   echo "Cloning..."
-  git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR"
+  git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}"
 
-  git -C "$CHECKOUT_DIR" config user.name "data-plane-api(CircleCI)"
+  git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)"
   git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com
-  git -C "$CHECKOUT_DIR" fetch
-  git -C "$CHECKOUT_DIR" checkout -B master origin/master
 
   # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api
   MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy"
@@ -40,6 +39,6 @@ then
   done
 
   echo "Pushing..."
-  git -C "$CHECKOUT_DIR" push origin master
+  git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}"
   echo "Done"
 fi

ci/do_ci.sh

@@ -312,19 +312,21 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
   collect_build_profile build
   exit 0
 elif [[ "$CI_TARGET" == "bazel.api" ]]; then
+  # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib,
+  # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild.
+  ENVOY_STDLIB="libstdc++"
   setup_clang_toolchain
+  export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config
   echo "Validating API structure..."
   ./tools/api/validate_structure.py
+  echo "Testing API and API Boosting..."
+  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \
+    @envoy_api_canonical//tools:tap2pcap_test @envoy_dev//clang_tools/api_booster/...
   echo "Building API..."
   bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//envoy/...
-  echo "Testing API..."
-  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \
-    @envoy_api_canonical//tools:tap2pcap_test
-  echo "Testing API boosting (unit tests)..."
-  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_dev//clang_tools/api_booster/...
   echo "Testing API boosting (golden C++ tests)..."
   # We use custom BAZEL_BUILD_OPTIONS here; the API booster isn't capable of working with libc++ yet.
-  LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config BAZEL_BUILD_OPTIONS="--config=clang" python3.8 ./tools/api_boost/api_boost_test.py
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" python3.8 ./tools/api_boost/api_boost_test.py
   exit 0
 elif [[ "$CI_TARGET" == "bazel.coverage" || "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then
   setup_clang_toolchain

ci/go_mirror.sh

@@ -2,7 +2,11 @@
 
 set -e
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
-  tools/api/generate_go_protobuf.py
+MAIN_BRANCH="refs/heads/master"
+
+# shellcheck source=ci/setup_cache.sh
+. "$(dirname "$0")"/setup_cache.sh
+
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py
 fi

ci/run_envoy_docker.sh

@@ -56,6 +56,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
 
 [[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=("-it")
 [[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=(-v "$(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)")
+[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+=(-v "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" -e SSH_AUTH_SOCK)
 
 export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}"
 

tools/api/generate_go_protobuf.py

@@ -4,17 +4,21 @@
 from subprocess import check_call
 import glob
 import os
+import shlex
 import shutil
 import sys
 import re
 
+# Needed for CI to pass down bazel options.
+BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTIONS', ''))
+
 TARGETS = '@envoy_api//...'
 IMPORT_BASE = 'github.com/envoyproxy/go-control-plane'
 OUTPUT_BASE = 'build_go'
 REPO_BASE = 'go-control-plane'
 BRANCH = 'master'
 MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ '
-USER_NAME = 'go-control-plane(CircleCI)'
+USER_NAME = 'go-control-plane(Azure Pipelines)'
 USER_EMAIL = 'go-control-plane@users.noreply.github.com'
 
 
@@ -32,7 +36,7 @@ def generateProtobufs(output):
   check_call([
       'bazel', 'build', '-c', 'fastbuild',
       '--experimental_proto_descriptor_sets_include_source_info'
-  ] + go_protos)
+  ] + BAZEL_BUILD_OPTIONS + go_protos)
 
   for rule in go_protos:
     # Example rule:
@@ -67,9 +71,7 @@ def git(repo, *args):
 
 def cloneGoProtobufs(repo):
   # Create a local clone of go-control-plane
-  git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo)
-  git(repo, 'fetch')
-  git(repo, 'checkout', '-B', BRANCH, 'origin/master')
+  git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH)
 
 
 def findLastSyncSHA(repo):

tools/api_boost/api_boost.py

@@ -132,7 +132,6 @@ def ApiBoostTree(target_paths,
     sp.run([
         'bazel',
         'build',
-        '--config=libc++',
         '--strip=always',
     ] + BAZEL_BUILD_OPTIONS + dep_lib_build_targets,
            check=True)