GEODE-5864 - Refactor and parameterize CI env creation

Makes it easier for developers to deploy their own CI instances for testing of code and CI itself. * Use concourse-pipeline-resource to deploy instead of hand-rolling `fly` scripts * Add pararmeters for fork/branch/repo name * Rename all pipelines to be consistent * Rename all pipelines to contain fork/branch information * Reorganize artifact storage to include fork/branch information * `deploy_meta.sh` auto-starts Concourse tasks in required bootstrap order * Instance images names are created consistently and are unique * Require fewer VAULT secrets * Improve behavior of packer on windows GCP instances * Check to see if a previous iteration of start_instance created an instance and kill it before trying to create a new one. * Ensure we stop instances that cannot be reached via SSH * Add ssh server keep alives to prevent firewall timeouts Co-authored-by: Sean Goller <sgoller@pivotal.io> Co-authored-by: Patrick Rhomberg <prhomberg@pivotal.io> Co-authored-by: Robert Houghton <rhoughton@pivotal.io> Co-authored-by: Jacob Barrett <jbarrett@pivotal.io>
apache · Oct 15, 2018 · afdc9cb · afdc9cb
1 parent 4e8ea1c
commit afdc9cb
Show file tree

Hide file tree

Showing 31 changed files with 494 additions and 381 deletions.
diff --git a/README.md b/README.md
@@ -241,3 +241,4 @@ The following provides more details on the included cryptographic software:
   The [JCE Unlimited Strength Jurisdiction Policy](https://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
   may need to be installed separately to use keystore passwords with 7 or more characters.
 * Apache Geode links to and uses [OpenSSL](https://www.openssl.org/) ciphers.
+
diff --git a/ci/images/alpine-tools/Dockerfile b/ci/images/alpine-tools/Dockerfile
@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM alpine:3.7
+FROM alpine:latest
 
 COPY --from=google/cloud-sdk:alpine /google-cloud-sdk /google-cloud-sdk
 COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer
@@ -31,6 +31,7 @@ RUN apk --no-cache add \
       python \
       py2-pip \
       rsync \
+      util-linux \
   && gcloud config set core/disable_usage_reporting true \
   && gcloud config set component_manager/disable_update_check true \
   && gcloud config set metrics/environment github_docker_image \

diff --git a/ci/images/google-geode-builder/build_image.sh b/ci/images/google-geode-builder/build_image.sh
@@ -24,30 +24,38 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-pushd ${SCRIPTDIR}
-
-. ${SCRIPTDIR}/../../pipelines/shared/utilities.sh
 
+pushd ${SCRIPTDIR}
 
-IMAGE_FAMILY_PREFIX=""
-GEODE_DOCKER_IMAGE=${GEODE_DOCKER_IMAGE:-"gcr.io/apachegeode-ci/test-container"}
-if [[ -z "${GEODE_FORK}" ]]; then
-  echo "GEODE_FORK environment variable must be set for this script to work."
-  exit 1
+if [[ -n "${CONCOURSE_GCP_KEY}" ]]; then
+  dd of=credentials.json <<< "${CONCOURSE_GCP_KEY}"
+  export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPTDIR}/credentials.json
 fi
 
-GEODE_BRANCH=${GEODE_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}
-
-SANITIZED_GEODE_BRANCH=$(getSanitizedBranch ${GEODE_BRANCH})
-SANITIZED_GEODE_FORK=$(getSanitizedFork ${GEODE_FORK})
-
-if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
-  IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
+GCP_NETWORK="default"
+GCP_SUBNETWORK="default"
+
+MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
+if [[ -n "${MY_NAME}" ]]; then
+  MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
+  MY_ZONE=${MY_ZONE##*/}
+  NETWORK_INTERFACE_INFO="$(gcloud compute instances describe ${MY_NAME} --zone ${MY_ZONE} --format="json(networkInterfaces)")"
+  GCP_NETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].network')
+  GCP_NETWORK=${GCP_NETWORK##*/}
+  GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
+  GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
 fi
 
+HASHED_PIPELINE_PREFIX="i$(uuidgen -n @dns -s -N "${PIPELINE_PREFIX}")-"
+
 echo "Running packer"
-packer build \
+PACKER_LOG=1 packer build \
   --var "geode_docker_image=${GEODE_DOCKER_IMAGE}" \
-  --var "image_family_prefix=${IMAGE_FAMILY_PREFIX}" \
+  --var "pipeline_prefix=${PIPELINE_PREFIX}" \
+  --var "hashed_pipeline_prefix=${HASHED_PIPELINE_PREFIX}" \
+  --var "gcp_project=${GCP_PROJECT}" \
   --var "java_build_version=${JAVA_BUILD_VERSION}" \
+  --var "gcp_network=${GCP_NETWORK}" \
+  --var "gcp_subnetwork=${GCP_SUBNETWORK}" \
+  --var "use_internal_ip=true" \
   packer.json
diff --git a/ci/images/google-geode-builder/packer.json b/ci/images/google-geode-builder/packer.json
@@ -1,7 +1,9 @@
 {
   "variables": {
-    "image_family_prefix": "local-testing-",
-    "geode_docker_image": "gcr.io/apachegeode-ci/test-container",
+    "pipeline_prefix": "local-testing-",
+    "hashed_pipeline_prefix": "0b7ad56c-671b-5244-9f36-01f92928abf8",
+    "geode_docker_image": "unset",
+    "gcp_project": "unset",
     "java_build_version": "8"
   },
   "provisioners": [
@@ -29,12 +31,13 @@
   "builders": [
     {
       "type": "googlecompute",
-      "project_id": "apachegeode-ci",
+      "project_id": "{{user `gcp_project`}}",
       "source_image_family": "debian-9",
       "ssh_username": "packer",
       "zone": "us-central1-f",
-      "image_family": "{{user `image_family_prefix`}}geode-builder",
-      "image_name": "{{user `image_family_prefix`}}geode-builder-{{timestamp}}"
+      "image_family": "{{user `pipeline_prefix`}}geode-builder",
+      "image_name": "{{user `hashed_pipeline_prefix`}}gb-{{timestamp}}",
+      "tags": ["packer"]
     }
   ]
 }
diff --git a/ci/images/google-windows-geode-builder/build_image.sh b/ci/images/google-windows-geode-builder/build_image.sh
@@ -15,6 +15,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+set -x
 
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
@@ -26,21 +27,35 @@ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 pushd ${SCRIPTDIR}
 
-IMAGE_FAMILY_PREFIX=""
-if [[ -z "${GEODE_FORK}" ]]; then
-  echo "GEODE_FORK environment variable must be set for this script to work."
-  exit 1
+if [[ -n "${CONCOURSE_GCP_KEY}" ]]; then
+  dd of=credentials.json <<< "${CONCOURSE_GCP_KEY}"
+  export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPTDIR}/credentials.json
 fi
 
-GEODE_BRANCH=${GEODE_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}
-SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]' '[:lower:]' | cut -c 1-20)
-SANITIZED_GEODE_FORK=$(echo ${GEODE_FORK} | tr "/" "-" | tr '[:upper:]' '[:lower:]' | cut -c 1-16)
+GCP_NETWORK="default"
+GCP_SUBNETWORK="default"
 
-if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
-  IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
+MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
+if [[ -n "${MY_NAME}" ]]; then
+  MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
+  MY_ZONE=${MY_ZONE##*/}
+  NETWORK_INTERFACE_INFO="$(gcloud compute instances describe ${MY_NAME} --zone ${MY_ZONE} --format="json(networkInterfaces)")"
+  GCP_NETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].network')
+  GCP_NETWORK=${GCP_NETWORK##*/}
+  GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
+  GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
 fi
 
+HASHED_PIPELINE_PREFIX="i$(uuidgen -n @dns -s -N "${PIPELINE_PREFIX}")-"
+
 echo "Running packer"
-packer build \
-  --var "image_family_prefix=${IMAGE_FAMILY_PREFIX}" \
+PACKER_LOG=1 packer build \
+  --var "geode_docker_image=${GEODE_DOCKER_IMAGE}" \
+  --var "pipeline_prefix=${PIPELINE_PREFIX}" \
+  --var "hashed_pipeline_prefix=${HASHED_PIPELINE_PREFIX}" \
+  --var "java_build_version=${JAVA_BUILD_VERSION}" \
+  --var "gcp_project=${GCP_PROJECT}" \
+  --var "gcp_network=${GCP_NETWORK}" \
+  --var "gcp_subnetwork=${GCP_SUBNETWORK}" \
+  --var "use_internal_ip=true" \
   windows-packer.json
diff --git a/ci/images/google-windows-geode-builder/windows-packer.json b/ci/images/google-windows-geode-builder/windows-packer.json
@@ -1,27 +1,37 @@
 {
   "variables": {
-    "image_family_prefix": "local-testing-",
-    "geode_docker_image": "gcr.io/apachegeode-ci/test-container"
+    "pipeline_prefix": "local-testing-",
+    "hashed_pipeline_prefix": "0b7ad56c-671b-5244-9f36-01f92928abf8",
+    "geode_docker_image": "unset",
+    "gcp_project": "unset",
+    "gcp_network": "default",
+    "gcp_subnetwork": "default",
+    "use_internal_ip": "false",
+    "java_build_version": "8"
   },
   "builders": [
     {
-      "image_name": "{{user `image_family_prefix`}}windows-geode-builder-{{timestamp}}",
-      "image_family": "{{user `image_family_prefix`}}windows-geode-builder",
+      "image_name": "{{user `hashed_pipeline_prefix`}}wgb-{{timestamp}}",
+      "image_family": "{{user `pipeline_prefix`}}windows-geode-builder",
       "type": "googlecompute",
-      "project_id": "apachegeode-ci",
-      "source_image": "windows-server-1709-dc-core-for-containers-v20180814",
+      "project_id": "{{user `gcp_project`}}",
+      "network": "{{user `gcp_network`}}",
+      "subnetwork": "{{user `gcp_subnetwork`}}",
+      "source_image_family": "windows-1709-core-for-containers",
       "disk_size": "100",
       "machine_type": "n1-standard-1",
       "communicator": "winrm",
       "winrm_username": "geode",
       "winrm_insecure": true,
       "winrm_use_ssl": true,
+      "state_timeout": "10m",
       "metadata": {
         "windows-startup-script-cmd": "winrm quickconfig -quiet & net user /add geode & net localgroup administrators geode /add & winrm set winrm/config/service/auth @{Basic=\"true\"}"
       },
       "zone": "us-central1-a",
+      "tags": ["packer"],
       "omit_external_ip": false,
-      "use_internal_ip": false
+      "use_internal_ip": "{{user `use_internal_ip`}}"
     }
   ],
   "provisioners": [

diff --git a/ci/images/meta-mini/Dockerfile b/ci/images/meta-mini/Dockerfile
@@ -13,14 +13,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM alpine:3.7
+FROM alpine:latest
 
+COPY --from=google/cloud-sdk:alpine /google-cloud-sdk /google-cloud-sdk
+ENV PATH /google-cloud-sdk/bin:$PATH
 RUN apk --no-cache add \
       bash \
       curl \
       jq \
       python3 \
-  && curl -L -o /usr/local/bin/fly "https://concourse.apachegeode-ci.info/api/v1/cli?arch=amd64&platform=linux" \
   && pip3 install --upgrade pip \
-  && pip3 install Jinja2 pyYAML \
-  && chmod +x /usr/local/bin/* \
+  && pip3 install Jinja2 pyYAML
diff --git a/ci/pipelines/examples/jinja.template.yml b/ci/pipelines/examples/jinja.template.yml
@@ -45,7 +45,7 @@ resources:
   source:
     username: ((!docker-username))
     password: ((!docker-password))
-    repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
+    repository: gcr.io/((gcp-project))/((!docker-image-prefix))((!docker-image-name))
     tag: latest
 - name: 24h
   type: time
@@ -78,7 +78,7 @@ jobs:
       params:
         MAINTENANCE_VERSION: {{repository.branch}}
         SERVICE_ACCOUNT: ((!concourse-gcp-account))
-        PUBLIC_BUCKET: ((!public-bucket))
+        ARTIFACT_BUCKET: ((artifact-bucket))
       run:
         args:
         path: geode-ci/ci/scripts/build-examples.sh
diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh
@@ -23,17 +23,9 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
   [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
 SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+GEODEBUILDDIR="${SCRIPTDIR}/../geode-build"
 
-for cmd in Jinja2 PyYAML; do
-  if ! [[ $(pip3 list |grep ${cmd}) ]]; then
-    echo "${cmd} must be installed for pipeline deployment to work."
-    echo " 'pip3 install ${cmd}'"
-    echo ""
-    exit 1
-  fi
-done
-
-set -ex
+set -e
 
 if [ -z "${GEODE_BRANCH}" ]; then
   GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
@@ -44,47 +36,39 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
   exit 1
 fi
 
-. ${SCRIPTDIR}/../shared/utilities.sh
-
-SANITIZED_GEODE_BRANCH=$(getSanitizedBranch ${GEODE_BRANCH})
-SANITIZED_GEODE_FORK=$(getSanitizedFork ${GEODE_FORK})
 
-BIN_DIR=${OUTPUT_DIRECTORY}/bin
-TMP_DIR=${OUTPUT_DIRECTORY}/tmp
-mkdir -p ${BIN_DIR} ${TMP_DIR}
-curl -o ${BIN_DIR}/fly "https://concourse.apachegeode-ci.info/api/v1/cli?arch=amd64&platform=linux"
-chmod +x ${BIN_DIR}/fly
+echo "Sanitized Geode Fork = ${SANITIZED_GEODE_FORK}"
+echo "Sanitized Goede Branch = ${SANITIZED_GEODE_BRANCH}"
 
-PATH=${PATH}:${BIN_DIR}
-
-TARGET="geode"
-
-if [[ "${SANITIZED_GEODE_FORK}" == "apache" ]]; then
-  PIPELINE_NAME=${SANITIZED_GEODE_BRANCH}
-  DOCKER_IMAGE_PREFIX=""
-else
-  PIPELINE_NAME="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}"
-  DOCKER_IMAGE_PREFIX="${PIPELINE_NAME}-"
-fi
+MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
+MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
+MY_ZONE=${MY_ZONE##*/}
+NETWORK_INTERFACE_INFO="$(gcloud compute instances describe ${MY_NAME} --zone ${MY_ZONE} --format="json(networkInterfaces)")"
+GCP_NETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].network')
+GCP_NETWORK=${GCP_NETWORK##*/}
+GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
+GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
+ENV_ID=$(echo ${GCP_NETWORK} | awk -F- '{ print $1}')
+VERSION_BUCKET="concourse-${ENV_ID}-version"
 
+#echo "DEBUG INFO *****************************"
+#echo "Pipeline prefix = ${PIPELINE_PREFIX}"
+#echo "Docker image prefix = ${DOCKER_IMAGE_PREFIX}"]
 pushd ${SCRIPTDIR} 2>&1 > /dev/null
-  # Template and output share a directory with this script, but variables are shared in the parent directory.
-  python3 ../render.py $(basename ${SCRIPTDIR}) || exit 1
-
-  grep -n . generated-pipeline.yml
+  python3 ../render.py $(basename ${SCRIPTDIR}) ${GEODE_FORK} ${GEODE_BRANCH} ${UPSTREAM_FORK} || exit 1
+popd 2>&1 > /dev/null
+cp ${SCRIPTDIR}/generated-pipeline.yml ${OUTPUT_DIRECTORY}/generated-pipeline.yml
 
-  fly login -t ${TARGET} \
-            -c https://concourse.apachegeode-ci.info \
-            -u ${CONCOURSE_USERNAME} \
-            -p ${CONCOURSE_PASSWORD}
+cat > ${OUTPUT_DIRECTORY}/pipeline-vars.yml <<YML
+geode-build-branch: ${GEODE_BRANCH}
+geode-fork: ${GEODE_FORK}
+geode-repo-name: ${GEODE_REPO_NAME}
+upstream-fork: ${UPSTREAM_FORK}
+pipeline-prefix: "${PIPELINE_PREFIX}"
+public-pipelines: ${PUBLIC_PIPELINES}
+gcp-project: ${GCP_PROJECT}
+version-bucket: ${VERSION_BUCKET}
+artifact-bucket: ${ARTIFACT_BUCKET}
+YML
 
-  fly -t ${TARGET} set-pipeline \
-    --non-interactive \
-    --pipeline ${PIPELINE_NAME} \
-    --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
-    --config ${SCRIPTDIR}/generated-pipeline.yml
 
-  if [[ "${SANITIZED_GEODE_FORK}" == "apache" ]]; then
-    fly -t ${TARGET} expose-pipeline -p ${PIPELINE_NAME}
-  fi
-popd 2>&1 > /dev/null