Merge pull request #1169 from EOSIO/merge-release-1.8.x-to-master

Merge release/1.8.x to master for v1.8.1

.cicd/create-docker-from-binary.sh

@@ -1,53 +1,70 @@
 #!/bin/bash
 echo '--- :evergreen_tree: Configuring Environment'
 set -euo pipefail
-
+. ./.cicd/helpers/sanitize.sh
 buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' .
-echo ":done: download successful"
-
-SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g')
-SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_')
-echo "$SANITIZED_BRANCH"
-echo "$SANITIZED_TAG"
-
-# do docker build
-echo '+++ :docker: Building Container'
-echo ":docker::build: Building image..."
+SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")"
+echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'."
+SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")"
+[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'."
+# docker build
+echo '+++ :docker: Build Docker Container'
 DOCKERHUB_REGISTRY="docker.io/eosio/eosio.cdt"
-
-BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest}
-DOCKER_BUILD_GEN="docker build -t eosio_cdt_image:$BUILD_TAG -f ./docker/dockerfile ."
-echo "$ $DOCKER_BUILD_GEN"
-eval $DOCKER_BUILD_GEN
-
-#tag and push on each destination AWS & DOCKERHUB
-echo '+++ :arrow_up: Pushing Container'
-EOSIO_REGS=("$EOSIO_CDT_REGISTRY" "$DOCKERHUB_REGISTRY")
-for REG in ${EOSIO_REGS[@]}; do
-    DOCKER_TAG_COMMIT="docker tag eosio_cdt_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT"
-    DOCKER_TAG_BRANCH="docker tag eosio_cdt_image:$BUILD_TAG $REG:$SANITIZED_BRANCH"
-    echo -e "$ Tagging Images: \n$DOCKER_TAG_COMMIT \n$DOCKER_TAG_BRANCH"
-    eval $DOCKER_TAG_COMMIT 
+IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}"
+DOCKER_BUILD="docker build -t '$IMAGE' -f ./docker/dockerfile ."
+echo "$ $DOCKER_BUILD"
+eval $DOCKER_BUILD
+# docker tag
+echo '--- :label: Tag Container'
+if [[ "$BUILDKITE_PIPELINE_SLUG" =~ "-sec" ]] ; then
+    REGISTRIES=("$EOSIO_CDT_REGISTRY")
+else
+    REGISTRIES=("$EOSIO_CDT_REGISTRY" "$DOCKERHUB_REGISTRY")
+fi
+for REG in ${REGISTRIES[@]}; do
+    DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'"
+    echo "$ $DOCKER_TAG_BRANCH"
     eval $DOCKER_TAG_BRANCH
-    DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT"
-    DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH"
-    echo -e "$ Pushing Images: \n$DOCKER_PUSH_COMMIT \n$DOCKER_PUSH_BRANCH"
-    eval $DOCKER_PUSH_COMMIT 
+    DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'"
+    echo "$ $DOCKER_TAG_COMMIT"
+    eval $DOCKER_TAG_COMMIT
+    if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then
+        DOCKER_TAG="docker tag '$IMAGE' '$REG:$SANITIZED_TAG'"
+        echo "$ $DOCKER_TAG"
+        eval $DOCKER_TAG
+    fi
+done
+# docker push
+echo '--- :arrow_up: Push Container'
+for REG in ${REGISTRIES[@]}; do
+    DOCKER_PUSH_BRANCH="docker push '$REG:$SANITIZED_BRANCH'"
+    echo "$ $DOCKER_PUSH_BRANCH"
     eval $DOCKER_PUSH_BRANCH
-    CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT"
-    CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH"
-    echo -e "Cleaning Up: \n$CLEAN_IMAGE_COMMIT \n$CLEAN_IMAGE_BRANCH$"
-    eval $CLEAN_IMAGE_COMMIT 
+    DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'"
+    echo "$ $DOCKER_PUSH_COMMIT"
+    eval $DOCKER_PUSH_COMMIT
+    if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then
+        DOCKER_PUSH_TAG="docker push '$REG:$SANITIZED_TAG'"
+        echo "$ $DOCKER_PUSH_TAG"
+        eval $DOCKER_PUSH_TAG
+    fi
+done
+# docker rmi
+echo '--- :put_litter_in_its_place: Cleanup'
+for REG in ${REGISTRIES[@]}; do
+    CLEAN_IMAGE_BRANCH="docker rmi '$REG:$SANITIZED_BRANCH' || :"
+    echo "$ $CLEAN_IMAGE_BRANCH"
     eval $CLEAN_IMAGE_BRANCH
-    if [[ ! -z "$SANITIZED_TAG" ]]; then
-        DOCKER_TAG="docker tag eosio_cdt_image $REG:$SANITIZED_TAG"
-        DOCKER_REM="docker rmi $REG:$SANITIZED_TAG"
-        echo -e "$ \n Tagging Image: \n$DOCKER_TAG \n Cleaning Up: \n$DOCKER_REM"
-        eval $DOCKER_TAG 
-        eval $DOCKER_REM
+    CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :"
+    echo "$ $CLEAN_IMAGE_COMMIT"
+    eval $CLEAN_IMAGE_COMMIT
+    if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then
+        DOCKER_RMI="docker rmi '$REG:$SANITIZED_TAG' || :"
+        echo "$ $DOCKER_RMI"
+        eval $DOCKER_RMI
     fi
 done
-
-DOCKER_GEN="docker rmi eosio_cdt_image:$BUILD_TAG"
-echo "Clean up base image"
-eval $DOCKER_GEN
+DOCKER_RMI="docker rmi '$IMAGE' || :"
+echo "$ $DOCKER_RMI"
+eval $DOCKER_RMI
+echo 'Done.'

.cicd/generate-base-images.sh

@@ -9,7 +9,7 @@ TAG=$(echo $FULL_TAG | cut -d: -f2)
 EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG")
 # build, if neccessary
 if [[ $EXISTS =~ '404 page not found' || $EXISTS =~ 'manifest unknown' ]]; then # if we cannot pull the image, we build and push it first
-    docker build -t $FULL_TAG -f $CICD_DIR/docker/${IMAGE_TAG}.dockerfile .
+    docker build -t $FULL_TAG -f $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile .
     docker push $FULL_TAG
 else
     echo "$FULL_TAG already exists."

.cicd/helpers/docker-hash.sh

@@ -16,7 +16,7 @@ function determine-hash() {
 }
 
 if [[ ! -z $IMAGE_TAG ]]; then
-    determine-hash "$CICD_DIR/docker/${IMAGE_TAG}.dockerfile"
+    determine-hash "$CICD_DIR/platforms/${IMAGE_TAG}.dockerfile"
     export FULL_TAG="eosio/ci:eosio-cdt-$HASHED_IMAGE_TAG"
 else
     echo "Please set ENV::IMAGE_TAG to match the name of a platform dockerfile..."

.cicd/helpers/sanitize.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+            ##### sanitize branch names for use in URIs (docker containers) #####
+# tr '/' '_'                       # convert forward-slashes '/' to underscores '_'
+# sed -E 's/[^-_.a-zA-Z0-9]+/-/g'  # convert invalid docker chars to '-'
+# sed -E 's/-+/-/g'                # replace multiple dashes in a series "----" with a single dash '-'
+# sed -E 's/-*_+-*/_/g'            # replace dashes '-' and underscores '_' in-series with a single underscore '_'
+# sed -E 's/_+/_/g'                # replace multiple underscores in a row "___" with a single underscore '_'
+# sed -E 's/(^[-_.]+|[-_.]+$)//g'  # ensure tags do not begin or end with separator characters [-_.]
+function sanitize()
+{
+    echo "$1" | tr '/' '_' | sed -E 's/[^-_.a-zA-Z0-9]+/-/g' | sed -E 's/-+/-/g' | sed -E 's/-*_+-*/_/g' | sed -E 's/_+/_/g' | sed -E 's/(^[-_.]+|[-_.]+$)//g'
+}

.cicd/pipeline-upload.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eou pipefail
+
+export MACOS_10_14_TAG="eosio-cdt-macos-10.14-$(sha1sum ./.cicd/platforms/macos-10.14.sh | awk '{print $1}')"
+export MACOS_10_15_TAG="eosio-cdt-macos-10.15-$(sha1sum ./.cicd/platforms/macos-10.15.sh | awk '{print $1}')"
+export VARS='$MACOS_10_14_TAG:$MACOS_10_15_TAG'
+envsubst "$VARS" < "./.cicd/pipeline.yml" > "./.cicd/pipeline.yml.out"
+buildkite-agent artifact upload ./.cicd/pipeline.yml.out
+buildkite-agent pipeline upload ./.cicd/pipeline.yml.out