Merge pull request #208 from IBM/v1.1.1_power_support

v1.1.1 : Add support for IBM Power Systems

Dockerfile

@@ -7,7 +7,7 @@ RUN CGO_ENABLED=1 GOOS=linux go build -tags netgo -v -a --ldflags '-w -linkmode
 
 
 FROM alpine:3.7
-RUN apk --no-cache add ca-certificates=20171114-r0 openssl=1.0.2n-r0
+RUN apk --no-cache add ca-certificates=20171114-r0 openssl=1.0.2o-r0
 WORKDIR /root/
 COPY --from=0 /go/src/github.com/IBM/ubiquity/ubiquity .
 COPY --from=0 /go/src/github.com/IBM/ubiquity/LICENSE .

scripts/helper_to_push_internal_images_2hub.sh

@@ -5,38 +5,71 @@
 
 function usage()
 {
-    echo $0 [internal-image-path]  [external-image-path]
+    echo $0 [internal-image-path]  [external-image-path] [latest-tag-optional]
     exit 1
 }
 
-[ $# != 2 ] && { usage; }
+[ $# -eq 2 -o $# -eq 3 ] || { usage; }
 
 internal_image=$1
 external_image=$2
+latest="$3"
 
-echo "1. Validate no external_image exist yet before pushing it."
+# if also latest, then define latest_external_image (replace the external tag with the $latest wanted tag)
+if [ -n "$latest" ]; then
+   latest_external_image=`echo $external_image | sed "s|^\(.*/.*:\)\(.*\)$|\1$latest|"`
+   latest_str_msg=" And latest tag image [$latest_external_image]"
+fi
+
+echo "Preparing to push internal_image --> external_image:"
+echo "   internal_image=[$internal_image]"
+echo "   external_image=[$external_image]"
+[ -n "$latest" ] && echo "   latest_image  =[$latest_external_image]"
+
+
+echo "1. Validate no external_image exist yet before pushing it."  # Note: no need to test latest tag since its already exist
 docker pull $external_image && { echo "Error : the $external_image exist in remote. Cannot overwrite it."; exit 1; } || { echo "$external_image is not exist on the remote."; }
 echo ""
 
+
 echo "2. Validate internal_image not exist yet on local."
 docker images $internal_image
 docker rmi $internal_image && { echo "Remove the internal_image image to pull it again"; } || { echo "internal_image not exist on local. Continue."; }
 echo ""
 
+
 echo "3. Pull internal_image to local"
 docker pull $internal_image
 echo ""
 
-echo "4. Tag internal_image to external_image and remove the internal_image"
+
+echo "4. Tag internal_image to external_image (and latest=[$latest_external_image]) and remove the internal_image"
 docker tag  $internal_image $external_image
+[ -n "$latest_external_image" ] && docker tag  $internal_image $latest_external_image
 docker rmi $internal_image
 docker push $external_image
+[ -n "$latest_external_image" ] && docker push $latest_external_image
 echo ""
 
+
 echo "5. Test pushed image by delete the local and pull it back"
 docker rmi $external_image
 docker pull $external_image
 docker rmi $external_image
 
+
+if [ -n "$latest_external_image" ]; then
+    echo "6. Test pushed [latest] image by delete the local and pull it back"
+    docker rmi $latest_external_image
+    docker pull $latest_external_image
+    docker rmi $latest_external_image
+fi
+
+set +x
 echo ""
-echo "Succeeded to push [$internal_image]   --->   [$external_image]"
+echo "Succeeded to push internal_image --> external_image"
+echo "   internal_image=[$internal_image]"
+echo "   external_image=[$external_image]"
+[ -n "$latest" ] && echo "   latest_image  =[$latest_external_image]"
+set -x
+

scripts/helper_to_push_manifest.sh

@@ -0,0 +1,83 @@
+#!/bin/bash -xe
+
+# ----------------------------------------------------------------
+# This script is for internal use in CI
+# The script create, inspect and push new docker manifest.
+# The script validates the whole process. If something gets wrong the script will fail with error.
+# Pre-requisites:
+#    1. Run docker login to the external registry in advance.
+#    2. The internal images should be exist in advance.
+#    3. The designated manifest should NOT be exist (the script will create it).
+#    4. "docker manifest" is enabled.
+# ----------------------------------------------------------------
+
+function abort()
+{
+    exitcode=$1
+    msg=$2
+    echo "Error: $msg"
+    exit $exitcode
+}
+
+USAGE="Usage : $0 [manifest-path] [image-amd64-path] [image-ppc64le-path] [fail-if-exist-optional]"
+
+[ $# -eq 3 -o $# -eq 4 ] || abort 1 $USAGE
+
+expected_manifest_arch_number=2
+manifest=$1
+image_amd64=$2
+image_ppc64le=$3
+fail_if_exist="${4:-true}"  # by default fail if external manifest already exist
+
+echo "Preparing to create and push manifest for image_amd64 and image_ppc64le:"
+echo "   manifest=[$manifest]"
+echo "   image_amd64=[$image_amd64]"
+echo "   image_ppc64le=[$image_ppc64le]"
+
+
+manifest_dirname="~/.docker/manifests"
+dockerhub_prefix_manifest="docker.io_"
+convert_manifest_filename=`echo "$manifest" | sed -e 's|/|_|g' -e 's|:|-|g'`
+specific_manifest_dirname="$manifest_dirname/${convert_manifest_filename}"
+specific_manifest_dirname_with_prefix="$manifest_dirname/${dockerhub_prefix_manifest}${convert_manifest_filename}"
+echo "1. Make sure architecture images are not exist locally and if so remove them first for clean state..."
+[ -n "`docker images -q $image_amd64`" ]   && { docker rmi $image_amd64   || abort 1 "fail to clean image before creating manifest. [$image_amd64]"; }   || :
+[ -n "`docker images -q $image_ppc64le`" ] && { docker rmi $image_ppc64le || abort 1 "fail to clean image before creating manifest. [$image_ppc64le]"; } || :
+
+
+echo "2. Manifest validation \(manifest should not exit, not local nor remote\)..."
+ls -ld "$specific_manifest_dirname $specific_manifest_dirname_with_prefix" 	&& abort 1 "local manifest dir should NOT exist before creating the manifest. Please clean it and rerun."
+if [ "$fail_if_exist" = "true" ]; then
+    docker manifest inspect $manifest 	&& abort 1 "manifest inspect should NOT exist before pushing it. Please clean it and rerun." || :
+else
+    docker manifest inspect $manifest 	&& echo "manifest inspect should NOT exist before pushing it. BUT fail_if_exist is not true so skip validation." || :
+fi
+
+echo "3. Manifest creation and push..."
+docker manifest create $manifest ${image_amd64} ${image_ppc64le} 	|| abort 2 "fail to create manifest."
+docker manifest inspect $manifest 	                                ||   abort 2 "fail to inspect local manifest."
+actual_manifest_arch_number=`docker manifest inspect $manifest | grep architecture | wc -l`
+[ $actual_manifest_arch_number -ne $expected_manifest_arch_number ] && abort 3 "Manifest created but its not contain [$expected_manifest_arch_number] architectures as expected."
+docker manifest push --purge $manifest 	||  abort 2 "fail to push manifest to remote repo"
+ls -ld $specific_manifest_dirname $specific_manifest_dirname_with_prefix       && abort 2 "Local manifest file should NOT exist after successful manifest push. Please check." || :
+
+
+
+echo "4. Remote manifest validation..."
+docker manifest inspect $manifest 	    ||   abort 3 "fail to inspect remote manifest."
+docker pull $manifest                   ||   abort 3 "fail pull remote manifest."
+expected_arch=`uname -m`
+docker run  --rm $manifest uname -m | grep $expected_arch || abort 3 "The manifest run did not bring the expected arch"
+docker rmi $manifest  # just remove the local manifest that was pulled for testing
+actual_manifest_arch_number=`docker manifest inspect $manifest | grep architecture | wc -l`
+[ $actual_manifest_arch_number -ne $expected_manifest_arch_number ] && abort 3 "Manifest pushed but its not contain [$expected_manifest_arch_number] architectures as expected."
+
+
+set +x
+echo "================================================================================"
+echo "Succeeded to create and push manifest for image_amd64 and image_ppc64le:"
+echo "   manifest=[$manifest]"
+echo "   image_amd64=[$image_amd64]"
+echo "   image_ppc64le=[$image_ppc64le]"
+set -x
+

scripts/helper_to_push_ubiquity_images_and_manifests.sh

@@ -0,0 +1,86 @@
+#!/bin/bash -ex
+
+# ----------------------------------------------------------------
+# This script is for internal use in CI
+# The script push all ubiquity images from internal registry to external registry.
+# Images for amd64 and ppc64le for each ubiquity image : ubiquity, ubiquity-db, flex and provisioner.
+# It also creates and pushes relevant manifests per architecture into the external repository.
+# The script validates the whole process. If something gets wrong the script will fail with error.
+# Pre-requisites:
+#    1. Run docker login to the external registry in advance.
+#    2. The internal images should be exist in advance.
+#    3. The external images should NOT be exist (the script will creates them).
+#    4. Helper scripts should be accessible: ./helper_to_push_internal_images_2hub.sh and ./helper_to_push_manifest.sh
+#    5. Scripts input comes from environment variables, see ubiquity_*_envs and optional TAG_LATEST
+# ----------------------------------------------------------------
+
+ubiquity_envs="in_UBIQUITY_IMAGE_AMD64 out_UBIQUITY_IMAGE_AMD64 in_UBIQUITY_IMAGE_PPC64LE out_UBIQUITY_IMAGE_PPC64LE out_UBIQUITY_IMAGE_MULTIARCH"
+ubiquity_db_envs="in_UBIQUITY_DB_IMAGE_AMD64 out_UBIQUITY_DB_IMAGE_AMD64 in_UBIQUITY_DB_IMAGE_PPC64LE out_UBIQUITY_DB_IMAGE_PPC64LE out_UBIQUITY_DB_IMAGE_MULTIARCH"
+ubiquity_provisioner_envs="in_UBIQUITY_K8S_PROVISIONER_IMAGE_AMD64 out_UBIQUITY_K8S_PROVISIONER_IMAGE_AMD64 in_UBIQUITY_K8S_PROVISIONER_IMAGE_PPC64LE out_UBIQUITY_K8S_PROVISIONER_IMAGE_PPC64LE out_UBIQUITY_K8S_PROVISIONER_IMAGE_MULTIARCH"
+ubiquity_flex_envs="in_UBIQUITY_K8S_FLEX_IMAGE_AMD64 out_UBIQUITY_K8S_FLEX_IMAGE_AMD64 in_UBIQUITY_K8S_FLEX_IMAGE_PPC64LE out_UBIQUITY_K8S_FLEX_IMAGE_PPC64LE out_UBIQUITY_K8S_FLEX_IMAGE_MULTIARCH"
+
+date
+# Validations
+[ -f ./helper_to_push_internal_images_2hub.sh -a -f ./helper_to_push_manifest.sh ]  && : || exit 1
+for expected_env in $ubiquity_envs $ubiquity_db_envs $ubiquity_provisioner_envs $ubiquity_flex_envs; do
+   [ -z "`printenv $expected_env`" ] && { echo "Error: expected env [$expected_env] does not exist. Please set it first."; exit 1; } || :
+   echo "$expected_env=`printenv $expected_env`"
+done
+
+echo "TAG_LATEST=$TAG_LATEST"
+
+echo ""
+echo "Start to push Ubiquity images and manifests..."
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_IMAGE_AMD64                 $out_UBIQUITY_IMAGE_AMD64         $TAG_LATEST
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_IMAGE_PPC64LE               $out_UBIQUITY_IMAGE_PPC64LE       $TAG_LATEST
+./helper_to_push_manifest.sh $out_UBIQUITY_IMAGE_MULTIARCH   $out_UBIQUITY_IMAGE_AMD64  $out_UBIQUITY_IMAGE_PPC64LE
+if [ -n "$TAG_LATEST" ]; then
+    latest_external_image=`echo $out_UBIQUITY_IMAGE_MULTIARCH | sed "s|^\(.*/.*:\)\(.*\)$|\1$TAG_LATEST|"` # replace tag with $TAG_LATEST
+    ./helper_to_push_manifest.sh $latest_external_image   $out_UBIQUITY_IMAGE_AMD64  $out_UBIQUITY_IMAGE_PPC64LE
+fi
+
+echo ""
+echo "Start to push Ubiquity DB images and manifests..."
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_DB_IMAGE_AMD64                $out_UBIQUITY_DB_IMAGE_AMD64             $TAG_LATEST
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_DB_IMAGE_PPC64LE              $out_UBIQUITY_DB_IMAGE_PPC64LE           $TAG_LATEST
+./helper_to_push_manifest.sh $out_UBIQUITY_DB_IMAGE_MULTIARCH   $out_UBIQUITY_DB_IMAGE_AMD64  $out_UBIQUITY_DB_IMAGE_PPC64LE $TAG_LATEST
+if [ -n "$TAG_LATEST" ]; then
+    latest_external_image=`echo $out_UBIQUITY_DB_IMAGE_MULTIARCH | sed "s|^\(.*/.*:\)\(.*\)$|\1$TAG_LATEST|"` # replace tag with $TAG_LATEST
+    ./helper_to_push_manifest.sh $latest_external_image   $out_UBIQUITY_DB_IMAGE_AMD64  $out_UBIQUITY_DB_IMAGE_PPC64LE $TAG_LATEST
+fi
+
+
+echo ""
+echo "Start to push Ubiquity provisioner images and manifests..."
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_K8S_PROVISIONER_IMAGE_AMD64                 $out_UBIQUITY_K8S_PROVISIONER_IMAGE_AMD64        $TAG_LATEST
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_K8S_PROVISIONER_IMAGE_PPC64LE               $out_UBIQUITY_K8S_PROVISIONER_IMAGE_PPC64LE      $TAG_LATEST
+./helper_to_push_manifest.sh $out_UBIQUITY_K8S_PROVISIONER_IMAGE_MULTIARCH   $out_UBIQUITY_K8S_PROVISIONER_IMAGE_AMD64  $out_UBIQUITY_K8S_PROVISIONER_IMAGE_PPC64LE  $TAG_LATEST
+if [ -n "$TAG_LATEST" ]; then
+    latest_external_image=`echo $out_UBIQUITY_K8S_PROVISIONER_IMAGE_MULTIARCH | sed "s|^\(.*/.*:\)\(.*\)$|\1$TAG_LATEST|"` # replace tag with $TAG_LATEST
+    ./helper_to_push_manifest.sh $latest_external_image   $out_UBIQUITY_K8S_PROVISIONER_IMAGE_AMD64  $out_UBIQUITY_K8S_PROVISIONER_IMAGE_PPC64LE  $TAG_LATEST
+fi
+
+
+echo ""
+echo "Start to push Ubiquity flex images and manifests..."
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_K8S_FLEX_IMAGE_AMD64                 $out_UBIQUITY_K8S_FLEX_IMAGE_AMD64     $TAG_LATEST
+./helper_to_push_internal_images_2hub.sh $in_UBIQUITY_K8S_FLEX_IMAGE_PPC64LE               $out_UBIQUITY_K8S_FLEX_IMAGE_PPC64LE   $TAG_LATEST
+./helper_to_push_manifest.sh $out_UBIQUITY_K8S_FLEX_IMAGE_MULTIARCH   $out_UBIQUITY_K8S_FLEX_IMAGE_AMD64  $out_UBIQUITY_K8S_FLEX_IMAGE_PPC64LE   $TAG_LATEST
+if [ -n "$TAG_LATEST" ]; then
+    latest_external_image=`echo $out_UBIQUITY_K8S_FLEX_IMAGE_MULTIARCH | sed "s|^\(.*/.*:\)\(.*\)$|\1$TAG_LATEST|"` # replace tag with $TAG_LATEST
+    ./helper_to_push_manifest.sh $latest_external_image   $out_UBIQUITY_K8S_FLEX_IMAGE_AMD64  $out_UBIQUITY_K8S_FLEX_IMAGE_PPC64LE   $TAG_LATEST
+fi
+
+
+date
+echo "######################################"
+echo "Finish to push successfully all images"
+echo "######################################"
+
+
+echo $out_UBIQUITY_IMAGE_MULTIARCH
+echo $out_UBIQUITY_DB_IMAGE_MULTIARCH
+echo $out_UBIQUITY_K8S_PROVISIONER_IMAGE_MULTIARCH
+echo $out_UBIQUITY_K8S_FLEX_IMAGE_MULTIARCH
+
+

scripts/images_verification/ubiquity_image_verification/ubiquity_image_command_test.yaml

@@ -1,7 +1,7 @@
 schemaVersion: '2.0.0'
 
 commandTests:
-  # check that alpine version is 3.7.0 and ca-certificates version 20171114-r0 and openssl version 1.0.2n-r0 packages installed
+  # check that alpine version is 3.7.0 and ca-certificates version 20171114-r0 and openssl version 1.0.2o-r0 packages installed
   - name: "alpine version 3.7.0"
     command: "cat"
     args: ["/etc/alpine-release"]
@@ -10,7 +10,7 @@ commandTests:
     command: "apk"
     args: ["info", "-vv"]
     expectedOutput: ["ca-certificates-20171114-r0"]
-  - name: "openssl version 1.0.2n-r0 package"
+  - name: "openssl version 1.0.2o-r0 package"
     command: "apk"
     args: ["info", "-vv"]
-    expectedOutput: ["openssl-1.0.2n-r0"]
+    expectedOutput: ["openssl-1.0.2o-r0"]