Merge branch 'v2-master' of github.com:cloudfoundry-incubator/stratos…

… into autoscaler

.cf-extensions

@@ -2,6 +2,6 @@ lead_company: SUSE
 contact_email: neil.macdougall@suse.com
 description: Stratos - Console UI for Cloud Foundry
 slack_channel: '#stratos'
-tracker_url: https://app.zenhub.com/workspace/o/cloudfoundry-incubator/stratos
+tracker_url: https://github.com/cloudfoundry-incubator/stratos/issues
 proposal_url: https://docs.google.com/document/d/1GgizNSutb3Lku1TshD333-7Dc_6fpKU9Wdi7XloKuK4
 proposed_date: 2017-12-19T11:15:00

.travis.yml

@@ -61,11 +61,16 @@ jobs:
   - name: Helm Chart Unit Tests
     script:
     - "./deploy/ci/travis/helm-chart-unit-tests.sh"
-  - name: E2E Tests - Long Suite
+  - name: E2E Tests - Long Suite 1
     before_script:
     - "./deploy/ci/travis/job-e2e-before_script.sh"
     script:
     - "./deploy/ci/travis/job-e2e-script.sh longSuite"
+  - name: E2E Tests - Long Suite 2
+    before_script:
+    - "./deploy/ci/travis/job-e2e-before_script.sh true"
+    script:
+    - "./deploy/ci/travis/job-e2e-script.sh longSuite2"
   - name: E2E Tests - All Other Suite
     before_script:
     - "./deploy/ci/travis/job-e2e-before_script.sh"

README.md

@@ -3,7 +3,6 @@
 <a href="https://travis-ci.org/cloudfoundry-incubator/stratos/branches"><img src="https://travis-ci.org/cloudfoundry-incubator/stratos.svg?branch=v2-master"></a>&nbsp;<a style="padding-left: 4px" href="https://codeclimate.com/github/cloudfoundry-incubator/stratos/maintainability"><img src="https://api.codeclimate.com/v1/badges/61af8b605f385e894632/maintainability" /></a>
 <a href="https://goreportcard.com/github.com/cloudfoundry-incubator/stratos"><img src="https://goreportcard.com/badge/github.com/cloudfoundry-incubator/stratos"/></a>
 <a href="https://codecov.io/gh/cloudfoundry-incubator/stratos/branch/v2-master"><img src="https://codecov.io/gh/cloudfoundry-incubator/stratos/branch/v2-master/graph/badge.svg"/></a>
-<a href="https://app.zenhub.com/workspace/o/cloudfoundry-incubator/stratos/boards"><img src="https://raw.githubusercontent.com/ZenHubIO/support/master/zenhub-badge.png"/></a>
 [![GitHub release](https://img.shields.io/github/release/cloudfoundry-incubator/stratos.svg)](https://github.com/cloudfoundry-incubator/stratos/releases/latest)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/cloudfoundry-incubator/stratos/blob/master/LICENSE)
 [![slack.cloudfoundry.org](https://slack.cloudfoundry.org/badge.svg)](https://cloudfoundry.slack.com/messages/C80EP4Y57/)

deploy/ci/automation/e2e-clean-remnants.sh

@@ -40,13 +40,20 @@ function clean() {
   local REGEX="^($PREFIX)(.*)\.([0-9]*)[Tt]([0-9]*)[zZ].*"
   local NOW=$(date "+%s")
 
+  if [ -z "$4" ]; then
+    local REGEX="^($PREFIX)(.*)\.([0-9]*)[Tt]([0-9]*)[zZ].*"
+  else
+    local REGEX="$4"
+  fi
+
   while IFS= read -r line
   do
     NAME="${line%% *}"
     if [[ $NAME =~ $REGEX ]]; then
       DS="${BASH_REMATCH[3]}"
       TS="${BASH_REMATCH[4]}"
       TS="${TS:0:6}"
+
       if [[ "$unamestr" == 'Darwin' ]]; then
         EPOCH=$(date -j -f "%Y%m%d:%H%M%S" "$DS:$TS" "+%s")
       else
@@ -89,4 +96,9 @@ echo "Cleaning old Spaces in e2e org"
 SPACES="$(cf spaces)"
 clean "$SPACES" "acceptance\.e2e\." "delete-space"
 
+# Users
+echo "Cleaning test Users"
+USERS=$(cf space-users e2e e2e | grep "accept" | sed -e 's/^[[:space:]]*//')
+clean "$USERS" "-" "delete-user" "^(acceptancee2etravis)(invite[0-9])(20[0-9]*)[Tt]([0-9]*)[zZ].*"
+
 echo "Done"

deploy/ci/tasks/dev-releases/create-chart-helper.sh

@@ -72,3 +72,21 @@ fetchImageTag() {
 nightlyTag() {
   echo "$(cat ${STRATOS}/deploy/ci/tasks/dev-releases/nightly-tag)"
 }
+
+# patchHelmChartAppVersion - update appVersion when patching the Helm Chart if configured
+patchHelmChartAppVersion() {
+  local CHART_PATH=$1
+  local STRATOS_FOLDER=$2
+
+  if [ -f "${STRATOS_FOLDER}/custom-src/stratos.yaml" ]; then
+    PROD_VERSION=$(cat "${STRATOS_FOLDER}/custom-src/stratos.yaml" | grep "productVersion")
+    if [ ! -z "${PROD_VERSION}" ]; then
+      PROD_VERSION=$(echo $PROD_VERSION | grep --extended --only-matching '[0-9\.]+')
+      if [ ! -z "${PROD_VERSION}" ]; then
+        echo "Setting appVersion to: ${PROD_VERSION}"
+        sed -i.bak -e 's/appVersion: [0-9\.]*/appVersion: '"${PROD_VERSION}"'/g' ${CHART_PATH}/Chart.yaml
+      fi
+    fi
+  fi
+
+}

deploy/ci/tasks/dev-releases/create-chart.yml

@@ -33,8 +33,15 @@ run:
       IMAGE_TAG=${GIT_TAG}
       cd ${STRATOS}/deploy/kubernetes/
       patchHelmChart ${GIT_TAG} ${DOCKER_ORG} ${DOCKER_REGISTRY} ./console ${RELEASE_VERSION} ${VERSION}
+      patchHelmChartAppVersion ./console ${STRATOS}
+
       # Generate Helm package
       updateHelmDependency
+
+      # Generate imagelist
+      ./imagelist-gen.sh ./console
+
+      # Package console Helm Chart
       helm package console
       cp console*.tgz ${ROOT_DIR}/helm-chart/console-helm-chart-${RELEASE_VERSION}-${COMMIT}.tgz
       cd ${ROOT_DIR}/helm-chart/

deploy/ci/travis/config.properties

@@ -17,3 +17,9 @@ CONSOLE_PROXY_CERT_KEY_PATH=../../dev-ssl/server.key
 ENCRYPTION_KEY=B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF
 SQLITE_KEEP_DB=true
 UI_PATH=../../dist
+SMTP_FROM_ADDRESS=Stratos<test@test.com>
+SMTP_HOST=127.0.0.1
+SMTP_PASSWORD=
+SMTP_PORT=1025
+SMTP_USER=
+TEMPLATE_DIR=./templates

deploy/ci/travis/job-e2e-before_script.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 set -e
 
+MAILCATCHER=$1
+
 DIRPATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../../.. && pwd)"
 
 chmod +x ${DIRPATH}/deploy/ci/travis/run-e2e-tests.sh
 # We will install ffmpeg so we can capture a video of the display as the tests run
 sudo add-apt-repository -y ppa:mc3man/trusty-media
 sudo apt-get -qq update
+
+if [ "${MAILCATCHER}" == "true" ]; then
+  docker run -d -p 1080:80 -p 1025:25 --name mail tophfr/mailcatcher
+fi

deploy/kubernetes/build.sh

@@ -8,7 +8,12 @@
 #
 #####
 
-set -eu
+set -u
+
+CYAN="\033[96m"
+YELLOW="\033[93m"
+RESET="\033[0m"
+BOLD="\033[1m"
 
 # Set defaults
 PROD_RELEASE=false
@@ -21,6 +26,7 @@ ADD_OFFICIAL_TAG="false"
 TAG_LATEST="false"
 NO_PUSH="true"
 DOCKER_REG_DEFAULTS="true"
+
 while getopts ":ho:r:t:Tclb:On" opt; do
   case $opt in
     h)
@@ -70,22 +76,48 @@ while getopts ":ho:r:t:Tclb:On" opt; do
 done
 
 echo
-echo "PRODUCTION BUILD/RELEASE: ${PROD_RELEASE}"
-echo "REGISTRY: ${DOCKER_REGISTRY}"
-echo "ORG: ${DOCKER_ORG}"
-echo "TAG: ${TAG}"
-echo "BASE_IMAGE_TAG: ${BASE_IMAGE_TAG}"
+printf "${CYAN}${BOLD}"
+
+echo "==========================================================================="
+echo "== Stratos Image and Helm Chart build                                    =="
+echo "==========================================================================="
+
+printf "${RESET}${CYAN}"
+echo
+echo "PRODUCTION BUILD/RELEASE : ${PROD_RELEASE}"
+echo "REGISTRY                 : ${DOCKER_REGISTRY}"
+echo "ORG                      : ${DOCKER_ORG}"
+echo "TAG                      : ${TAG}"
+echo "BASE_IMAGE_TAG           : ${BASE_IMAGE_TAG}"
+
+printf "${RESET}"
+echo
 
 if [ "${NO_PUSH}" != "false" ]; then
-  echo "Images will NOT be pushed"
+  printf "${YELLOW}Images will ${BOLD}NOT${RESET}${YELLOW} be pushed${RESET}\n"
 else
-  echo "Images will be pushed"
-  echo "  REGISTRY: ${DOCKER_REGISTRY}"
-  echo "  ORG: ${DOCKER_ORG}"
+  printf "${YELLOW}${BOLD}Images will be pushed${RESET}\n"
+  echo "  REGISTRY : ${DOCKER_REGISTRY}"
+  echo "  ORG      : ${DOCKER_ORG}"
 fi
 
 echo
-echo "Starting build"
+printf "${CYAN}${BOLD}Starting build${RESET}\n"
+echo
+
+function log {
+  set +e
+  printf "${BOLD}${YELLOW}"
+  echo
+  echo "==========================================================================="
+  echo "$1"
+  echo "==========================================================================="
+  echo
+  printf "${RESET}"
+  set -e
+}
+
+set -e
 
 # Copy values template
 __DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -97,89 +129,70 @@ function patchAndPushImage {
   DOCKER_FILE=${2}
   FOLDER=${3}
   TARGET=${4:-none}
+  PATCHED_DOCKER_FILE="${DOCKER_FILE}.patched"
 
   patchDockerfile ${DOCKER_FILE} ${FOLDER}
-  buildAndPublishImage ${NAME} ${DOCKER_FILE} ${FOLDER} ${TARGET}
-  unPatchDockerfile ${DOCKER_FILE} ${FOLDER}
+  buildAndPublishImage ${NAME} "${PATCHED_DOCKER_FILE}" ${FOLDER} ${TARGET}
+
+  rm -rf ${FOLDER}/${PATCHED_DOCKER_FILE}
+  rm -rf ${FOLDER}/${PATCHED_DOCKER_FILE}.bak
 }
 
 function patchDockerfile {
   DOCKER_FILE=${1}
   FOLDER=${2}
+  PATCHED_DOCKER_FILE=${DOCKER_FILE}.patched
 
   # Replace registry/organization
   pushd ${FOLDER} > /dev/null 2>&1
-  pwd
+  ls
+  rm -rf ${PATCHED_DOCKER_FILE}
+  cp ${DOCKER_FILE} ${PATCHED_DOCKER_FILE}
   if [ "${DOCKER_REG_DEFAULTS}" == "false" ]; then
-    sed -i.bak "s@splatform@${DOCKER_REGISTRY}/${DOCKER_ORG}@g" ${FOLDER}/${DOCKER_FILE}
+    sed -i.bak "s@splatform@${DOCKER_REGISTRY}/${DOCKER_ORG}@g" ${FOLDER}/${PATCHED_DOCKER_FILE}
   fi
-  sed -i.bak "s/opensuse/${BASE_IMAGE_TAG}/g" ${FOLDER}/${DOCKER_FILE}
+  sed -i.bak "s/opensuse/${BASE_IMAGE_TAG}/g" ${FOLDER}/${PATCHED_DOCKER_FILE}
   popd > /dev/null 2>&1
-
 }
 
-function unPatchDockerfile {
-  DOCKER_FILE=${1}
-  FOLDER=${2}
 
-  # Replace registry/organization
-  pushd ${FOLDER} > /dev/null 2>&1
-  pwd
-  if [ "${DOCKER_REG_DEFAULTS}" == "false" ]; then
-    sed -i.bak "s@${DOCKER_REGISTRY}/${DOCKER_ORG}@splatform@g" ${FOLDER}/${DOCKER_FILE}
-  fi
-  sed -i.bak "s/${BASE_IMAGE_TAG}/opensuse/g" ${FOLDER}/${DOCKER_FILE}
-  popd > /dev/null 2>&1
+#
+# MAIN -------------------------------------------------------------------------------------------
+#
 
-}
+pushd "${STRATOS_PATH}" > /dev/null 2>&1
+STRATOS_PATH="$(pwd)"
+popd > /dev/null 2>&1
+echo "Base path: ${STRATOS_PATH}"
 
-function buildJetstream {
-  echo
-  echo "-- Building the Stratos Backend"
+# cleanup output, intermediate artifacts
+cleanup
 
-  echo
-  echo "-- Build & publish the runtime container image for the Console jetstream"
-  patchAndPushImage stratos-jetstream deploy/Dockerfile.bk "${STRATOS_PATH}" prod-build
-}
+# Clean any old patched docker files left if previously errored
+# rm -rf ${STRATOS_PATH}/deploy/Dockerfile.*.patched
+# rm -rf ${STRATOS_PATH}/deploy/Dockerfile.*.bak
+# rm -rf ${STRATOS_PATH}/deploy/Dockerfile.*.patched.bak
 
-function buildPostflightJob {
-  # Build the postflight container
-  echo
-  echo "-- Build & publish the runtime container image for the postflight job"
-  patchAndPushImage stratos-postflight-job deploy/Dockerfile.bk "${STRATOS_PATH}" postflight-job
-}
+updateTagForRelease
 
-function buildMariaDb {
-  echo
-  echo "-- Building/publishing MariaDB"
-  # Download and retag image to save bandwidth
-  patchAndPushImage stratos-mariadb Dockerfile.mariadb "${STRATOS_PATH}/deploy/db"
-}
+# Build all of the components that make up the Console
 
-function buildUI {
-  # Build and push an image based on the nginx container
-  echo
-  echo "-- Building/publishing the runtime container image for the Console web server"
-  # Download and retag image to save bandwidth
-  patchAndPushImage stratos-console deploy/Dockerfile.ui "${STRATOS_PATH}" prod-build
-}
+log "-- Build & publish the runtime container image for Jetstream (backend)"
+patchAndPushImage stratos-jetstream deploy/Dockerfile.bk "${STRATOS_PATH}" prod-build
 
-# MAIN ------------------------------------------------------
-#
+# Build the postflight container
+log "-- Build & publish the runtime container image for the postflight job"
+patchAndPushImage stratos-postflight-job deploy/Dockerfile.bk "${STRATOS_PATH}" postflight-job
 
-# Set the path to the portal jetstream
-STRATOS_PATH=${STRATOS_PATH}
+# Build and push an image based on the mariab db container
+log "-- Building/publishing MariaDB"
+patchAndPushImage stratos-mariadb Dockerfile.mariadb "${STRATOS_PATH}/deploy/db"
 
-# cleanup output, intermediate artifacts
-cleanup
-
-updateTagForRelease
+# Build and push an image based on the nginx container (Front-end)
+log "-- Building/publishing the runtime container image for the Console web server (frontend)"
+patchAndPushImage stratos-console deploy/Dockerfile.ui "${STRATOS_PATH}" prod-build
 
-# Build all of the components that make up the Console
-buildJetstream
-buildPostflightJob
-buildMariaDb
-buildUI
+log "-- Building Helm Chart"
 
 # Don't change the chart in the repo, copy it and modify it locally
 
@@ -200,15 +213,29 @@ sed -i.bak -e 's/organization: splatform/organization: '"${DOCKER_ORG}"'/g' valu
 sed -i.bak -e 's/hostname: docker.io/hostname: '"${DOCKER_REGISTRY}"'/g' values.yaml
 
 sed -i.bak -e 's/version: 0.1.0/version: '"${TAG}"'/g' Chart.yaml
-
+sed -i.bak -e 's/appVersion: 0.1.0/appVersion: '"${TAG}"'/g' Chart.yaml
 rm -rf *.bak
+
+# Generate image list
+echo ${STRATOS_PATH}
+echo
+${STRATOS_PATH}/deploy/kubernetes/imagelist-gen.sh .
+
 popd > /dev/null
 
+set +e
+
+printf "${BOLD}${YELLOW}"
 echo
 echo "Build complete...."
-echo "Registry: ${DOCKER_REGISTRY}"
-echo "Org: ${DOCKER_ORG}"
-echo "Tag: ${TAG}"
+printf "${CYAN}"
+echo "  Registry            : ${DOCKER_REGISTRY}"
+echo "  Org                 : ${DOCKER_ORG}"
+echo "  Tag                 : ${TAG}"
+printf "${RESET}"
 
-echo "To deploy using Helm, execute the following: "
-echo "helm install helm-chart --namespace console --name my-console"
+echo
+echo "To deploy using Helm, execute the following:"
+echo 
+echo "    helm install helm-chart --namespace console --name my-console"
+echo

deploy/kubernetes/imagelist-gen.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Colours
+CYAN="\033[96m"
+YELLOW="\033[93m"
+RESET="\033[0m"
+BOLD="\033[1m"
+
+__DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+printf "${BOLD}${CYAN}Generating ${YELLOW}imagelist.txt${RESET}\n"
+echo ""
+
+
+CHART_FOLDER=${1}
+pushd ${CHART_FOLDER} > /dev/null
+helm template -f ${__DIRNAME}/imagelist.values.yaml ${CHART_FOLDER} | grep "image:" | grep --extended --only-matching '([^"/[:space:]]+/)?[^"/[:space:]]+/[^:[:space:]]+:[a-zA-Z0-9\._-]+' | sort | uniq | awk -F'/' '{print $2}' > imagelist.txt
+popd > /dev/null
+
+printf "${CYAN}"
+cat ${CHART_FOLDER}/imagelist.txt
+printf "${RESET}"
+echo ""