build-app

#!/bin/bash
# Copyright 2014 - Rackspace Hosting
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.


# Solum Build Script for Docker and lp-cedarish


SCRIPT_START_TIME=$(date +"%s")

IMAGE_STORAGE=${IMAGE_STORAGE:-null}
ASSEMBLY_ID=${ASSEMBLY_ID:-null}
PROJECT_ID=${PROJECT_ID:-null}
BUILD_ID=${BUILD_ID:-null}
TASKNAME=build
REUSE_IMAGES_IF_REPO_UNCHANGED=${REUSE_IMAGES_IF_REPO_UNCHANGED:="0"}
USER_PARAMS=${USER_PARAMS:-null}
SOLUM_PARAMS=${SOLUM_PARAMS:-null}
GIT_PRIVATE_KEY=${REPO_DEPLOY_KEYS:-''}
RUN_CMD=${RUN_CMD:-''}
DELETE_LOCAL_CACHE=${DELETE_LOCAL_CACHE:-null}
LP_ACCESS=${ACCESS:-null}

OS_AUTH_TOKEN=${OS_AUTH_TOKEN:-null}
OS_REGION_NAME=${OS_REGION_NAME:-null}
OS_STORAGE_URL=${OS_STORAGE_URL:-null}

TEMP_URL_SECRET=${TEMP_URL_SECRET:-null}
TEMP_URL_PROTOCOL=${TEMP_URL_PROTOCOL:-null}
TEMP_URL_TTL=${TEMP_URL_TTL:-null}

OPR_LP_DOWNLOAD_STRATEGY=${OPR_LP_DOWNLOAD_STRATEGY:-null}
OPER_AUTH_TOKEN=${OPER_AUTH_TOKEN:-null}
OPER_OS_STORAGE_URL=${OPER_OS_STORAGE_URL:-null}

# TLOG, PRUN, etc. defined in common/utils
HERE=$(dirname $0)
source $HERE/../../common/utils

LOG_FILE=$(GET_LOGFILE)

# Get the image_id of the image named $1
function app_glance_id () {
  glance --os-image-api-version 2 --os-auth-token $OS_AUTH_TOKEN image-list --sort-key updated_at --sort-dir asc | grep $1 | grep -v "+--" | tail -1 | cut -d'|' -f2
}

function cleanup_on_exit () {
  if [[ $DELETE_LOCAL_CACHE == "true" ]]; then
    # Not Deleting languagepack image because we found it conflicts with docker load in
    # performance tests, which might be due to a concurrency bug in docker load and rmi.
    #if [[ -n $BASE_IMG ]]; then
    #  sudo docker rmi -f $BASE_IMG
    #fi

    # Delete DU image after uploading to backend
    if [[ -n $DU_IMG_TAG ]]; then
      sudo docker rmi $DU_IMG_TAG
    fi

    # Delete temp output files
    if [[ -n $APP_DU_FILE ]] && [[ -e $APP_DU_FILE ]]; then
      rm -f $APP_DU_FILE
    fi

    if [[ -n $OUTPUT ]] && [[ -e $OUTPUT ]]; then
      rm -f $OUTPUT
    fi

    # Delete the cloned git repo
    if [[ -n $APP_DIR ]]; then
      rm -rf $APP_DIR
    fi

    if [[ -n $TMP_APP_DIR ]]; then
      rm -rf $TMP_APP_DIR
    fi

    remove_ssh_creds "$GIT_PRIVATE_KEY"
  fi
}

TLOG ===== Starting Build Script $0 $*

# Make sure tenant auth credentials were passed in.
if [[ -z "$OS_AUTH_TOKEN" ]]; then
  TLOG OpenStack credentials not passed via ENV. && exit 1
fi

# Check command line arguments
if [[ $# -lt 6 ]]; then
  TLOG Usage: $0 git_url commit_sha appname project_id img_external_ref lp_img_tag && exit 1
fi

PRUN silent sudo docker ps
[[ $? != 0 ]] && TLOG cannot talk to docker. && exit 1


GIT=$1
shift
COMMIT_ID=$1
shift
APP=$1
shift
TENANT=$1
shift
IMG_EXTERNAL_REF=$1
shift
LP_IMG_TAG=$1

if ! (test_public_repo $GIT); then
    TLOG Could not reach $GIT with curl. Failing. && exit 1
fi

BASE_DIR=/dev/shm
GIT_CHECKSUM=$(git ls-remote $GIT | head -1 | awk '{print $1}')
TS=$(date +"%Y%m%dt%H%M%S%N")
APP_DIR="$BASE_DIR/apps/$TENANT/$ASSEMBLY_ID"
TMP_APP_DIR="/tmp/apps/$TENANT/$ASSEMBLY_ID"
mkdir -p $APP_DIR
mkdir -p $TMP_APP_DIR

add_ssh_creds "$GIT_PRIVATE_KEY" "$APP_DIR"
[[ $? != 0 ]] && TLOG FAILED to register ssh key with ssh-agent && exit 1

trap cleanup_on_exit EXIT

if [[ -d "$APP_DIR/build" ]]; then
  cd $APP_DIR/build
  OUT=$(git pull | grep -c 'Already up-to-date')
  # Check to see if this is the same as last build, and don't rebuild if allowed to skip
  if [[ "$OUT" != "0" ]]; then
    if [[ "$REUSE_IMAGES_IF_REPO_UNCHANGED" -eq "1" ]]; then
      image_id=$(app_glance_id $APP)
      if [[ ${#image_id} == 36 ]]; then # uuid4 is 36 characters
        TLOG Repo is unchanged. Reusing image $image_id.
        TLOG created_image_id=$image_id
        # Need stdout for solum-worker to parse the image_id
        echo created_image_id=$image_id
        TOTAL_TIME=$(elapsed $SCRIPT_START_TIME)
        TLOG ===== Total elapsed time: $TOTAL_TIME sec
        exit 0
      fi
    fi
  fi
else
  git_clone_with_commit_sha_retry $GIT $APP_DIR/build $COMMIT_ID
  [[ $? != 0 ]] && TLOG Git clone failed. Check repo $GIT && exit 1
fi

cd $APP_DIR/build
if [[ "$COMMIT_ID" == "" ]]; then
  COMMIT_ID=$(git log -1 --pretty=%H)
fi

# if $RUN_CMD contains a run script in repo ($APP_DIR/build)
# and not executable, make it executable
# docker ADD and COPY will inherit permissions
run_cmd_prefix="./"
if [[ -n $RUN_CMD ]] && [[ "$RUN_CMD" == $run_cmd_prefix* ]]; then
  script_name=${RUN_CMD:2}
  if [[ -e "$script_name" ]] && [[ ! -x "$script_name" ]]; then
    chmod +x "$script_name"
  fi
fi

# If languagepack is 'auto', build the application slug
if [[ $IMG_EXTERNAL_REF == "auto" ]]; then
  TLOG "===>" Building App
  BUILD_ID=$(git archive master | sudo docker run -i -a stdin \
             -v /opt/solum/cache:/tmp/cache:rw  \
             -v /opt/solum/buildpacks:/tmp/buildpacks:rw  \
             solum/slugbuilder)

  PRUN sudo docker logs --tail=all -f $BUILD_ID

  cd $APP_DIR
  PRUN sudo docker cp $BUILD_ID:/tmp/slug.tgz $APP_DIR/
  if [[ ! -f "$APP_DIR/slug.tgz" ]]; then
    TLOG Slug build failed see container: $BUILD_ID && exit 1
  fi
  sudo docker rm -f $BUILD_ID
else
  # download base image (languagepack) if it is not 'auto'
  TLOG downloading LP image from $IMAGE_STORAGE
  if [[ $IMAGE_STORAGE == "glance" ]]; then
    OUTPUT="$TMP_APP_DIR/$LP_IMG_TAG"
    PRUN silent glance --os-image-api-version 2 --os-auth-token $OS_AUTH_TOKEN image-list
    if [[ $? != 0 ]]; then
      TLOG Cannot talk to Glance. Check your OpenStack credentials. && exit 1
    fi

    glance --os-image-api-version 2 --os-auth-token $OS_AUTH_TOKEN image-download --file $OUTPUT $IMG_EXTERNAL_REF
    if [[ $? != 0 ]]; then
      TLOG Failed to download image $IMG_EXTERNAL_REF from glance. && exit 1
    fi

    docker_load_with_retry $OUTPUT
    if [[ $? != 0 ]]; then
      OUTPUT_FILE_SIZE=$(stat -c%s "$OUTPUT")
      TLOG Failed docker load, file size $OUTPUT_FILE_SIZE && exit 1
    fi

    BASE_IMG=$LP_IMG_TAG
  elif [[ $IMAGE_STORAGE == "swift" ]]; then
    OUTPUT="$TMP_APP_DIR/$LP_IMG_TAG"
    if [[ $LP_ACCESS == "custom" ]]; then
      python $HERE/swift-handler.py $OS_REGION_NAME $OS_AUTH_TOKEN $OS_STORAGE_URL download solum_lp $IMG_EXTERNAL_REF $OUTPUT \
      > >(while read ALINE; do TLOG $ALINE; done)
    elif [[ $LP_ACCESS == "operator" ]]; then
      if [[ $OPR_LP_DOWNLOAD_STRATEGY == "swift-client" ]]; then
        python $HERE/swift-handler.py $OS_REGION_NAME $OPER_AUTH_TOKEN $OPER_OS_STORAGE_URL download solum_lp $IMG_EXTERNAL_REF $OUTPUT \
        > >(while read ALINE; do TLOG $ALINE; done)
      elif [[ $OPR_LP_DOWNLOAD_STRATEGY == "wget" ]]; then
        wget -q "$IMG_EXTERNAL_REF" --output-document=$OUTPUT
      fi
    fi
    if [[ $? != 0 ]]; then
      TLOG Failed to download image $IMG_EXTERNAL_REF from swift. && exit 1
    fi

    docker_load_with_retry $OUTPUT
    if [[ $? != 0 ]]; then
      OUTPUT_FILE_SIZE=$(stat -c%s "$OUTPUT")
      TLOG Failed docker load, file size $OUTPUT_FILE_SIZE && exit 1
    fi

    BASE_IMG=$LP_IMG_TAG
  elif [[ $IMAGE_STORAGE == "docker_registry" ]]; then
    sudo docker pull $IMG_EXTERNAL_REF
    if [[ $? != 0 ]]; then
      TLOG Failed to download image $IMG_EXTERNAL_REF from docker registry. && exit 1
    fi

    BASE_IMG=$IMG_EXTERNAL_REF
  else
    TLOG Unsupported Image storage backend - $IMAGE_STORAGE && exit 1
  fi
fi

DOCKER_RUN_CMD=$RUN_CMD

# copy params to the working dir
EXT=$(mktemp -u XXX | head -n 1)
mkdir $APP_DIR/build/params_$EXT
if [[ $USER_PARAMS != null ]]; then
  cp $USER_PARAMS $APP_DIR/build/params_$EXT/user_params
  DOCKER_RUN_CMD="[\"/bin/bash\", \"-c\", \"source params_$EXT/user_params && $RUN_CMD\"]"
fi
if [[ $SOLUM_PARAMS != null ]]; then
  cp $SOLUM_PARAMS $APP_DIR/build/params_$EXT/solum_params
fi

if [[ $IMG_EXTERNAL_REF == "auto" ]]; then
# Build the application image by injecting slug into runner
  TLOG Creating Dockerfile
  cat << EOF > $APP_DIR/Dockerfile
# SOLUM APP BUILDER
FROM solum/slugrunner
ADD slug.tgz /app
COPY build/params_$EXT /root/params
ENTRYPOINT ["/runner/init"]
CMD ["start","web"]
EOF
else
  TLOG Creating Dockerfile
  cat << EOF > $APP_DIR/Dockerfile
FROM $BASE_IMG
COPY build /app
WORKDIR /solum/bin
RUN chmod +x build.sh
RUN ./build.sh
WORKDIR /app
CMD $DOCKER_RUN_CMD
EOF
fi

cd $APP_DIR
DU_IMG_TAG="$TENANT-$APP-$TS-$COMMIT_ID"
STORAGE_OBJ_NAME="$APP-$TS-$COMMIT_ID"

echo "build/.git" > .dockerignore

if [[ $IMAGE_STORAGE == "glance" ]]; then

  docker_build_with_retry $DU_IMG_TAG .
  [[ $? != 0 ]] && TLOG Docker build failed. && exit 1

  glance_upload_with_retry $DU_IMG_TAG
  image_id="$(app_glance_id $DU_IMG_TAG)"
  TLOG ===== finished uploading DU to $IMAGE_STORAGE
elif [[ $IMAGE_STORAGE == "docker_registry" ]]; then

  DOCKER_REGISTRY=${DOCKER_REGISTRY:-'10.0.2.15:5042'}
  APP_NAME=$DOCKER_REGISTRY/$DU_IMG_TAG

  docker_build_with_retry $APP_NAME .
  [[ $? != 0 ]] && TLOG Docker build failed. && exit 1

  sudo docker push $APP_NAME
  [[ $? != 0 ]] && TLOG Docker push failed. && exit 1

  # just to make worker/shell easier to process
  image_id="${APP_NAME}"
  sudo docker rmi -f $APP_NAME
  TLOG ===== finished uploading DU to $IMAGE_STORAGE
elif [[ $IMAGE_STORAGE == "swift" ]]; then
  docker_build_with_retry $DU_IMG_TAG .
  if [[ $? != 0 ]]; then
    TLOG Docker build failed. && exit 1
  fi

  APP_DU_FILE="$TMP_APP_DIR/$STORAGE_OBJ_NAME"
  docker_save_with_retry $APP_DU_FILE $DU_IMG_TAG
  if [[ $? != 0 ]]; then
    TLOG Docker save failed. && exit 1
  fi

  #TODO(devkulkarni): Read the SECRET and TTL from config file
  SECRET=secret
  TTL=604800

  python $HERE/swift-handler.py $OS_REGION_NAME $OS_AUTH_TOKEN $OS_STORAGE_URL upload solum_du $STORAGE_OBJ_NAME $APP_DU_FILE \
  > >(while read ALINE; do TLOG $ALINE; done)
  if [[ $? != 0 ]]; then
    TLOG Swift upload failed. && exit 1
  fi

  ACCOUNT=$(echo $OS_STORAGE_URL | sed 's/\// /'g | awk '{print $4}')
  TLOG "ACCOUNT=$ACCOUNT"
  STORAGE_HOST=$(echo $OS_STORAGE_URL | sed 's/\// /'g | awk '{print $2}')
  TLOG "STORAGE_HOST=$STORAGE_HOST"
  curl -i -X POST -H X-Auth-Token:$OS_AUTH_TOKEN -H X-Account-Meta-Temp-URL-Key:$TEMP_URL_SECRET $TEMP_URL_PROTOCOL://$STORAGE_HOST/v1/$ACCOUNT
  TLOG "HERE:$HERE"

  TEMP_URL=$(python $HERE/get-temp-url.py $STORAGE_HOST solum_du $STORAGE_OBJ_NAME $ACCOUNT $TEMP_URL_SECRET $TEMP_URL_TTL $TEMP_URL_PROTOCOL)
  TLOG "TEMP_URL:$TEMP_URL"

  image_id="${TEMP_URL}"
  TLOG ===== finished uploading DU to $IMAGE_STORAGE
else
  TLOG Unsupported Image storage backend - $IMAGE_STORAGE && exit 1
fi

TOTAL_TIME=$(elapsed $SCRIPT_START_TIME)
TLOG ===== Total elapsed time: $TOTAL_TIME sec

TLOG created_image_id=$image_id
TLOG docker_image_name=$DU_IMG_TAG

# Need stdout for solum-worker to parse the image_id
echo created_image_id=$image_id
echo docker_image_name=$DU_IMG_TAG

exit 0