From 4073376b68756350142ea16a36ddebca79a9b3bb Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Tue, 24 Jan 2017 10:59:55 -0500 Subject: [PATCH 1/2] Don't give up copying from `hack/env` on failure Signed-off-by: Steve Kuznetsov --- hack/lib/build/environment.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hack/lib/build/environment.sh b/hack/lib/build/environment.sh index d8f63180cfbe..0fef3949433e 100644 --- a/hack/lib/build/environment.sh +++ b/hack/lib/build/environment.sh @@ -124,7 +124,10 @@ function os::build::environment::start() { mkdir -p "${parent}" fi os::log::debug "Copying from ${container}:${workingdir}/${path} to ${parent}" - docker cp "${container}:${workingdir}/${path}" "${parent}" + if ! output="$( docker cp "${container}:${workingdir}/${path}" "${parent}" 2>&1 )"; then + os::log::warn "Copying ${path} from the container failed!" + os::log::warn "${output}" + fi done IFS="${oldIFS}" fi From ccaa821074c8f0571df4c9fb5b4db78bfb859395 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Tue, 24 Jan 2017 13:36:29 -0500 Subject: [PATCH 2/2] Added an option to `hack/env` to use a new volume When doing iteration on `hack/env`, especially with `tito` builds that change the state of the git repository on the local volume used with `hack/env` without changing the git state of the repository on the local system, it is necessary to force `hack/env` to use a new volume. Signed-off-by: Steve Kuznetsov --- hack/env | 3 +++ hack/lib/build/environment.sh | 34 ++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/hack/env b/hack/env index e5b27f21d6d3..a60871d35d46 100755 --- a/hack/env +++ b/hack/env @@ -26,6 +26,9 @@ # $ hack/env make # slow # $ hack/env make # fast! # +# # force a new volume to get created from the current source +# $ OS_BUILD_ENV_VOLUME_FORCE_NEW=TRUE hack/env +# # NOTE: only committed code is built. source "$(dirname "${BASH_SOURCE}")/lib/init.sh" diff --git a/hack/lib/build/environment.sh b/hack/lib/build/environment.sh index 0fef3949433e..2592a25186c6 100644 --- a/hack/lib/build/environment.sh +++ b/hack/lib/build/environment.sh @@ -19,15 +19,18 @@ function os::build::environment::create() { local workingdir workingdir=$( os::build::environment::release::workingdir ) additional_context+=" -v ${OS_ROOT}:${workingdir} -u $(id -u)" - elif [[ -n "${OS_BUILD_ENV_REUSE_VOLUME:-}" ]]; then - # if OS_BUILD_ENV_REUSE_VOLUME is set, create a docker volume to store the working output so - # successive iterations can reuse shared code. - os::log::debug "Re-using volume ${OS_BUILD_ENV_REUSE_VOLUME}" + elif [[ -n "${OS_BUILD_ENV_VOLUME:-}" ]]; then + if docker volume inspect "${OS_BUILD_ENV_VOLUME}" >/dev/null 2>&1; then + os::log::debug "Re-using volume ${OS_BUILD_ENV_VOLUME}" + else + # if OS_BUILD_ENV_VOLUME is set and no volume already exists, create a docker volume to + # store the working output so successive iterations can reuse shared code. + os::log::debug "Creating volume ${OS_BUILD_ENV_VOLUME}" + docker volume create --name "${OS_BUILD_ENV_VOLUME}" > /dev/null + fi local workingdir workingdir=$( os::build::environment::release::workingdir ) - name="$( echo "${OS_BUILD_ENV_REUSE_VOLUME}" | tr '[:upper:]' '[:lower:]' )" - docker volume create --name "${name}" > /dev/null - additional_context+=" -v ${name}:${workingdir}" + additional_context+=" -v ${OS_BUILD_ENV_VOLUME}:${workingdir}" fi fi @@ -169,11 +172,9 @@ function os::build::environment::withsource() { excluded+=("--exclude=${exclude}") done IFS="${oldIFS}" - if which rsync &>/dev/null && [[ -n "${OS_BUILD_ENV_REUSE_VOLUME-}" ]]; then - local name - name="$( echo "${OS_BUILD_ENV_REUSE_VOLUME}" | tr '[:upper:]' '[:lower:]' )" + if which rsync &>/dev/null && [[ -n "${OS_BUILD_ENV_VOLUME-}" ]]; then os::log::debug "Syncing source using \`rsync\`" - if ! rsync -a --blocking-io "${excluded[@]}" --delete --omit-dir-times --numeric-ids -e "docker run --rm -i -v \"${name}:${workingdir}\" --entrypoint=/bin/bash \"${OS_BUILD_ENV_IMAGE}\" -c '\$@'" . remote:"${workingdir}"; then + if ! rsync -a --blocking-io "${excluded[@]}" --delete --omit-dir-times --numeric-ids -e "docker run --rm -i -v \"${OS_BUILD_ENV_VOLUME}:${workingdir}\" --entrypoint=/bin/bash \"${OS_BUILD_ENV_IMAGE}\" -c '\$@'" . remote:"${workingdir}"; then os::log::debug "Falling back to \`tar\` and \`docker cp\` as \`rsync\` is not in container" tar -cf - "${excluded[@]}" . | docker cp - "${container}:${workingdir}" fi @@ -194,12 +195,21 @@ function os::build::environment::run() { if [[ -z "${volume}" ]]; then volume="origin-build-$( git rev-parse "${commit}" )" fi + volume="$( echo "${volume}" | tr '[:upper:]' '[:lower:]' )" + export OS_BUILD_ENV_VOLUME="${volume}" + + if [[ -n "${OS_BUILD_ENV_VOLUME_FORCE_NEW:-}" ]]; then + if docker volume inspect "${volume}" >/dev/null 2>&1; then + os::log::debug "Removing volume ${volume}" + docker volume rm "${volume}" + fi + fi os::log::debug "Using commit ${commit}" os::log::debug "Using volume ${volume}" local container - container="$( OS_BUILD_ENV_REUSE_VOLUME=${volume} os::build::environment::create "$@" )" + container="$( os::build::environment::create "$@" )" trap "os::build::environment::cleanup ${container}" EXIT os::log::debug "Using container ${container}"