Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add options for build container rsync optimization #36361

Merged
merged 1 commit into from Dec 11, 2016
Merged

Add options for build container rsync optimization #36361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 25 additions & 17 deletions build-tools/common.sh
View file
Open in desktop
Expand Up @@ -643,33 +643,45 @@ function kube::build::stop_rsyncd_container() {
kube::build::destroy_container "${KUBE_RSYNC_CONTAINER_NAME}"
}

function kube::build::rsync {
local -a rsync_opts=(
--archive
--prune-empty-dirs
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
)
if (( ${KUBE_VERBOSE} >= 6 )); then
rsync_opts+=("-iv")
fi
if (( ${KUBE_RSYNC_COMPRESS} > 0 )); then
rsync_opts+=("--compress-level=${KUBE_RSYNC_COMPRESS}")
fi
V=3 kube::log::status "Running rsync"
rsync "${rsync_opts[@]}" "$@"
}

# This will launch rsyncd in a container and then sync the source tree to the
# container over the local network.
function kube::build::sync_to_container() {
kube::log::status "Syncing sources to container"

kube::build::start_rsyncd_container

local rsync_extra=""
if (( ${KUBE_VERBOSE} >= 6 )); then
rsync_extra="-iv"
fi

# rsync filters are a bit confusing. Here we are syncing everything except
# output only directories and things that are not necessary like the git
# directory. The '- /' filter prevents rsync from trying to set the
# uid/gid/perms on the root of the sync tree.
V=3 kube::log::status "Running rsync"
rsync ${rsync_extra} \
--archive \
# directory and generated files. The '- /' filter prevents rsync
# from trying to set the uid/gid/perms on the root of the sync tree.
# As an exception, we need to sync generated files in staging/, because
# they will not be re-generated by 'make'.
kube::build::rsync \
--delete \
--prune-empty-dirs \
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
--filter='+ /staging/**' \
--filter='- /.git/' \
--filter='- /.make/' \
--filter='- /_tmp/' \
--filter='- /_output/' \
--filter='- /' \
--filter='- zz_generated.*' \
--filter='- generated.proto' \
"${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"

kube::build::stop_rsyncd_container
Expand All @@ -694,11 +706,7 @@ function kube::build::copy_output() {
#
# We are looking to copy out all of the built binaries along with various
# generated files.
V=3 kube::log::status "Running rsync"
rsync ${rsync_extra} \
--archive \
--prune-empty-dirs \
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
kube::build::rsync \
--filter='- /vendor/' \
--filter='- /_temp/' \
--filter='+ /_output/dockerized/bin/**' \
Expand Down
4 changes: 4 additions & 0 deletions hack/lib/init.sh
View file
Open in desktop
Expand Up @@ -25,6 +25,10 @@ KUBE_OUTPUT_SUBPATH="${KUBE_OUTPUT_SUBPATH:-_output/local}"
KUBE_OUTPUT="${KUBE_ROOT}/${KUBE_OUTPUT_SUBPATH}"
KUBE_OUTPUT_BINPATH="${KUBE_OUTPUT}/bin"

# This controls rsync compression. Set to a value > 0 to enable rsync
# compression for build container
KUBE_RSYNC_COMPRESS="${KUBE_RSYNC_COMPRESS:-0}"

# Set no_proxy for localhost if behind a proxy, otherwise,
# the connections to localhost in scripts will time out
export no_proxy=127.0.0.1,localhost
Expand Down