Permalink
Browse files

Merge "Add GIT_TIMEOUT variable to watch git operations"

  • Loading branch information...
2 parents df43b5c + d53ad0b commit cf7237c505689055521a23d9ba4c18c116fec9c6 Jenkins committed with openstack-gerrit Feb 27, 2014
Showing with 47 additions and 5 deletions.
  1. +36 −5 functions-common
  2. +11 −0 stackrc
View
@@ -527,16 +527,16 @@ function git_clone {
if [[ ! -d $GIT_DEST ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
- git clone $GIT_REMOTE $GIT_DEST
+ git_timed clone $GIT_REMOTE $GIT_DEST
fi
cd $GIT_DEST
- git fetch $GIT_REMOTE $GIT_REF && git checkout FETCH_HEAD
+ git_timed fetch $GIT_REMOTE $GIT_REF && git checkout FETCH_HEAD
else
# do a full clone only if the directory doesn't exist
if [[ ! -d $GIT_DEST ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
- git clone $GIT_REMOTE $GIT_DEST
+ git_timed clone $GIT_REMOTE $GIT_DEST
cd $GIT_DEST
# This checkout syntax works for both branches and tags
git checkout $GIT_REF
@@ -545,7 +545,7 @@ function git_clone {
cd $GIT_DEST
# set the url to pull from and fetch
git remote set-url origin $GIT_REMOTE
- git fetch origin
+ git_timed fetch origin
# remove the existing ignored files (like pyc) as they cause breakage
# (due to the py files having older timestamps than our pyc, so python
# thinks the pyc files are correct using them)
@@ -570,6 +570,37 @@ function git_clone {
git show --oneline | head -1
}
+# git can sometimes get itself infinitely stuck with transient network
+# errors or other issues with the remote end. This wraps git in a
+# timeout/retry loop and is intended to watch over non-local git
+# processes that might hang. GIT_TIMEOUT, if set, is passed directly
+# to timeout(1); otherwise the default value of 0 maintains the status
+# quo of waiting forever.
+# usage: git_timed <git-command>
+function git_timed() {
+ local count=0
+ local timeout=0
+
+ if [[ -n "${GIT_TIMEOUT}" ]]; then
+ timeout=${GIT_TIMEOUT}
+ fi
+
+ until timeout -s SIGINT ${timeout} git "$@"; do
+ # 124 is timeout(1)'s special return code when it reached the
+ # timeout; otherwise assume fatal failure
+ if [[ $? -ne 124 ]]; then
+ die $LINENO "git call failed: [git $@]"
+ fi
+
+ count=$(($count + 1))
+ warn "timeout ${count} for git call: [git $@]"
+ if [ $count -eq 3 ]; then
+ die $LINENO "Maximum of 3 git retries reached"
+ fi
+ sleep 5
+ done
+}
+
# git update using reference as a branch.
# git_update_branch ref
function git_update_branch() {
@@ -600,7 +631,7 @@ function git_update_tag() {
git tag -d $GIT_TAG
# fetching given tag only
- git fetch origin tag $GIT_TAG
+ git_timed fetch origin tag $GIT_TAG
git checkout -f $GIT_TAG
}
View
11 stackrc
@@ -80,6 +80,17 @@ fi
# (currently only implemented for MySQL backend)
DATABASE_QUERY_LOGGING=$(trueorfalse True $DATABASE_QUERY_LOGGING)
+# Set a timeout for git operations. If git is still running when the
+# timeout expires, the command will be retried up to 3 times. This is
+# in the format for timeout(1);
+#
+# DURATION is a floating point number with an optional suffix: 's'
+# for seconds (the default), 'm' for minutes, 'h' for hours or 'd'
+# for days.
+#
+# Zero disables timeouts
+GIT_TIMEOUT=${GIT_TIMEOUT:-0}
+
# Repositories
# ------------

0 comments on commit cf7237c

Please sign in to comment.