Permalink
Fetching contributors…
Cannot retrieve contributors at this time
executable file 4603 lines (4379 sloc) 177 KB
#!/bin/bash
export LANG="C"
export LC_ALL="C"
GEM_RE='([^0-9].*)-([0-9].*)'
PULL_BUNDLE_RE='Crowbar-Pull-ID: ([0-9a-f]{40})'
PULL_RELEASE_RE='Crowbar-Release: ([^ ]+)'
PULL_TITLE_RE='^(.*)\[([0-9]+)/([0-9]+)\]$'
readonly currdir="$PWD"
export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
# Ubuntu hack to make sure gem executables are in our path.
[[ -d /var/lib/gems/1.8/bin ]] && export PATH=$PATH:/var/lib/gems/1.8/bin
declare -A DEV_BRANCHES DEV_REMOTE_SOURCES DEV_REMOTE_BRANCHES DEV_ORIGIN_TYPES
declare -A DEV_REMOTE_PRIORITY DEV_REMOTE_URLBASE __AVAILABLE_REMOTES
declare -a DEV_SORTED_REMOTES
declare -A DEV_COMMANDS DEV_SHORT_HELP DEV_LONG_HELP
# The generation of the dev tool this is.
readonly DEV_VERSION=2
# The key -> value mapping in DEV_BRANCHES defines child -> parent relations
# between branches. A branch is considered a root branch if it has itself
# as a parent.
DEV_BRANCHES["master"]="master"
DEV_BRANCHES["openstack-os-build"]="master"
DEV_BRANCHES["hadoop-os-build"]="master"
DEV_BRANCHES["cloudera-os-build"]="master"
github_re='^(https?|ssh|git)://.*github\.com/([^/]+)'
# Associative array to handle skeleton of the state transition machine.
# This is roughly a reverse lookup table for use by ci_get_state().
# The state transition rules are implemented by ci_all_next_states()
declare -A CI_STATES
CI_STATES["new"]="failed needs-work"
CI_STATES["merge-testing"]="new"
CI_STATES["merge-tested"]="merge-testing"
# These are shortened versions of the real next states.
# The real next states are unit-(testing|tested)-release/build
CI_STATES["unit-testing"]="merge-tested"
CI_STATES["unit-tested"]="unit-testing"
# These are also shortened versions. The real states are:
# (build|smoke)-(testing|tested)-<release>/<build>/<os>
CI_STATES["build-testing"]="merge-tested unit-tested"
CI_STATES["build-tested"]="build-testing"
CI_STATES["smoke-testing"]="build-tested"
CI_STATES["smoke-tested"]="smoke-testing"
# These are the actual states.
CI_STATES["code-reviewing"]="smoke-tested"
CI_STATES["needs-work"]="code-reviewing"
CI_STATES["code-reviewed"]="code-reviewing"
CI_STATES["mergeable"]="code-reviewed"
CI_STATES["merged"]="mergeable"
CI_STATES["failed"]="all"
CI_STATES["closed"]="all"
# DEV_REMOTE_BRANCHES defines what branches in the main Crowbar repository
# should be pulled and synced with what remotes.
# Barclamps do care about remote branches.
DEV_REMOTE_BRANCHES["origin"]="master openstack-os-build hadoop-os-build cloudera-os-build"
VERBOSE=true
# Source our config file if we have one
[[ -f $HOME/.build-crowbar.conf ]] && \
. "$HOME/.build-crowbar.conf"
# Look for a local one.
[[ -f build-crowbar.conf ]] && \
. "build-crowbar.conf"
# Location of the Crowbar checkout we are building from.
[[ $CROWBAR_DIR ]] || CROWBAR_DIR="${0%/*}"
[[ $CROWBAR_DIR = /* ]] || CROWBAR_DIR="$currdir/$CROWBAR_DIR"
[[ -f $CROWBAR_DIR/build_crowbar.sh && -d $CROWBAR_DIR/.git ]] || \
die "$CROWBAR_DIR is not a git checkout of Crowbar!"
[[ $CI_TRACKING_REPO ]] || CI_TRACKING_REPO=ci-tracking
[[ $CROWBAR_TEST_DIR ]] || CROWBAR_TEST_DIR="/tmp/crowbar-dev-test"
[[ $LOCAL_PULL_TRACKING ]] || LOCAL_PULL_TRACKING="$CROWBAR_DIR/.ci-tracking"
[[ $OPEN_PULL_REQUESTS ]] || OPEN_PULL_REQUESTS="$LOCAL_PULL_TRACKING/local"
export CROWBAR_DIR CROWBAR_TEST_DIR LOCAL_PULL_TRACKING
export OPEN_PULL_REQUESTS
. "$CROWBAR_DIR/build_lib.sh" || exit 1
trap - 0 INT QUIT TERM
trap 'rm -rf "$CROWBAR_TMP"' 0 INT QUIT TERM
which gem &>/dev/null || \
die "Rubygems not installed, and some of our helpers need a JSON gem."
gem list -i json &>/dev/null || \
die "JSON gem not installed. Please install it with gem install json."
set -o pipefail
# A little wrapper for git to help people see what dev is doing.
git() {
[[ $SHOW_GIT_OPERATIONS ]] && debug "$PWD: git $@"
command git "$@"
}
# Test to see if a remote is available.
# Filters based on $DEV_AVAILABLE_REMOTES
remote_available() {
# $1 = remote to test.
# Returns 0 if the remote is available, 1 otherwise.
local r urlbase url_re='^(file|https?|ssh|git)://([^/]+)'
local -A remote_hash
if [[ ! $DEV_AVAILABLE_REMOTES ]]; then
crowbar_remote_exists "$1"
return $?
fi
if [[ ${__AVAILABLE_REMOTES[$1]} ]]; then
[[ ${__AVAILABLE_REMOTES[$1]} = true ]]
return $?
fi
# Handle "origin" specially as shorthand for $(origin_remote)
for r in ${DEV_AVAILABLE_REMOTES//origin/$(origin_remote)}; do
crowbar_remote_exists "$r" || \
die "Unknown remote $r in DEV_AVAILABLE_REMOTES!"
urlbase=${DEV_REMOTE_URLBASE[$r]}
[[ $urlbase =~ $url_re ]] || continue
remote_hash["${BASH_REMATCH[2]}"]=true
done
urlbase=${DEV_REMOTE_URLBASE[$1]}
[[ $urlbase =~ $url_re ]] || return 1
if [[ ${BASH_REMATCH[1]} = file || ${remote_hash["${BASH_REMATCH[2]}"]} ]]; then
__AVAILABLE_REMOTES[$1]=true
return 0
else
__AVAILABLE_REMOTES[$1]=false
return 1
fi
}
# The remote with the highest priority is considered to be the "origin" remote.
# This is a function and not a variable because not all the repos we work
# with will have the same origin remotes.
origin_remote() {
local r
for r in "${DEV_SORTED_REMOTES[@]}"; do
git_remote_exists "$r" || continue
echo "$r"
return 0
done
return 1
}
# Check out the appropriate branches in each barclamp needed to switch to
# the new build. This function takes care to minimize the amount of branch
# swizzling that occurs.
switch_barclamps_to_build() {
# $1 = new build
local bc new_head current_head
build_exists "$1" || die "Build $1 does not exist"
for bc in $(barclamps_in_build "$1"); do
[[ -d $CROWBAR_DIR/barclamps/$bc/.git ]] || \
die "Build $1 requires $bc, which is not available locally." \
"dev clone-barclamps should fetch it, if not you may not have access to it."
done
for bc in "$CROWBAR_DIR/barclamps/"*; do
bc=${bc##*/}
new_branch=$(barclamp_branch_for_build "$1" "$bc")
current_head=$(in_barclamp "$bc" git symbolic-ref HEAD 2>/dev/null || \
in_barclamp "$bc" git rev-parse HEAD)
if [[ $new_branch =~ [0-9a-f]{40} && $new_branch = $current_head ]]; then
# We want HEAD to be on a raw commit, and it is on the one we want.
continue
elif [[ $current_head = refs/heads/$new_branch ]]; then
# We want HEAD to be on a branch, and it is on the one we want.
continue
elif [[ $new_branch = empty-branch ]]; then
# We want to be on the empty branch, and we are not. Switch to the
# empty branch, creating it if we have to.
debug "Switching $bc to the empty branch."
in_barclamp "$bc" to_empty_branch
else
# We are not where we want to be. Go there.
debug "Switching $bc to $new_branch"
in_barclamp "$bc" quiet_checkout "$new_branch" && continue
die "$new_branch does not exist in $bc!" \
"You should try to fix it with a ./dev fetch followed by ./dev sync." \
"If that does not work, then someone forgot to push the branch when" \
"$1 was created."
fi
done
# Record that we are on the new build.
in_repo git config 'crowbar.release' "${1%/*}"
in_repo git config 'crowbar.build' "$1"
}
# Check out the appropriate release branch for all the barclamps in a given
# release or build.
switch_release() {
# $1 = build or release to switch to
local l br bc current_branch new_base rel repo
local -A barclamps
new_build="$(current_build)"
if build_exists "$1"; then
new_build="$1"
elif release_exists "$1"; then
new_build="$1/${new_build##*/}"
if ! build_exists "$new_build"; then
debug "Release $1 does not have build $new_build, switching to $1/master instead."
new_build="$1/master"
fi
elif [[ $1 ]]; then
die "$1 is not a release or a build I can switch to!"
fi
barclamps_are_clean || \
die "Crowbar repo must be clean before trying to switch releases!"
switch_barclamps_to_build "$new_build"
in_repo quiet_checkout master
for l in change-image extra; do
[[ $(in_repo readlink -f $l) = "releases/$new_build/$l" ]] && continue
in_repo rm -f "$l"
in_repo ln -sf "releases/$new_build/$l" "$l"
done
debug "Switched to $new_build"
}
# This function is kept around for legacy workflow reasons.
checkout() {
local br new_build rel
rel="$(current_release)"
new_build="$rel/$1"
switch_barclamps_to_build "$rel/$1"
}
# Test to see of we are on the right metadata version.
dev_is_setup() {
in_repo git_config_has crowbar.dev.version || [[ $1 = setup ]] || return 1
local thisrev
thisrev=$(get_repo_cfg crowbar.dev.version) && \
(( thisrev == DEV_VERSION )) || [[ $1 = setup ]] || {
VERBOSE=true
debug "Crowbar repository out of sync with dev tool revision."
debug "Please run $0 setup to update it."
exit 1
}
}
# Given a branch, print the first remote that branch is "owned" by.
# This assumes update_tracking_branches is keeping things up to date.
remote_for_branch() {
local -a remotes
local remote
if [[ ! $DEV_FROM_REMOTES ]]; then
if git_config_has "branch.$1.remote"; then
git config --get "branch.$1.remote"
return $?
else
remotes=("${DEV_SORTED_REMOTES[@]}")
fi
fi
[[ $DEV_FROM_REMOTES ]] && remotes=("${DEV_FROM_REMOTES[@]}")
for remote in "${remotes[@]}"; do
git show-ref --quiet --verify "refs/remotes/$remote/$1" || continue
echo "$remote"
return 0
done
return 1
}
# Test to see if a specific repository is clean.
# Ignores submodules and unchecked-in directories that are git repositories.
git_is_clean() {
local line hpath ret=0 opt
local stat_cmd="git status --porcelain" quiet=''
local paths=()
while [[ $1 ]]; do
opt="$1"; shift
case $opt in
--barclamps) stat_cmd="git status --porcelain";;
-q|--quiet) quiet=true;;
--paths)
while [[ $1 && $1 != '-'* ]]; do
paths+=("$1")
shift
done;;
*) die "Unknown option $opt passed to git_is_clean.";;
esac
done
[[ $paths ]] && stat_cmd+=" -- ${paths[*]}"
while read line; do
case $line in
# Untracked file. Ignore it if it is also a git repo,
# complain otherwise.
'??'*) hpath=${line%% ->*}
hpath=${hpath#* }
[[ -d $PWD/$hpath.git || -f $PWD/$hpath.git || $hpath = barclamps/ ]] && continue
ret=1; break;;
'') continue;;
*) ret=1; break;;
esac
done < <($stat_cmd)
[[ $ret = 0 ]] && return
[[ $quiet ]] || {
echo "$PWD:"
git status -- "${paths[@]}"
}
[[ $IGNORE_CLEAN ]] && return 0
return 1
}
# Stupid wrapper around git push for debugging.
git_push() {
if [[ $DEBUG = true || $DRY_RUN ]]; then
echo "would have done git push $@"
return
fi
git push "$@"
}
# Test to see if a barclamp is clean.
barclamp_is_clean() {
local bc="$1"; shift
in_barclamp "$bc" git_is_clean "$@"
}
# Test to see if all our barclamps are clean.
barclamps_are_clean() {
local bc res=0
for bc in "$CROWBAR_DIR/barclamps/"*; do
is_barclamp "${bc##*/}" || continue
in_barclamp "${bc##*/}" git_is_clean "$@" || res=1
done
return $res
}
# Test to see if all the Crowbar repositories are clean.
crowbar_is_clean() {
local res=0
barclamps_are_clean "$@" && in_repo git_is_clean "$@" && return 0
echo "Your crowbar repositories are not clean."
echo "Please review the git status output above, and add and commit/stash as needed."
return 1
}
# Check to see if a remote exists.
test_remote() { git ls-remote "$1" "refs/heads/master" &> /dev/null; }
# Fork a barclamp on Github.
fork_barclamp() {
# $1 = remote to fork from. Must be a github remote.
# $2 = barclamp to fork.
# $3 = remote on Github to fork to. Must be on Github. and defaults to personal.
local from_remote="${DEV_REMOTE_URLBASE[$1]}"
local to_remote="${DEV_REMOTE_URLBASE[${3:-personal}]}"
if ! [[ $from_remote && $from_remote =~ $github_re ]]; then
die "Source remote $1 is not Github remote!"
elif ! [[ $to_remote && $to_remote =~ $github_re ]]; then
[[ $3 ]] && die "Target remote $3 is not a Github remote!"
[[ $to_remote ]] || die "Personal remote not configured, cannot fork $2 to it."
die "Personal remote does not point at a Github remote, cannot fork $2 to it."
elif [[ $to_remote != */$DEV_GITHUB_ID ]]; then
die "Remote ${3:-personal} does not map to your Github account!"
fi
test_remote "$to_remote/barclamp-$2.git" && return 0 # already forked
test_remote "$from_remote/barclamp-$2.git" || die "Barclamp $2 does not exist at remote $1"
github_fork "${DEV_REMOTE_URLBASE[$1]##*/}" "barclamp-$2"
}
# Look for barclamps that a build references, but that we don't have locally
find_missing_barclamps() {
# $1 = '', release, release/build
local bc barclamps=()
if build_exists "$1"; then
barclamps=($(barclamps_in_build "$1"))
elif release_exists "$1"; then
barclamps=($(barclamps_in_release "$1"))
else
barclamps=($(all_barclamps))
fi
for bc in "${barclamps[@]}"; do
bc="$CROWBAR_DIR/barclamps/$bc"
[[ -d $bc/.git || -f $bc/.git ]] && continue
echo "${bc##*/}"
done
}
# Look for barclamps that exist locally, but that are not referenced by
# any local builds.
find_orphaned_barclamps() {
local bc
local -A barclamps
for bc in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
barclamps["${bc##*/}"]=present
done
for bc in $(all_barclamps); do
[[ ${barclamps[$bc]} ]] && continue
echo "$bc"
done
}
# Check to see if a barclamp exists at a given remote.
# Barclamps can either exist at:
# barclamp-$1 for Github, Bitbucket and the like, or
# barclamps/$1 for trees that have been layed out by dev.
# We test for both to make it possible to use dev setup for local clones.
probe_barclamp_remote() {
# $1 = barclamp
# $2 = urlpart
local remote
for remote in "barclamp-$1" "crowbar/barclamps/$1"; do
test_remote "$2/$remote" || continue
echo "$remote"
return 0
done
return 1
}
# Perform an initial clone of a barclamp.
# Takes care to ensure that the origin remote is set appropriatly.
# This will also create a personal remote if needed.
clone_and_sync_barclamp() {
# $1 = name of the barclamp
local repo remote build head urlbase
if ! [[ -d $CROWBAR_DIR/barclamps/$1/.git || \
-f $CROWBAR_DIR/barclamps/$1/.git ]]; then
for remote in "${DEV_SORTED_REMOTES[@]}"; do
urlbase="${DEV_REMOTE_URLBASE[$remote]}"
repo="$(probe_barclamp_remote "$1" "$urlbase")" && break
done
[[ $repo ]] || return 1
in_repo git clone -l -o "$remote" "$urlbase/$repo" "barclamps/$1" || {
rm -rf "$CROWBAR_DIR/barclamps/$1"
die "Unable to clone barclamp $bc from $urlbase/$repo.git"
}
fi
[[ -f $CROWBAR_DIR/barclamps/$1/.git && \
! -d $CROWBAR_DIR/barclamps/$1/.git ]] && (
cd "$CROWBAR_DIR/barclamps/$1/"
debug "De-submoduleizing barclamp ${1}"
read gpath < ".git"
if [[ $gpath = 'gitdir: '* ]]; then
rm -f ".git"
mv "${gpath#gitdir: }" ".git"
(export GIT_WORK_TREE=.; git config --unset core.worktree)
git reset --hard HEAD
git clean -f -x -d
else
echo "Malformed .git file in $1. Skipping." >&2
fi
)
sync_barclamp_remotes "$1"
[[ $remote ]] || remote=$(in_barclamp "$1" origin_remote) || \
die "Cannot find origin remote for barclamp $1!"
urlbase="${DEV_REMOTE_URLBASE[$remote]}"
if [[ $urlbase =~ $github_re ]] && remote_available personal && \
! in_barclamp "$1" git_remote_exists personal; then
# test to see if we need to fork this barclamp at Github
if ! github_repo_exists "barclamp-$1"; then
debug "Creating a personal fork of barclamp-$1"
fork_barclamp "$remote" "$1" || return 2
sync_barclamp_remotes "$1"
fi
fi
in_barclamp "$1" git_remote_exists origin && in_barclamp "$1" git remote rm origin
in_barclamp "$1" update_tracking_branches
}
# Clone and synchronize remotes for barclamps on the command line, as needed.
clone_barclamps() {
# $@ = barclamps to clone. If none are passed, defaults to missing ones.
local barclamps=() bc
if [[ $1 = all ]]; then
barclamps=($(all_barclamps))
elif [[ $1 ]]; then
barclamps=("$@")
else
barclamps=($(find_missing_barclamps))
fi
for bc in "${barclamps[@]}"; do
clone_and_sync_barclamp "$bc" && continue
case $? in
1) debug "Unable to find barclamp $bc in any available remotes.";;
2) debug "Unable to create a personal fork of barclamp $bc";;
esac
done
# Create a fork of Crowbar if we just created a personal remote.
if crowbar_remote_exists personal && ! github_repo_exists crowbar; then
echo "Creating your fork of Crowbar on Github."
github_fork "$(origin_remote)" crowbar || \
die "Unable to create your fork of Crowbar."
fi
}
# Check out a branch, but be quiet about it unless something goes wrong.
quiet_checkout() {
local res=''
res=$(git checkout -q "$@" 2>&1) && return
echo "$res" >&2
return 1
}
# Update tracking branches for all remotes in a specific repo.
# Expects to be called within a specific repository
# It is structured to minimize forking, please be careful modifying it.
__update_tracking_branches() {
# Create tracking branches for any branches from this remote
# that do not already exist.
local remote p br in_tracking_update=true
local -A branches
while read p br; do
# We never care about HEAD branches at all.
[[ ${br##*/} = HEAD ]] && continue
if [[ $br = refs/heads/* ]]; then
br="${br#refs/heads/}"
if [[ $br = personal/* ]]; then
# This should not be here. Kill it.
git branch -D "$br"
# Nuke any config that might have come from a remote we don't care about.
git_config_has "branch.$br.remote" || continue
git config --remove-section "branch.$br"
continue
fi
# Record that we have seen this branch, but don't know what its remote should be.
branches["$br"]="no remote"
continue
elif [[ $br = refs/remotes/* ]]; then
br="${br#refs/remotes/}"
# Grab our remote, and go on to the next ref if we don't care about it.
remote="${br%%/*}"
[[ ${remotes[$remote]} ]] || continue
br="${br#${remote}/}"
# Skip personal or pull-request branches.
[[ $br = personal/* || $br = pull-req* ]] && continue
# If we have never seen this branch before, or
# we don't have a remote for it, or
# the remote we have for it is lower priority than our remote, then
# give this branch us as a remote instead.
if [[ ! ${branches[$br]} || \
${branches[$br]} = "no remote" ]] || \
(( ${remotes[${branches[$br]}]} > ${remotes[$remote]} )); then
branches[$br]="$remote"
fi
fi
done < <(LC_ALL=C git show-ref |sort -k2) # Ensure that heads come first!
# Now, we have our list of branches. Operate on it.
for br in "${!branches[@]}"; do
remote="${branches[$br]}"
if ! branch_exists "$br"; then
# We need to create a local ref for this branch.
git branch "$br" "$remote/$br"
git config "branch.$br.remote" "$remote"
git config "branch.$br.merge" "refs/heads/$br"
elif [[ $remote = "no remote" ]]; then
continue
else
git config --remove-section "branch.$br"
git config "branch.$br.remote" "$remote"
git config "branch.$br.merge" "refs/heads/$br"
fi
done
}
update_tracking_branches() {
local p=0 remote
local -A remotes
for remote in "${DEV_SORTED_REMOTES[@]}"; do
remotes[$remote]="$p"
p=$((p + 1))
done
__update_tracking_branches
}
update_cache_tracking_branches() {
local -A remotes
remotes["origin"]=0
in_cache __update_tracking_branches
}
# Update tracking references for all branches in all the
# repositories that dev is managing.
update_all_tracking_branches() {
local bc
debug "Updating tracking branch references in Crowbar"
in_repo update_tracking_branches
for bc in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
debug "Updating tracking branch references in barclamp ${bc##*/}"
(cd "$bc"; update_tracking_branches)
done
if [[ -d $LOCAL_PULL_TRACKING/.git ]]; then
debug "Updating tracking branch references in the CI tracking repo."
(cd "$LOCAL_PULL_TRACKING"; update_tracking_branches)
fi
}
# Helper for sourcing pull request data as variables.
source_prq_vars() (
# $1 = path to pull request metadata
# $2 = whether to scope the variables as local
local f val
cd "$1" || return 1
[[ -f source_repo && -f created_at && -f title ]] || return 1
for f in *; do
[[ -f $f ]] || continue
read val < "$f"
local s=''
[[ $3 && $3 = "unset" ]] && s+="unset prq_$f; "
[[ $2 && $2 = "local" ]] && s+="local "
s+="prq_$f=\"$val\""
echo "$s"
done
)
# Check remote references at the personal remote to see if there
# are any merged pull requests. If there are, delete them.
scrub_merged_pull_requests() {
remote_available personal && remote_is_github personal || return 0
# $@ = branches to test for mergedness
local br ref pull_req remote
local -A to_remove pull_reqs heads
while read ref br; do
case $br in
refs/heads/*)
ref=${br#refs/heads/}
heads[${ref//\//-}]+="$br ";;
refs/remotes/personal/pull-req-*)
ref=${br#refs/remotes/personal/pull-req-}
ref=${ref#heads-}
ref=${ref%-*}
ref=${ref%-0}
pull_reqs["$br"]="$ref";;
esac
done < <(git show-ref)
[[ ${pull_reqs[*]} ]] || return 0
for pull_req in "${!pull_reqs[@]}"; do
ref="${pull_reqs[$pull_req]}"
[[ ${heads[$ref]} ]] || continue
for br in ${heads["$ref"]}; do
remote=$(remote_for_branch "$br")
branches_synced . "$remote/$br" "$pull_req" || \
[[ $1 = '--all' ]] || continue
to_remove["${pull_req#refs/remotes/personal/}"]="true"
continue 2
done
done
[[ ${!to_remove[*]} ]] || return
git_push --delete personal "${!to_remove[@]}"
git remote prune personal
}
# Helper function only for use by fetch_pull_request_metadata.
# If we have applicable metadata from the CI repo, we will
# pull it in as well.
save_pull_request_metadata() (
local fetchcmd
[[ $base_branch && \
$pull_req_branch && \
$pull_req_repo && \
$pull_req_target_repo && \
$pull_req_sha && \
$pull_req_state && \
$repo && \
$title && \
$github_id && \
$created_at && \
$github_target_user ]] || return
if [[ $pull_req_id ]]; then
prq="bundles/$pull_req_id/$repo"
if [[ -d $LOCAL_CI_TRACKING/${prq%/*} && \
! -d $OPEN_PULL_REQUESTS/${prq%/*} ]]; then
mkdir -p "$OPEN_PULL_REQUESTS/${prq%/*}"
cp -a "$LOCAL_CI_TRACKING/${prq%/*}/." \
"$OPEN_PULL_REQUESTS/${prq%/*}/."
fi
else
prq="singletons/$github_target_user/$repo/$github_id"
[[ $order ]] || order='1:1'
if [[ -d $LOCAL_CI_TRACKING/${prq} && \
! -d $OPEN_PULL_REQUESTS/${prq} ]]; then
mkdir -p "$OPEN_PULL_REQUESTS/${prq}"
cp -a "$LOCAL_CI_TRACKING/${prq}/." \
"$OPEN_PULL_REQUESTS/${prq}/."
fi
fi
local dest="$OPEN_PULL_REQUESTS/$prq"
mkdir -p "$dest"
case $repo in
crowbar) fetchcmd="in_repo git fetch";;
barclamp-*) fetchcmd="in_barclamp "${repo#barclamp-}" git fetch";;
*) die "Unknown repo $repo!";;
esac
local local_branch="pull-req/$github_user/$github_id"
$fetchcmd "$pull_req_repo" "+$pull_req_branch:$local_branch" &>/dev/null || return
cd "$dest"
echo "$order" > order
echo "$pull_req_repo" > source_repo
echo "$repo" > local_repo
echo "$pull_req_target_repo" > target_repo
echo "$pull_req_branch" > source_branch
echo "$pull_req_sha" > source_sha
echo "$pull_req_target_sha" > target_sha
echo "$base_branch" > target_branch
echo "$github_id" > number
echo "$created_at" > created_at
echo "$title" > title
echo "$pull_req_state" > state
echo "$github_url" > github_url
echo "$local_branch" > local_branch
echo "$github_user" > source_account
echo "$github_target_user" > target_account
[[ $updated_at ]] && echo "$updated_at" > updated_at
if [[ $rel ]]; then
echo "$rel" > release
else
echo "$(release_for_branch "$base_branch")" >release
fi
)
# Clear out local pull request metadata, including local pull request branches.
clear_pull_request_metadata() (
for bc in "$CROWBAR_DIR/barclamps/"* "$CROWBAR_DIR"; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
cd "$bc"
while read sha ref; do
[[ $ref = refs/heads/pull-req/* ]] || continue
git branch -D ${ref#refs/heads/} &>/dev/null
done < <(git show-ref --heads)
done
[[ -d $OPEN_PULL_REQUESTS ]] && rm -rf "$OPEN_PULL_REQUESTS"
)
# Fetch pull request metadata from Github.
# When this function is done, metadata for all pull requests generated by
# dev for this repo will have the right metadata fragments in the tracking directory
# for the repository in question.
fetch_pull_request_metadata() {
# $1 = remote name
# $2 = repository name.
# Not a github remote? Goodbye.
[[ ${DEV_REMOTE_URLBASE[$1]} =~ $github_re ]] || return 0
local acct="${BASH_REMATCH[2]}"
# Get our raw data. This is what we in the business call fugly.
local -A pulls
. <(parse_yml_or_json - pulls < <(
curl_and_res "https://api.github.com/repos/$acct/$2/pulls"))
local pull_req_id pull_req_sha order pull_req_repo pull_req_branch created_at
local base_branch key ord repo title rel github_id updated_at github_url github_user
local github_target_user pull_req_target_repo pull_req_target_sha pull_req_state
for key in $(printf '%s\n' "${!pulls[@]}" |sort); do
if [[ $ord && $ord != ${key%%.*} ]]; then
# The first part of the key has changed, so we should have a complete set of data.
save_pull_request_metadata
unset pull_req_id pull_req_sha order pull_req_repo pull_req_branch created_at
unset base_branch key ord repo title rel github_id updated_at github_url github_user
unset github_target_user pull_req_target_repo pull_req_target_sha pull_req_state
local pull_req_id pull_req_sha order pull_req_repo pull_req_branch created_at
local base_branch key ord repo title rel github_id updated_at github_url github_user
local github_target_user pull_req_target_repo pull_req_target_sha pull_req_state
fi
ord=${key%%.*}
# Strip off the inital array component of the hash key.
# We will not need it anyways.
case ${key#*.} in
body)
# If the body text does not have a pull request bundle ID, skip it.
[[ ${pulls[$key]} =~ $PULL_BUNDLE_RE ]] && pull_req_id="${BASH_REMATCH[1]}"
[[ ${pulls[$key]} =~ $PULL_RELEASE_RE ]] && rel="${BASH_REMATCH[1]}";;
# The repo and branch that contains changes to be tested.
head.ref) pull_req_branch="${pulls[$key]}";;
head.repo.owner.login) github_user="${pulls[$key]}";;
base.repo.owner.login) github_target_user="${pulls[$key]}";;
head.sha) pull_req_sha="${pulls[$key]}";;
head.repo.clone_url) pull_req_repo="${pulls[$key]}";;
base.repo.clone_url) pull_req_target_repo="${pulls[$key]}";;
base.sha) pull_req_target_sha="${pulls[$key]}";;
title)
if [[ ${pulls[$key]} =~ $PULL_TITLE_RE ]]; then
order="${BASH_REMATCH[2]}:${BASH_REMATCH[3]}"
title="${BASH_REMATCH[1]}"
else
title="${pulls[$key]}"
fi;;
base.ref) base_branch="${pulls[$key]}";;
base.repo.name) repo="${pulls[$key]}";;
number) github_id="${pulls[$key]}";;
state) pull_req_state="${pulls[$key]}";;
created_at) created_at="${pulls[$key]}";;
updated_at) updated_at="${pulls[$key]}";;
html_url) github_url="${pulls[$key]}";;
esac
done
save_pull_request_metadata
}
# Fetch pull request metadata for all repostories at a given remote.
fetch_pull_requests_for_remote() {
# $@ = remote to check for updates from
local -a remotes
local remote
[[ $1 ]] && remotes=("$@") || remotes=($(origin_remote))
fetch_ci_tracking
clear_pull_request_metadata
for remote in "${remotes[@]}"; do
if ! (crowbar_remote_exists "$remote" && remote_available "$remote" && \
[[ ${DEV_REMOTE_URLBASE[$remote]} =~ $github_re ]]); then
debug "Cannot fetch pull requests from $remote, skipping."
continue
fi
debug "Fetching pull requests from $remote:"
for bc in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
in_barclamp "${bc##*/}" git_remote_exists "$remote" || continue
debug "Fetching open pull requests for barclamp ${bc##*/}"
fetch_pull_request_metadata "$remote" "barclamp-${bc##*/}"
done
debug "Fetching open pull requests for Crowbar"
fetch_pull_request_metadata "$remote" "crowbar"
echo
done
sort_pull_requests
show_open_pull_requests
}
# Check to see of a pull request bundle is sane.
# Sanity consists of having all the pull requests we need and
# ensuring that the branch -> release mapping is consistent.
pull_request_bundle_sane() {
# $1 = bundle ID.
# Returns true if all components of the bundle are present, and if
# all the target branches in the barclamps match the release we are
# pulling against.
local -A orders releases branches
local -a order
local total_count=0 indicated_count=0 d f r repo o c release branch
local branch_for_release build bc source_account su tu
for d in "$OPEN_PULL_REQUESTS/bundles/$1/"*; do
[[ -d "$d" ]] || continue
repo="${d##*/}"
for f in order source_repo source_branch source_sha source_account \
target_branch number created_at title release; do
[[ -f $d/$f ]] && continue
debug "Missing metadata info $d/$f"
return 1
done
read o < "$d/order"
read r < "$d/release"
read su < "$d/source_account"
read branch < "$d/target_branch"
if ((indicated_count == 0)); then
indicated_count="${o##*:}"
elif ((indicated_count != "${o##*:}")); then
debug "Indicated count changed for pull request bundle $1"
return 1
fi
if [[ ! $release ]]; then
release=$r
if ! release_exists "$release"; then
debug "Release $release does not exist locally."
return 1
fi
elif [[ $r != $release ]]; then
debug "Release changed in pull request bundle $1"
return 1
fi
if [[ ! $source_account ]]; then
source_account=$su
elif
[[ $su != $source_account ]]; then
debug "Source Github account for pull request changed"
return 1
fi
if [[ $d = *barclamp-* ]]; then
bc="${d##barclamp-}"
for build in $(builds_for_barclamp_in_release "$bc" "$release"); do
[[ $branch = $(barclamp_branch_for_build "$build" "$bc") ]] && continue
debug "Branch $branch for barclamp $bc in bundle $1 does not match metadata."
return 1
done
fi
( cd "$d"
for f in title release source_account; do
[[ -f ../$f ]] || cp "$f" ..
done )
orders[$repo]=$o
releases[$repo]=$release
branches[$repo]=$branch
total_count=$(($total_count + 1))
done
if ((indicated_count != total_count)); then
debug "Not all barclamps in pull request bundle $1 at Github yet."
fi
}
# Show a count of all open pull request bundles and singleton pull requests.
show_open_pull_requests() {
local -a open_bundles open_singletons
local d r
for d in "$OPEN_PULL_REQUESTS/bundles/"*; do
[[ -d $d ]] || continue
pull_request_bundle_sane "${d##*/}" || continue
open_bundles+=("$d")
done
for d in "$OPEN_PULL_REQUESTS/singletons/"*/*/*; do
[[ -d $d ]] || continue
open_singletons+=("$d")
done
if (( ${#open_bundles[@]} == 0 && ${#open_singletons[@]} == 0)); then
r+="No open pull requests"
elif (( ${#open_bundles[@]} == 0 )); then
r+="${#open_singletons[@]} open singleton pull requests"
elif (( ${#open_singletons[@]} == 0)); then
r+="${#open_bundles[@]} open pull request bundles"
else
r+="${#open_bundles[@]} open pull request bundles and ${#open_singletons[@]} open singleton pull requests"
fi
debug "$r"
}
# Helper to determine when a pull request bundle was last touched.
pull_request_last_touched() (
# $1 = bundle
# Returns the most recent date that any of the bundle components were touched.
local mtime
if [[ $1 = bundles/* ]]; then
cd "$OPEN_PULL_REQUESTS/$1" || die "Bundle $1 does not exist!"
for d in *; do
[[ -d $d ]] || continue
[[ $ltime ]] || read ltime <"$d/created_at"
[[ -f updated_at ]] && read $d/ltime <"$d/updated_at"
[[ $ltime > $mtime ]] && mtime=$ltime
done
echo $mtime
elif [[ $1 = singletons/* ]]; then
if [[ -f $OPEN_PULL_REQUESTS/$1/updated_at ]]; then
cat "$OPEN_PULL_REQUESTS/$1/updated_at"
else
cat "$OPEN_PULL_REQUESTS/$1/created_at"
fi
else
die "No idea how to find last touched time for $1"
fi
)
# Helper for finding out all the pull request metadata we have.
pull_request_directories() (
[[ -d $OPEN_PULL_REQUESTS ]] || return 0
cd "$OPEN_PULL_REQUESTS"
for d in bundles/* singletons/*/*/*; do
if [[ $d = bundles/* ]]; then
pull_request_bundle_sane "${d##*/}" || continue
fi
[[ -f $d/title ]] || continue
echo "$(pull_request_last_touched "$d")|$d"
done |sort |cut -d \| -f 2
)
# Sort pull requests according to their last modification time.
sort_pull_requests() {
local count=1 p
rm "$OPEN_PULL_REQUESTS/pull_request_index" &>/dev/null
for p in $(pull_request_directories); do
echo "$count $p" >>"$OPEN_PULL_REQUESTS/pull_request_index"
count=$((count + 1))
done
}
# translate a pull request number (as displayed by dev pull-requests list)
# into an internal ID.
pull_request_number_to_id() {
local idx id
if [[ $1 && $1 =~ ^[0-9]+$ ]]; then
read idx id < <(grep "^$1 " "$OPEN_PULL_REQUESTS/pull_request_index")
[[ $1 = $idx ]] || die "$1 is not a valid pull request." \
"dev pull-requests list shows the open ones we know about."
elif grep -q " $1\$" "$OPEN_PULL_REQUESTS/pull_request_index"; then
id="$1"
elif [[ -d $LOCAL_PULL_TRACKING/$1/ci_state ]]; then
id="$1"
else
die "$1 is not a valid pull request ID"
fi
echo $id
}
# Figure our which builds need to be built to test a given pull request
builds_for_one_pull_request() {
# $1 =path to pull request
local -a builds
case $prq_local_repo in
barclamp-*)
builds=($(builds_for_barclamp_in_release "${prq_local_repo#barclamp-}" "$prq_release" ));;
crowbar)
builds=($(builds_in_release "$prq_release"));;
*) die "Unknown repo $prq_local_repo in builds_for_one_pull_request!";;
esac
echo "${builds[*]}"
}
# Return a list of builds that a pull request should trigger.
# This is based on barclamp membership.
# Returns a list of builds in the order in which they should be built.
builds_for_pull_request() {
local idx id b p release build os repo prq res=() all_oses=()
local -A oses build_tmp PULL_REQUEST_BARCLAMPS barclamp_oses build_oses _t _r
idx=$1
id=$(pull_request_number_to_id "$idx") || exit 1
prq="$OPEN_PULL_REQUESTS/$id"
read release <"$prq/release"
case $id in
singletons/*)
. <(source_prq_vars "$prq" "local")
if [[ $prq_local_repo = barclamp-* ]]; then
PULL_REQUEST_BARCLAMPS[${prq_local_repo#barclamp-}]="$prq_local_branch"
fi
for b in $(builds_for_one_pull_request "$prq"); do
build_tmp[$b]="build"
done;;
bundles/*)
for repo in "$prq"/*; do
[[ -d $repo ]] || continue
. <(source_prq_vars "$repo" "local")
[[ $prq_local_repo = barclamp-* ]] && \
PULL_REQUEST_BARCLAMPS[${prq_local_repo#barclamp-}]="$prq_local_branch"
for b in $(builds_for_one_pull_request "$repo"); do
build_tmp[$b]="build"
done
done;;
esac
# Now, we have all the possibly-applicable builds.
# Get all the possible OSes for all possible barclamps in those builds, and use
# it to derive all the OSes applicable to a build.
all_oses=($(all_supported_oses))
for build in "${!build_tmp[@]}"; do
# Start off assuming that all builds can use all OSes
unset _t
local -A _t
for os in "${all_oses[@]}"; do _t[$os]=build; done
for b in $(barclamps_in_build "$release/$build"); do
if ! [[ ${barclamp_oses[$b]} ]]; then
local br
if [[ ${PULL_REQUEST_BARCLAMPS[$b]} ]]; then
br="${PULL_REQUEST_BARCLAMPS[$b]}"
else
br="$(barclamp_branch_for_build "$release/$build" "$b")"
fi
p=$(extract_barclamp_metadata "$b" "$br")
barclamp_oses[$b]=$(read_barclamp_metadata "$p" "barclamp" "os_support")
[[ ${barclamp_oses[$b]} ]] || barclamp_oses[$b]=none
fi
if [[ ${barclamp_oses[$b]} && ${barclamp_oses[$b]} != none ]]; then
unset _r
local -A _r
for os in ${barclamp_oses[$b]}; do _r[$os]=$b; done
for os in "${!_t[@]}"; do
[[ ${_r[$os]} ]] && continue
unset _t[$os]
done
fi
done
build_oses[$build]="${!_t[*]}"
done
# Now, filter the per-OS lists down to leaf nodes.
res=($(builds_in_release "$release"))
# Filter out the builds that will be masked.
for id in "${res[@]}"; do
[[ ${build_tmp[$id]} ]] || continue
# Find the parent for this build.
p="${DEV_BRANCHES[$id]}"
# Special case for the master build.
[[ $p = $id ]] && continue
# If we wanted to build the parent for the current build, don't.
# It will be implicitly tested when we build this build.
[[ ${build_tmp[$p]} ]] && build_tmp[$p]=skip
done
# Display the ones that did not get masked
for id in "${res[@]}"; do
[[ ${build_tmp[$id]} && ${build_tmp[$id]} = build ]] || continue
echo "$release/$id: ${build_oses[$id]}"
done
}
# Show information for a pull request in human-readable format.
show_pull_request() {
local idx id prq repo release target_account
local -a builds
idx=$1
id=$(pull_request_number_to_id $idx) || exit 1
prq="$OPEN_PULL_REQUESTS/$id"
[[ -f $prq/title ]] || die "Something Wicked happened trying to show $idx." \
"Please re-run dev pull-request fetch"
if [[ $id = singletons/* ]]; then
. <(source_prq_vars "$prq" "local")
if [[ $prq_local_repo = barclamp-* ]]; then
builds=($(builds_for_barclamp_in_release "${prq_local_repo#barclamp-}" "$prq_release"))
else
builds=($(builds_in_release "$prq_release"))
fi
echo "Title: $prq_title"
echo "Target Account: $prq_target_account"
echo "Source Account: $prq_source_account"
echo "Release: $prq_release"
echo "Last Updated: $(pull_request_last_touched "$id")"
echo "Type: singleton"
echo "Repo: $prq_local_repo"
echo "Repo with Changes: $prq_source_repo"
echo "Branch with Changes: $prq_source_branch"
echo "Local Branch: $prq_local_branch"
echo "Target Branch: $prq_target_branch"
echo "Pull Request URL: $prq_github_url"
echo "Builds: ${builds[*]}"
elif [[ $id = bundles/* ]]; then
echo "Title: $(cat "$prq/title")"
echo "Release: $(cat "$prq/release")"
echo "Last Updated: $(pull_request_last_touched "$id")"
echo "Type: bundle"
echo "Bundle ID: ${id##*/}"
local repo
for repo in "$prq/"*; do
[[ -d $repo ]] || continue
. <(source_prq_vars "$repo" "local")
if [[ $repo = */barclamp-* ]]; then
builds=($(builds_for_barclamp_in_release "${repo#*/barclamp-}" "$prq_release"))
else
builds=($(builds_in_release "$prq_release"))
fi
echo "Repo: ${repo##*/}"
echo " Repo with Changes: $prq_source_repo"
echo " Target Account: $prq_target_account"
echo " Source Account: $prq_source_account"
echo " Branch with Changes: $prq_source_branch"
echo " Local Branch: $prq_local_branch"
echo " Target Branch: $prq_target_branch"
echo " Pull Request URL: $prq_github_url"
echo " Builds: ${builds[*]}"
done
else
die "Unknown pull request type $id!"
fi
}
# List pull requests. Uses the quasi-stable numbering provided by sorting.
list_pull_requests() {
local idx id
while read idx id; do
[[ -d $OPEN_PULL_REQUESTS/$id ]] || continue
cf="$OPEN_PULL_REQUESTS/$id/title"
cr="$OPEN_PULL_REQUESTS/$id/release"
ct="${id%%/*}"
echo "$idx: ($(cat "$cr")) $(cat "$cf")"
done <"$OPEN_PULL_REQUESTS/pull_request_index"
}
pull_request_checkout_one() (
. <(source_prq_vars "$1" "local")
case $prq_local_repo in
barclamp-*) cd "$CROWBAR_DIR/barclamps/${prq_local_repo#barclamp-}";;
crowbar) cd "$CROWBAR_DIR";;
esac
debug "$prq_local_repo: Checking out $prq_local_branch"
quiet_checkout "$prq_local_branch" || \
die "Could not checkout $prq_local_branch!"
)
pull_request_switch_one() (
. <(source_prq_vars "$1" "local")
case $prq_local_repo in
barclamp-*) cd "$CROWBAR_DIR/barclamps/${prq_local_repo#barclamp-}";;
crowbar) cd "$CROWBAR_DIR";;
esac
pull_request_checkout_one "$1" || exit 1
git checkout "$(git rev-parse HEAD)"
git merge "$(git rev-parse "$prq_target_branch")" || \
die "Could not merge $prq_target_branch into $prq_local_branch!"
)
# Switch to a specific build in a pull request.
# The release is inferred by the preferred release of the build request.
pull_request_switch() {
# $1 = the local number of the pull request.
# $2 = the build within the pull request. If only one
# build is applicable for a pull request, it defaults to that one.
# If more than one is applicable, you must set this arg.
crowbar_is_clean || die "Trees must be clean before switching to a pull request"
local id idx prq release bc barclamps build builds repo_dirs repo_dir repo res
idx=$1
id=$(pull_request_number_to_id "$idx") || \
die "$id is not a valid pull request. dev pull-requests list will help."
while read build; do
builds+=("${build%%:*}")
done < <(builds_for_pull_request "$idx")
[[ $builds ]] || \
die "Pull request $idx has no associated builds!"
if (( ${#builds[@]} == 1 )) && [[ ! $2 || $2 = $builds ]]; then
build=${builds}
elif [[ $2 ]] && ! is_in "$2" "${builds[*]}"; then
die "Build $2 is not applicable to pull request $idx." \
"Run ./dev pull-requests builds $idx to see what is."
elif [[ ! $2 ]]; then
die "You must pass a build to pull-requests switch, because there is no obvious default." \
"Run ./dev pull-requests builds $idx to see what is available."
else
build=$2
fi
prq="$OPEN_PULL_REQUESTS/$id"
switch_release "$build" || \
die "Could not switch to release $build"
res=0
case $id in
singletons/*)
pull_request_switch_one "$prq"
res=$?;;
bundles/*)
for repo in "$prq"/*; do
[[ -d $repo ]] || continue
if ((res == 0)); then
pull_request_switch_one "$repo" && continue
debug "Switch failed, skipping the rest."
res=1
else
debug "Skipping $(cat "$repo/github_url")"
fi
done;;
esac
if ((res == 0)); then
debug "You are now on build $build in pull request $idx."
else
switch_release
die "Pull request $idx does not merge cleanly. It should be updated or closed."
fi
}
pull_request_merge_one () {
# $1 = path to saved metadata
. <(source_prq_vars "$1" local)
# Skip if we have already been merged.
if curl_and_res -f -X GET \
"https://api.github.com/repos/$prq_target_account/$prq_local_repo/pulls/$prq_number/merge" &>/dev/null; then
return 0
fi
if curl_and_res -f -X PUT \
-d "{\"commit_message\": \"Merged by devtool for $DEV_GITHUB_ID\"}" \
"https://api.github.com/repos/$prq_target_account/$prq_local_repo/pulls/$prq_number/merge" &>/dev/null; then
debug "$(cat "$1/github_url") merged."
return 0
else
debug "Merge of $(cat "$1/github_url") failed."
return 1
fi
}
# Check out the branches specific to a pull request.
# Unlike pull-requests merge, this leaves you on actual branches.
pull_request_checkout() {
# $1 = index
local id res=0 repo
crowbar_is_clean || die "Trees must be clean before switching to a pull request!"
id=$(pull_request_number_to_id "$1") || \
die "$1 is not a valid pull request!"
case $id in
singletons/*)
pull_request_checkout_one "$OPEN_PULL_REQUESTS/$id"
res=$?;;
bundles/*)
for repo in "$OPEN_PULL_REQUESTS/$id/"*; do
[[ -d $repo ]] || continue
if ((res == 0)); then
pull_request_checkout_one "$repo" && continue
debug "Skipping remaining pull requests"
res=1
else
debug "Skipping $(cat $repo/github_url)"
fi
done;;
esac
if ((res == 0)); then
debug "Pull request checked out." \
"You can get back to your regular tree with ./dev switch"
return 0
else
switch_release
debug "Could not check out pull request."
return 1
fi
}
# Have Github merge all the changes in a pull request bundle.
pull_request_merge() {
# $1 = index
local id res=0 repo
id=$(pull_request_number_to_id "$1") || \
die "$1 is not a pull request number!"
case $id in
singletons/*)
pull_request_merge_one "$OPEN_PULL_REQUESTS/$id"
res=$?;;
bundles/*)
for repo in "$OPEN_PULL_REQUESTS/$id/"*; do
[[ -d $repo ]] || continue
if ((res == 0)); then
pull_request_merge_one "$repo" && continue
debug "Aborting remaining pull requests."
res=1
else
debug "Skipping $(cat "$repo/github_url")"
fi
done;;
esac
if ((res == 0)); then
debug "Pull requests merged." \
"You can use $0 fetch to pull in updated pull request information."
return 0
else
debug "Please fix up the broken pull requests and retry the merge."
return 1
fi
}
pull_request_comment_one() {
# $1 = path to pull request
# $2 = body of comment
. <(source_prq_vars "$1" local)
curl_and_res -f -X POST \
-d "{ \"body\": \"$2\"}" \
"https://api.github.com/repos/$prq_target_account/$prq_local_repo/issues/$prq_number/comments"
}
# Add a comment to a pull request.
pull_request_comment() {
# $1 = pull request number
# $2 = body of comment
local id repo
id=$(pull_request_number_to_id "$1") || exit 1
case $id in
singletons/*)
pull_request_comment_one "$OPEN_PULL_REQUESTS/$id" "$2";;
bundles/*)
for repo in "$OPEN_PULL_REQUESTS/$id/"*; do
[[ -d $repo ]] || continue
pull_request_comment_one "$repo" "$2"
done;;
esac
}
pull_request_close_one() {
# $1 = path to local pull request info
# $2 = reason for closing the pull request
pull_request_comment_one "$1" "$2" || return 1
. <(source_prq_vars "$1" local)
curl_and_res -f -X PATCH \
-d "{ \"state\": \"closed\"}" \
"https://api.github.com/repos/$prq_target_account/$prq_local_repo/pulls/$prq_number"
}
# Close a pull request without merging it.
pull_request_close() {
# $1 = pull request number
# $2 = closing comment
local id repo
id=$(pull_request_number_to_id "$1") || exit 1
case $id in
singletons/*) pull_request_close_one "$OPEN_PULL_REQUESTS/$id" "$2";;
bundles/*)
for repo in "$OPEN_PULL_REQUESTS/$id/"*; do
[[ -d $repo ]] || continue
pull_request_close_one "$repo" "$2"
done;;
esac
}
# Helper function for trying to do something with the CI system and push out the results.
# The thing to do should be idempotent.
ci_do_and_push() {
while true; do
$@ || return $?
push_ci_tracking
local __res=$?
case $__res in
0) return 0;;
1) continue;;
2) die "No local pull tracking repository!";;
3) die "Could not find remote to push CI tracking to!";;
128) die "Could not talk to upstream CI tracking remote!";;
*) return $__res;;
esac
done
}
# Fetch CI tracking information and try to merge it with any local information
# that has not been pushed upstream. If we cannot merge, we throw away the
# local changes and let the caller know that we failed to merge local changes.
fetch_ci_tracking() (
[[ ! -d $LOCAL_PULL_TRACKING ]] && mkdir -p "$LOCAL_PULL_TRACKING"
if [[ ! -d $LOCAL_PULL_TRACKING/.git ]]; then
for remote in "${DEV_SORTED_REMOTES[@]}"; do
remote_available "$remote" || continue
test_remote "${DEV_REMOTE_URLBASE[$remote]}/$CI_TRACKING_REPO" || continue
git clone -o "$remote" \
"${DEV_REMOTE_URLBASE[$remote]}/$CI_TRACKING_REPO" \
"$LOCAL_PULL_TRACKING" || return 2
return 0
done
return 2
fi
cd "$LOCAL_PULL_TRACKING"
git fetch -q --all || return 128
quiet_checkout -f master
remote=$(git config --get branch.master.remote) || return 3
branches_synced "." "$remote/master" "master" && return 0
exec &>/dev/null
git rebase "$remote/master" master && return 0
git rebase --abort
git reset --hard "$remote/master"
return 1
)
# Push CI tracking to the upstream repository.
push_ci_tracking() (
local remote
fetch_ci_tracking || return $?
cd "$LOCAL_PULL_TRACKING"
remote=$(git config --get branch.master.remote) || return 3
git push -q "$remote" "master:master"
local __res=$?
(( $__res > 127 || $__res == 0 )) && return $res
)
# Get all the IDs that the pull request system knows about.
ci_ids() {
local id ids=()
ids=("$LOCAL_PULL_TRACKING/bundles/"* "$LOCAL_PULL_TRACKING/singletons/"*/*/*)
for id in "${ids[@]}"; do
echo "${id#$LOCAL_PULL_TRACKING/}"
done
}
# Get all the pull request IDs that are not closed or merged.
ci_open_ids() {
local id
local -a ids
while read id; do
local state=$(ci_get_current_states "$id")
[[ $state = closed || $state = merged || \
$state = failed || $state = needs-work ]] && continue
ids+=("$id")
done < <(ci_ids)
__random_order "${ids[@]}"
}
__ci_set_state() (
# $1 = id
# $2 = state
# $3 = comment
cd "$LOCAL_PULL_TRACKING/$1" || die "$1 is not being tracked by CI"
state="ci_state/$2"
mkdir -p "$state"
echo "$3" >>"$state/comment"
)
# Find the states that the current state transitioned from.
ci_last_states() {
# $1 = pull request ID
# $2 = state
[[ -d $LOCAL_PULL_TRACKING/$1 ]] || \
die "$1 is not a CI tracked pull request!"
local state_dir="$LOCAL_PULL_TRACKING/$1/ci_state/$2"
[[ -d $state_dir ]] || \
die "$2 is not a state in $1"
[[ -d $state_dir/last_states ]] || return 0
local sfile state
for sfile in "$state_dir/last_states/"*; do
echo "${sfile##*/}"
done
}
__ci_state_walker() {
local state this_states=()
while read -r state; do
[[ ${states[$state]} ]] && continue
states["$state"]="$3"
this_states+=("$state")
done < <(ci_last_states "$1" "$2")
[[ $this_states ]] || return 0
for state in "${this_states[@]}"; do
__ci_state_walker "$1" "$state" "$(( ${3} + 1))"
done
}
ci_all_last_states() {
# $1 = pull request ID
# $2 = starting state
local -A states
__ci_state_walker "$1" "$2" 0
(( ${#states[@]} == 0 )) && return 0
printf "%s\n" "${!states[@]}"
}
# Link an old state to a new one. This function is responsible for
# making sure that the overall state transition graph remains sane.
ci_link_state() {
# $1 = pull request ID
# $2 = old state
# $3 = new state
[[ -d $LOCAL_PULL_TRACKING/$1/ci_state ]] || \
die "$1 is not being tracked by the CI system!"
local state_dir="$LOCAL_PULL_TRACKING/$1/ci_state"
[[ -d $state_dir/$2 ]] || \
die "$1 does not have old state $2"
[[ -d $state_dir/$3 ]] || \
die "$1 does not have new state $3"
[[ $2 != $3 ]] || \
die "$1: Cannot link $2 to itself!"
local state
# Check to see if this link already exists.
# If it does, we just succeed without doing anything.
[[ -f $state_dir/$3/last_states/$2 ]] && return 0
# Check to see if adding this link would make the graph cyclical.
# If it will, just die.
while read -r state; do
[[ $state = $3 ]] && \
die "$1: Linking $2 to $3 would create a cyclical state graph!"
done < <(ci_all_last_states "$1" "$2")
mkdir -p "$LOCAL_PULL_TRACKING/$1/ci_state/$3/last_states"
touch "$LOCAL_PULL_TRACKING/$1/ci_state/$3/last_states/$2"
( cd "$LOCAL_PULL_TRACKING/$1/ci_state"
git add "$3/last_states/$2"
git commit -m "ci_link_state $*")
}
# Figure out the set of possible next states from the current state.
# This may not be the same as the next states we will wind up transitioning to.
# It is also the reference for what states the CI state machine can be in.
ci_all_next_states() {
# $1 = index or local ID.
# $2 = current state
local -A unit_test_whitelist
local -a next
local id release build os oses current_state
# As we get more releases that are unit test enabled, add them here,
# replace this with blacklist logic or (ideally) remove it altogether.
unit_test_whitelist["development"]=true
id=$(pull_request_number_to_id "$1") || \
die "$1 is not a valid pull request!"
[[ -d $LOCAL_PULL_TRACKING/$id ]] || \
die "Pull request $1 has not been imported into CI." \
"Please run $0 ci import."
if [[ $2 ]]; then
current_state="${2}"
else
local states=()
while read current_state; do
states+=("$current_state")
done < <(ci_get_current_states "$id")
if (( ${#states[@]} > 1)); then
die "Pull request $1 is in more than one state:" \
"${states[@]}" \
"Please pass one of these to next-states"
fi
current_state="${states[0]}"
fi
read -r release < "$LOCAL_PULL_TRACKING/$id/release"
case $current_state in
# The first two states are fairly simple in their allowed transitions.
new) next=(merge-testing);;
merge-testing) next=(merge-tested);;
# Unit testing happens on a per-release/build basis,
# and is not applicable to all releases. Our next state
# depends on the whitelist.
merge-tested)
while read build oses; do
build=${build%:}
if [[ ${unit_test_whitelist["$release"]} ]]; then
next+=("unit-testing $build")
else
for os in $oses; do
next+=("build-testing $build $os")
done
fi
done < <(builds_for_pull_request "$id");;
# unit testing happens on a per-release and per-build basis.
unit-testing*)
build=${2#unit-testing }
next=("unit-tested $build");;
unit-tested*)
local b=${2#unit-tested }
while read build oses; do
build=${build%:}
if [[ $build != $b ]]; then continue; fi
next+=("build-testing $build $os")
done < <(builds_for_pull_request "$id");;
# Build and smoke tests happen on a release/build/os basis.
build-testing*)
build=${2#build-testing }
next=("build-tested $build");;
build-tested*)
build=${2#build-tested }
next=("smoke-testing $build");;
smoke-testing*)
build=${2#smoke-testing }
next=("smoke-tested $build");;
smoke-tested*) next=(code-reviewing);;
# code-reviewing can actaully happen in parallel with the above
# steps.
code-reviewing) next=(code-reviewed);;
code-reviewed) next=(mergeable);;
# Mergeable cannot happen until we have a sufficient number of code-reviewed states
# and we have smoke-tested states for all the release/build/OS combos applicable.
mergeable) next=(merged);;
failed|needs-work) next=(new);;
# Merged and closed are terminal states.
# Nothing else can happen to them.
merged|closed) next=();;
*) die "Unknown state $current_state!";;
esac
printf "%s\n" "${next[@]}"
}
# Print any passed args in pseudo-random order
__random_order() {
local -a res
local k v
for v in "$@"; do
k="$RANDOM"
while [[ ${res[$k]} ]]; do k="$RANDOM"; done
res[$k]="$v"
done
printf "%s\n" "${res[@]}"
}
# Wrapper around ci_all_next_states that filters out next states that have
# an intent registered for them, and prints them in random order.
ci_next_states() {
local state id state_dir
local -a states
id=$(pull_request_number_to_id "$1") || \
die "$1 is not a valid pull request!"
[[ -d $LOCAL_PULL_TRACKING/$id ]] || \
die "Pull request $1 has not been imported into CI." \
"Please run $0 ci import."
state_dir="$LOCAL_PULL_TRACKING/$id/ci_state/$2"
while read -r state; do
[[ -d $state_dir/intents/$state ]] && continue
states+=("$state")
done < <(ci_all_next_states "$id" "$2")
__random_order "${states[@]}"
}
# Get the current states that a given pull request in the CI is currently in.
# In the interest of parallelization, a pull request can be in multiple CI
# states at any given time.
ci_get_current_states() {
# $1 = pull request ID.
local state_dir state last_state last_state_file id
local -A states
id=$(pull_request_number_to_id "$1") || \
die "$1 is not a valid pull request!"
[[ -d $LOCAL_PULL_TRACKING/$id/ci_state ]] || die "$id is not being tracked by CI"
# First, read all of the state paths in.
for state_dir in "$LOCAL_PULL_TRACKING/$id/ci_state/"*; do
[[ -d $state_dir ]] || continue
states[${state_dir##*/}]=current
done
# Second, mask out all of the states that have a last file pointing at them.
# This needs to be ornate-ified to handle per-state special conditions.
for state in "${!states[@]}"; do
[[ -d $LOCAL_PULL_TRACKING/$id/ci_state/$state/last_states ]] || continue
while read -r last_state; do
[[ ${states[$last_state]} ]] || \
die "State graph for CI item $id is invalid." \
"State $state says that it transitioned from $last_state," \
"but $last_state is not in the state graph!"
states[$last_state]=stale
done < <(ci_last_states "$id" "$state")
done
# Anything not marked as stale is a state that we are in.
for state in "${!states[@]}"; do
[[ ${states[$state]} = stale ]] && unset states[$state]
done
# See if we are in one of the terminal states
for last_state in closed failed needs-work merged new; do
for state in "${!states[@]}"; do
[[ $state = $last_state ]] || continue
echo "$state"
return
done
done
__random_order "${!states[@]}"
}
ci_kill_stale_intents() (
# $1 = pull request ID
[[ -d $LOCAL_PULL_TRACKING/$1/ci_state ]] || return 1
cd "$LOCAL_PULL_TRACKING/$1/ci_state"
local intent need_commit=false
for intent in */intents/*; do
[[ -d $intent ]] || continue
local from=${intent%/intents/*}
local to=${intent#*/intents/}
[[ -d $to ]] && continue
(( $(cat "$intent/timeout") >= $(date -u '+%s'))) && continue
git rm -rf "$intent"
need_commit=true
done
[[ $need_commit = true ]] || return 1
git commit -mq "Killed stale intents for $1"
)
ci_register_intent() {
# $1 = pull request ID
# $2 = current state
# $3 = intended next state
# $4 = timeout for intent. Defaults to 5 hours.
local timeout="${4:-18000}" id prq state state_ok=false
id=$(pull_request_number_to_id "$1") || \
die "$1 is not a valid pull request!"
[[ -d $LOCAL_PULL_TRACKING/$id ]] || \
die "Pull request $1 has not been imported into CI." \
"Please run $0 ci import."
prq="$LOCAL_PULL_TRACKING/$id"
[[ -d $prq/ci_state/$2 ]] || {
debug "$id does not have state $2, cannot register an intent to move to $3"
return 1
}
[[ -d $prq/ci_state/$3 ]] && {
debug "$id already has state $3"
return 1
}
while read state; do
[[ $state = $2 ]] || continue
state_ok=true
break
done < <(ci_get_current_states "$1")
[[ $state_ok = true ]] || \
die "$id is not in state $2, cannot register an intent for it."
state_ok=false
while read state; do
[[ $state = $3 ]] || continue
state_ok=true
break
done < <(ci_all_next_states "$id" "$2")
[[ $state_ok = true ]] || \
die "$id: State $2 does not have a next state of $3, cannot register an intent."
# See if we need to expire intents on the current state.
if [[ -d $prq/ci_state/$2/intents/$3 ]]; then
ci_kill_stale_intents "$id" && push_ci_tracking && \
[[ ! -d $prq/ci_state/$2/intents/$3 ]] || return 1
fi
mkdir -p "$prq/ci_state/$2/intents/$3/"
local intent_id=$((ip addr show; date -u "+%s%N"; printf "$RANDOM") |sha1sum -)
intent_id="${intent_id%% *}"
echo "$(( $(date -u '+%s') + ${timeout} ))" > "$prq/ci_state/$2/intents/$3/timeout"
touch "$prq/ci_state/$2/intents/$3/$intent_id"
( cd "$prq/ci_state/$2/intents/$3/"
git add .
git commit -m "ci_register_intent $*") &>/dev/null
echo "$intent_id"
}
ci_handle_intent() {
# $1 = intent ID
# $2 = passed or failed
local -a intents states
local intent state
while read -r intent; do
intents+=("$intent")
done < <(find "$LOCAL_PULL_TRACKING/singletons" "$LOCAL_PULL_TRACKING/bundles" \
-name "$1" -type f) 2>/dev/null
[[ ${intents[1]} ]] && \
die "More than one intent with id $1 found. This should never happen." \
"${intents[@]}"
[[ $2 = passed || $2 = failed ]] || \
die "Second arg to ci_handle_intent should either be passed or failed."
intent="${intents[0]#$LOCAL_PULL_TRACKING/}"
local prq="${intent%%/ci_state/*}"
intent="${intent#${prq}/ci_state/}"
local current_state=${intent%/intents/*}
local next_state=${intent#*/intents/}
next_state=${next_state%/*}
if [[ $2 = failed ]]; then
if [[ $(ci_get_current_states "$prq") = failed ]]; then
return 0
else
while read state; do
states+=("$state")
done < <(ci_get_current_states "$prq")
__ci_set_state "$prq" failed \
"Intent to transition from $current_state to $next_state failed."
for state in "${states[@]}"; do
ci_link_state "$prq" "$state" failed
done
fi
else
__ci_set_state "$prq" "$next_state" \
"Intent to transition from $current_state to $next_state passed."
ci_link_state "$prq" "$current_state" "$next_state"
fi
}
ci_commit_intent() { ci_handle_intent "$1" "passed"; }
ci_fail_intent() { ci_handle_intent "$1" "failed"; }
# Handle transitioning a pull request in the CI state machine to closed
# or merged once it has been closed on Github.
ci_close_stale_pull_request() (
local prq prqs=() p_closed p_merged cout p_state
local -A pull
# Don't close it until we positively confirm that the pull request is closed.
cd "$LOCAL_PULL_TRACKING"
case $1 in
singletons/*) prqs+=("$LOCAL_PULL_TRACKING/$1");;
bundles/*)
for prq in "$LOCAL_PULL_TRACKING/$1/"*; do
[[ -d $prq && -f $prq/state ]] || continue
prqs+=("$prq")
done;;
*) die "Cannot happen!"
esac
local p_closed=true
local p_merged="merged"
for prq in "${prqs[@]}"; do
local cout=""
. <(source_prq_vars "$prq" local)
cout=$(curl_and_res -X GET \
"https://api.github.com/repos/$prq_target_account/$prq_local_repo/pulls/$prq_number")
(( $? == 0 )) || return
local -A pull
. <(parse_yml_or_json - pull <<< "$cout")
[[ ${pull["state"]} = closed || ${pull["message"]} = 'Not Found' ]] || return
[[ ${pull["merged"]} && ${pull["merged"]} = true ]] || p_merged="closed"
done
debug "Setting state of pull request $1 to $p_merged"
__ci_set_state "$1" "$p_merged" "Upstream pull request $p_merged."
for p_state in "${states[@]}"; do
ci_link_state "$1" "$p_state" "$p_merged"
done
git add "$1"
git commit --allow-empty -qm "Pull request id $1 $p_merged upstream."
)
ci_close_stale_pull_requests() {
local id commit prqs=() prq
local -A states
for prq in "$LOCAL_PULL_TRACKING/bundles/"* "$LOCAL_PULL_TRACKING/singletons/"*/*/*; do
id="${prq#$LOCAL_PULL_TRACKING/}"
[[ -d $prq/ci_state ]] || continue
[[ -d $prq/ci_state/closed || -d $prq/ci_state/merged ]] && continue
[[ -d $OPEN_PULL_REQUESTS/$id ]] && continue
ci_close_stale_pull_request "$id"
done
}
# Test to see if a pull request needs to be reset back to the new
# state due to the underlying branches changing.
ci_maybe_reset_pull_request() {
# $1 = pull request ID
local prqs=() prq tag needs_reset=false
case $1 in
singletons/*) prqs=("$1");;
bundles/*)
for prq in "$LOCAL_PULL_TRACKING/$1/"*; do
[[ -d $prq && -f $prq/state ]] || continue
prqs+=("${prq#$LOCAL_PULL_TRACKING/}")
done;;
esac
for prq in "${prqs[@]}"; do
for tag in source_sha target_sha; do
# If only some of them are here, this pull request is probably
# in a partially-closed state. Ignore it until we think of something
# better to do.
[[ -f $OPEN_PULL_REQUESTS/$prq/$tag && \
-f $LOCAL_PULL_TRACKING/$prq/$tag ]] || continue
[[ $(cat "$OPEN_PULL_REQUESTS/$prq/$tag") = \
$(cat "$LOCAL_PULL_TRACKING/$prq/$tag") ]] || \
needs_reset=true
done
done
[[ $needs_reset = false ]] && return 0
case $(ci_get_current_states "$1") in
new) return 0;;
closed|merged) die "$1 is already closed or merged, cannot be reset!";;
esac
( cd "$LOCAL_PULL_TRACKING/$1"
git rm -rf ci_state
__ci_set_state "$1" "new" "State machine reset due to pull request update."
git add ci_state
git commit -q -m "State machine for $1 reset by $DEV_GITHUB_ID"
)
}
# Handle cleaning up old state transitions, resetting any pulls that need
# resetting and importing any pull requests we are not tracking.
ci_import_new_pull_requests() {
local nr id
[[ -f $OPEN_PULL_REQUESTS/pull_request_index ]] || return 0
ci_close_stale_pull_requests
while read nr id; do
if [[ -d $LOCAL_PULL_TRACKING/$id ]]; then
ci_maybe_reset_pull_request "$id"
continue
fi
mkdir -p "$LOCAL_PULL_TRACKING/$id"
cp -a "$OPEN_PULL_REQUESTS/$id/." "$LOCAL_PULL_TRACKING/$id/."
__ci_set_state "$id" "new"
debug "Adding new pull request $id"
( cd "$LOCAL_PULL_TRACKING/$id"
git add .
git commit -qm "Added new pull request $id")
done < "$OPEN_PULL_REQUESTS/pull_request_index"
}
__fetch_all() {
local remote remotes=() res=true
if [[ $@ || $DEV_FROM_REMOTES ]]; then
[[ $1 ]] && remotes+=("$@")
[[ $DEV_FROM_REMOTES ]] && remotes+=("${DEV_FROM_REMOTES[@]}")
else
remotes=("${DEV_SORTED_REMOTES[@]}")
fi
local logfile="${PWD}"
logfile=${logfile##$(readlink -f "$CROWBAR_DIR")}
logfile=${logfile//\//-}
logfile=${logfile#-}
logfile=${logfile//barclamps/barclamp}
[[ $logfile ]] || logfile="Crowbar"
for remote in "${remotes[@]}"; do
crowbar_remote_exists "$remote" && \
remote_available "$remote" && \
git_remote_exists "$remote" || \
continue
if git fetch -q "$remote" &>/dev/null; then
debug " Fetched from $remote." 2>>"$results_dir/$logfile"
printf '.'
if [[ $PWD = */barclamps/* ]]; then
fetch_pull_request_metadata "$remote" "barclamp-${PWD##*/}"
else
fetch_pull_request_metadata "$remote" "crowbar"
fi
else
debug " Failed to fetch from $remote" 2>>"$results_dir/$logfile"
printf '!'
res=false
fi
done
scrub_merged_pull_requests
[[ $res = true ]] && debug " All fetches passed." 2>> "$results_dir/$logfile"
}
# Fetch (but do not merge) updates from all our remotes, in both the
# main Crowbar repository and the barclamps.
fetch_all() {
local results_dir d
clear_pull_request_metadata
dt=$(date +%s)
results_dir="$(mktemp -d /tmp/crowbar_fetch_${USER}_${dt}_XXXXXX)" || \
die "Cannot create temporary storage for fetch results!"
fetch_ci_tracking
debug "Fetching updates:"
for d in "$CROWBAR_DIR/barclamps/"* "$CROWBAR_DIR"; do
[[ -d $d/.git ]] || continue
if [[ $BADLINK ]]; then
cd "$d" && __fetch_all "$@"
else
( cd "$d" && __fetch_all "$@") &
fi
done
wait
echo
for d in "$results_dir"/*; do
local n="${d##*/}"
n="${n//-/ }"
if grep -q 'All fetches passed\.$' "$d"; then
debug "${n}: All updates fetched"
else
debug "${n}:"
cat "$d"
fi
done
if [[ $BADLINK ]]; then
debug "not removing $results_dir"
else
rm -rf "$results_dir"
fi
update_all_tracking_branches
show_open_pull_requests
sort_pull_requests
if [[ -d $LOCAL_PULL_TRACKING/.git ]]; then
ci_import_new_pull_requests
fi
}
# Attempt to scrub merged pull requests across all of the barclamps.
scrub_merged_pulls() {
in_repo scrub_merged_pull_requests "$@"
for barclamp in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $barclamp ]] || continue
in_barclamp "${barclamp##*/}" scrub_merged_pull_requests "$@"
done
}
# Helper function for calling curl to talk to github.
curl_and_res() {
local __r
__r="$(curl -n "$@" 2>/dev/null)"
case $? in
0) printf '%s' "$__r";;
7) echo "Unable to contact Github, please try again later." >&2
return 1;;
22) return 2;;
*) echo "Curl reported error ${?}!." >&2
return 3;;
esac
}
# Check to see if a repository exists on Github using the Github API.
github_repo_exists() {
# $1 = repo to check for
# $2 = user to check for it in. Defaults to $DEV_GITHUB_ID
local repo=$1 user=${2-$DEV_GITHUB_ID}
curl_and_res -f -X GET "https://api.github.com/repos/$user/$repo" &>/dev/null
}
# Fork a repository on Github.
github_fork() {
# $1 = user to fork from
# $2 = repo to fork
curl_and_res -f -X POST \
"https://api.github.com/repos/$1/$2/forks" >&/dev/null || \
die "Could not fork $1/$2! (your .netrc file may be missing)"
}
git_url_for_remote() { git_remote_exists "$1" && git config --get "remote.$1.url"; }
# Get the name of this repo based on the URL of the origin remote.
get_repo_name() {
local repo
repo="$(git_url_for_remote $(origin_remote))" || \
repo="$(git_url_for_remote origin)" || \
die "$PWD: Could not get the URL for the origin remote for this repo."
repo="${repo##*:}"
echo "${repo##*/}"
}
git_remote_exists() { git_config_has "remote.$1.url"; }
crowbar_remote_exists() { in_repo git_config_has "crowbar.remote.$1.urlbase"; }
# Add a git remote to a repository.
add_remote() {
# $1 = name of the remote to add.
# $2 = base part of the repo to add.
# $3 = Name of the remote repository to add.
git_remote_exists "$1" && return 0
local repo
repo="${3:-$(get_repo_name)}" || exit 1
repo="$2/$repo"
git ls-remote "$repo" refs/heads/master &>/dev/null || {
debug "No git repo at $repo, skipping."
return 0
}
git remote add "$1" "$repo"
git fetch "$1"
}
rm_remote() { git_remote_exists "$1" && git remote rm "$1"; }
rename_remote() { git_remote_exists "$1" && git remote rename "$1" "$2"; }
# Change the URL that a remote points at.
set_url_remote() {
# $1 = name of the remote
# $2 = new baseurl for the remote
# $3 = Name of the remote repository.
git_remote_exists "$1" || return 0
local repo
repo="${3:-$(get_repo_name)}" || exit 1
repo="$2/$repo"
git ls-remote "$repo" refs/heads/master &>/dev/null || {
debug "No git repo at $repo, skipping."
return 1
}
git remote set-url "$1" "$repo"
git fetch "$1"
}
# Synchronize a remote definition as closely as possible
# This assumes that you are already in the repo you want to sync.
__sync_remotes() {
# $1 = function to call to test to see remote is already valid.
# $2 = function to call to get the proper remote URL.
local remote urlbase remote_repo barclamp_repo
for remote in "${DEV_SORTED_REMOTES[@]}"; do
urlbase="${DEV_REMOTE_URLBASE[$remote]}"
[[ $urlbase ]] || continue
# Do nothing if this remote is correct, otherwise set/add as appropriate.
remote_repo=$(git_url_for_remote "$remote") && \
$1 ${remote_repo%.git} && continue
remote_repo="$($2 "$urlbase")" || continue
remote_repo="${remote_repo#$urlbase/}"
debug "Synchronizing remote $remote for $PWD"
if git_remote_exists "$remote"; then
set_url_remote "$remote" "$urlbase" "$remote_repo" || continue
else
add_remote "$remote" "$urlbase" "$remote_repo" || \
die "Could not add new remote $remote for $PWD in sync_barclamp_remotes"
fi
local update_tracking=true
done
if [[ $update_tracking ]]; then
update_tracking_branches
fi
}
test_barclamp_remote() {
# $1 = local repo name
# $2 = remote repo
[[ $2 = $urlbase/barclamp-$1 || \
$2 = $urlbase/crowbar/barclamps/$1 ]]
}
# Synchronize remote definitions in the barclamps to match what is in the
# main Crowbar repository as closely as possible.
sync_barclamp_remotes() {
in_barclamp "$1" __sync_remotes "test_barclamp_remote $1" "probe_barclamp_remote $1"
}
test_simple_remote() [[ $2 = $urlbase/$1 ]]
get_simple_remote() {
test_remote "$2/$1" && echo "$2/$1"
}
sync_crowbar_remotes() {
in_repo __sync_remotes "test_simple_remote crowbar" "get_simple_remote crowbar"
}
sync_ci_remotes() (
cd "$LOCAL_PULL_TRACKING"
__sync_remotes "test_simple_remote $CI_TRACKING_REPO" "get_simple_remote $CI_TRACKING_REPO"
)
# Show saved parameters for either a specific remote or all of them.
show_remote() {
if [[ $1 ]]; then
[[ ${DEV_REMOTE_PRIORITY[$1]} ]] || \
die "$1 is not a remote!"
echo "$1 urlbase=${DEV_REMOTE_URLBASE[$remote]} priority=${DEV_REMOTE_PRIORITY[$remote]}"
exit 0
fi
for remote in "${DEV_SORTED_REMOTES[@]}"; do
echo "$remote urlbase=${DEV_REMOTE_URLBASE[$remote]} priority=${DEV_REMOTE_PRIORITY[$remote]}"
done
}
# This function sets up the hashes that we use to handle our remotes.
# It should be called whenever we need to update them, which is usually
# at startup time.
set_sorted_remotes() {
local line remote
local remote_re='^crowbar\.remote\.([^.]+)\.(urlbase|priority)=(.*)$'
while read line; do
[[ $line =~ $remote_re ]] || continue
local remote="${BASH_REMATCH[1]}" key="${BASH_REMATCH[2]}"
local val="${BASH_REMATCH[3]}"
case $key in
priority) DEV_REMOTE_PRIORITY[$remote]=$val;;
urlbase)
DEV_REMOTE_URLBASE[$remote]=$val
[[ ${DEV_REMOTE_PRIORITY[$remote]} ]] && continue
DEV_REMOTE_PRIORITY[$remote]=50;;
*) die "Cannot happen in sorted_remotes!"
esac
done < <(in_repo git config --list |grep '^crowbar\.remote\.')
for remote in "${!DEV_REMOTE_PRIORITY[@]}"; do
__remotes[${DEV_REMOTE_PRIORITY[$remote]}]+="$remote "
done
DEV_SORTED_REMOTES=(${__remotes[@]})
}
remote_is_github() {
git_remote_exists "$1" && [[ ${DEV_REMOTE_URLBASE[$1]} =~ $github_re ]]
}
remote_github_account() {
remote_is_github "$1" || return 1
echo "${BASH_REMATCH[2]}"
}
# Figure out which internal remote-handling function to call and how,
# and then do it.
remote_wrapper() {
# $1 = one of "add", "rm", "set-url", "show"
# $2 = name of remote
# $3 = base part of the remote URL. Only used for add and set-url.
local remote urlbase action bc cfgaction=()
local need_tracking_update need_remote_resort
remote="$2"
urlbase="$3"
case $1 in
add)
action=add_remote
if [[ ! $urlbase && $remote =~ $github_re ]]; then
urlbase="$2"
remote="${BASH_REMATCH[2]}"
fi
if crowbar_remote_exists "${remote}"; then
die "We already have a remote for $remote."
fi
cfgaction=("in_repo git config crowbar.remote.${remote}.urlbase $urlbase")
need_tracking_update=true
need_remote_resort=true
;;
rm)
action=rm_remote
if ! crowbar_remote_exists "${remote}"; then
die "No remote named $remote to remove."
fi
(( ${#DEV_SORTED_REMOTES[@]} == 1)) && \
die "$1 is your last remote. Can't remove it."
cfgaction=("in_repo git config --unset crowbar.remote.${remote}.urlbase")
need_tracking_update=true
need_remote_resort=true
;;
rename)
action=rename_remote
if ! crowbar_remote_exists "$remote"; then
die "Cannot rename $remote to $urlbase, $remote does not exist."
elif crowbar_remote_exists "$urlbase"; then
die "Cannot rename $remote to $urlbase, $urlbase already exists!"
fi
cfgaction=("in_repo git config --rename-section crowbar.remote.${remote} crowbar.remote.${urlbase}")
in_repo git_config_has "crowbar.backup.${remote}.method" && \
cfgaction+=("in_repo git config --rename-section crowbar.backup.${remote} crowbar.backup.${urlbase}")
need_remote_resort=true
;;
set-url)
action=set_url_remote
if ! crowbar_remote_exists "${remote}"; then
die "Cannot set-url for a remote your have not added."
fi
cfgaction=("in_repo git config crowbar.remote.${remote}.urlbase $urlbase")
;;
show) shift;
show_remote "$1"
return;;
sync)
for bc in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
sync_barclamp_remotes "${bc##*/}"
done
sync_crowbar_remotes
sync_ci_remotes
return
;;
priority)
crowbar_remote_exists "$2" || \
die "Remote $2 must be configured before you can set its priority."
[[ $3 =~ [0-9]+ ]] && (($3 > 0 && $3 <= 100)) || \
die "Priority must be a number between 1 and 100"
in_repo git config "crowbar.remote.$remote.priority" "$3"
set_sorted_remotes
update_all_tracking_branches
return;;
''|help) die "Please pass one of add, rm, rename, set-url, sync, priority, or show.";;
*) die "Unknown action $1 for remote.";;
esac
for bc in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
debug "Barclamp: ${bc##*/}"
in_barclamp "${bc##*/}" $action "$remote" "$urlbase" "barclamp-${bc##*/}"
done
debug "Crowbar:"
if [[ $action != sync_barclamp_remotes ]]; then
in_repo $action "$remote" "$urlbase"
fi
if [[ -d $LOCAL_PULL_TRACKING/.git ]]; then
debug "CI Tracking:"
(cd "$LOCAL_PULL_TRACKING"; $action "$remote" "$urlbase")
fi
local c
for c in "${cfgaction[@]}"; do
$c
done
[[ $need_remote_resort = true ]] && set_sorted_remotes
[[ $need_tracking_update = true ]] && update_all_tracking_branches
}
# Handle swizzling up the remotes needed to handle migrating from
# dev metadata v1 to metadata v2
migrate_1_to_2() {
local url_re='^(https?|ssh|git|file)://' url
url=$(get_repo_cfg "remote.origin.url") || return 0
[[ $url =~ $url_re ]] || \
die "Location for origin remote not in canonical form!" \
"Please use git remote set-url to update it so that it is in URL form" \
"(starting with protocol://). See the git-remote man page."
if [[ $url =~ $github_re ]]; then
local remote_name="${BASH_REMATCH[2]}"
local urlpart="https://github.com/${BASH_REMATCH[2]}"
else
local remote_name="upstream"
local urlpart="${url%/crowbar*}"
urlpart=${urlpart}
fi
remote_wrapper add "$remote_name" "$urlpart"
remote_wrapper priority "$remote_name" 5
for rb_source in "${!DEV_REMOTE_BRANCHES[@]}"; do
rb="${DEV_REMOTE_BRANCHES[$rb_source]}"
[[ $rb_source = origin ]] && rb_source="$remote_name"
if [[ ${DEV_REMOTE_SOURCES[$rb_source]} ]]; then
crowbar_remote_exists "$rb_source" && continue
remote_available "$rb_source" || continue
remote_wrapper add "$rb_source" "${DEV_REMOTE_SOURCES[$rb_source]}"
fi
done
crowbar_remote_exists origin && remote_wrapper rm origin
in_repo git_remote_exists origin && in_repo git remote rm origin
for bc in "${CROWBAR_DIR}/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
in_barclamp "${bc##*/}" git_remote_exists origin && \
in_barclamp "${bc##*/}" git remote rm origin
done
for bc in $(in_repo git config --list |grep '^crowbar.backup'); do
in_repo git config --unset "${bc%%=*}"
done
unset b rb rb_source DEV_REMOTE_BRANCHES DEV_REMOTE_SOURCES
}
# Wrapper to handle calling any future migration functions correctly.
migrate() {
# $1 = source revision
# $2 = target revision
local src_rev=${1:-0} target_rev=${2:-$DEV_VERSION} i
if ((src_rev == 0)) && in_repo git_config_has crowbar.dev.version; then
src_rev=$(get_repo_cfg crowbar.dev.version)
fi
((src_rev == target_rev)) && return
[[ $DEV_AVAILABLE_REMOTES ]] && {
echo "dev setup must have access to all your configured remotes to continue."
echo "DEV_AVAILABLE_REMOTES is set, indicating that some of your remotes are not available."
echo "dev setup is aborting."
exit 1
} >&2
((src_rev > target_rev)) && \
die "Cannot migrate down from $src_rev to $target_rev."
for ((i=src_rev; i < target_rev; i++)); do
grep -q 'function' < <(LC_ALL=C type "migrate_${i}_to_$(($i + 1))" 2>/dev/null) || continue
"migrate_${i}_to_$(($i + 1))" || \
die "Migration from $src_rev to $target_rev failed at rev $i"
done
}
# Perform initial setup. If you make any changes to this function, take
# care to make sure it stays idempotent.
setup() {
local p remote release br head
local -A touched_branches barclamps
[[ $(in_repo git symbolic-ref HEAD) = refs/heads/master ]] || \
die "You must be on the master branch in Crowbar to run setup!"
barclamps_are_clean || \
die "Crowbar repo must be clean before trying to set things up!"
# Make sure we have Github login credentials if our upstream remote
# was cloned from Github.
migrate "$(get_repo_cfg crowbar.dev.version)" "$DEV_VERSION"
if remote_is_github "$(origin_remote)" && [[ $1 != '--no-github' ]]; then
if [[ $DEV_GITHUB_ID ]]; then
debug "Validating your Github username ($DEV_GITHUB_ID):"
if curl_and_res -f \
"https://api.github.com/users/$DEV_GITHUB_ID" &>/dev/null; then
debug "$DEV_GITHUB_ID is a valid Github user."
else
die "Could not validate $DEV_GITHUB_ID with Github." \
"Please edit $HOME/.build-crowbar.conf and" \
"$HOME/.netrc to ensure your Githib credentials are correct."
fi
debug "Validating your password in .netrc:"
if curl_and_res -f "https://api.github.com/user" &>/dev/null; then
debug "Password for $DEV_GITHUB_ID OK."
else
die "Unable to authenticate as $DEV_GITHUB_ID at Github." \
"Please make sure your passwords in $HOME/.netrc are correct."
fi
else
local DEV_GITHUB_PASSWD
read -p "Enter your Github username: " DEV_GITHUB_ID
curl_and_res -f \
"https://api.github.com/users/$DEV_GITHUB_ID" &>/dev/null || \
die "Could not verify that $DEV_GITHUB_ID is a valid Github user."
while [[ $p != $DEV_GITHUB_PASSWD || ! $p ]]; do
[[ $p ]] && echo "Passwords did not match, try again."
read -s -p "Enter your Github password: " DEV_GITHUB_PASSWD
echo
read -s -p "Enter your Github password again: " p
echo
done
curl_and_res -f -u "$DEV_GITHUB_ID:$DEV_GITHUB_PASSWD" \
https://api.github.com/user &>/dev/null || {
echo "Unable to authenticate as Github user $DEV_GITHUB_ID." >&2
die "Please try again when you have Github access."
}
for mach in github.com api.github.com; do
grep -q "^$mach" "$HOME/.netrc" &>/dev/null && continue
printf "\nmachine %s login %s password %s\n" \
"$mach" "$DEV_GITHUB_ID" "$DEV_GITHUB_PASSWD" >> "$HOME/.netrc"
done
chmod 600 "$HOME/.netrc"
printf "DEV_GITHUB_ID=%q\n" "$DEV_GITHUB_ID" >> "$HOME/.build-crowbar.conf"
fi
fi
# Set up a personal remote if needed.
if [[ $DEV_GITHUB_ID ]] && remote_is_github "$(origin_remote)" && \
[[ ${BASH_REMATCH[1]} != $DEV_GITHUB_ID ]] && \
! crowbar_remote_exists personal; then
echo "Adding remote for personal fork of crowbar on Github."
remote_wrapper add personal "https://github.com/$DEV_GITHUB_ID"
remote_wrapper priority personal 95
fi
in_repo git config branch.autosetupmerge true &>/dev/null
in_repo git config crowbar.backup.method per-remote
# Set up the rest of our upstream remotes.
if crowbar_remote_exists personal && \
! in_repo git_config_has "crowbar.backup.$(origin_remote).method"; then
in_repo git config crowbar.backup.$(origin_remote).method remote
in_repo git config crowbar.backup.$(origin_remote).remote personal
fi
clone_barclamps all
in_repo git config crowbar.dev.version "$DEV_VERSION"
in_repo git_config_has 'crowbar.build' && return 0
in_repo git config 'crowbar.build' 'development/master'
switch_release
}
# Test repository $1 to see if commit $2 is in sync with $3.
# In this case, "in sync" is defined as:
# * $2 and $3 point at the same commit, or
# * There are no commits in the set of all commits reachable from $3 that
# are not also reachable from $2.
branches_synced() {
# $1 = repository to operate in
# $2 = local branch to test
# $3 = remote branch to test
[[ -d $1/.git || -f $1/.git ]] || \
die "branches_synced: $1 is not a git repo"
[[ $VERBOSE2 ]] && echo "Checking to see if out of sync: $2 $3"
(cd "$1"; git rev-parse --verify -q "$2" &>/dev/null) || \
return 1
(cd "$1"; git rev-parse --verify -q "$3" &>/dev/null) || \
return 1
# $2 and $3 resolve to the same commit, they are in sync.
(cd "$1"; [[ $(git rev-parse "$2") = $(git rev-parse "$3") ]] ) && return 0
# Test to see if there are any commits in $3 that are not
# reachable from $2. If there are, then the branches are not synced.
(cd "$1"; [[ ! $(git rev-list "$2..$3") ]] ) && return 0
return 1
}
# Back up any local commits that are not already present on our upstreams,
# or that have not already been backed up.
backup_repo() {
local id bc branch remote branch_get_func branch_backup_func
local -A branches backup_remotes
local backup_method target_remote target_method target_prefix
local remote_re='^refs/remotes/([^/]+)'
while read id branch; do
if [[ $branch = refs/heads/* ]]; then
# This is a local ref, see if it needs backed up.
branch=${branch#refs/heads/}
# Does this branch have a remote, and is it one that Crowbar cares about?
remote=$(remote_for_branch "$branch") || continue
crowbar_remote_exists "$remote" && remote_available "$remote" || continue
# If we already know what the backup remote is for this branch is based
# on the upstream remote for the branch is, carry on.
[[ ${backup_remotes[$remote]} ]] || {
# Otherwise, firgure out whether this branch is from a remote we are backing up.
backup_remote=$(get_repo_cfg "crowbar.backup.$remote.remote") && \
crowbar_remote_exists "$backup_remote" && \
git_remote_exists "$backup_remote" && \
remote_available "$backup_remote" || \
continue
backup_remotes[$remote]="${backup_remote}"
}
if ! git rev-parse --verify -q \
"refs/remotes/${backup_remotes[$remote]}/$branch" &> /dev/null || \
[[ $(git rev-parse "refs/remotes/${backup_remotes[$remote]}/$branch") != \
$(git rev-parse "refs/heads/${branch}") ]]; then
# Only back up branches that either don't exist on the backup remote
# for this branch, or that do exist but don't point at the commit we want.
branches[${backup_remotes[$remote]}]+="$branch "
fi
elif [[ $branch =~ $remote_re ]]; then
# This is a remote ref, see if we care about it and need to delete it.
remote="${BASH_REMATCH[1]}"
# Is this a remote we are using as a backup target?
is_in "$remote" "${backup_remotes[*]}" || continue
branch=${branch#refs/remotes/${remote}/}
# Skip any pull request branches.
[[ $branch = pull-req-* || $branch = HEAD ]] && continue
# Skip any branches that have a local head.
git show-ref --verify --quiet "refs/heads/$branch" && continue
# Schedule the branch for deletion.
branches["$remote"]+=":${branch} "
fi
done < <(LC_ALL=C git show-ref |sort -k2) # Sort ensures that local refs always come first.
# Now, we know what to back up, what to ignore, and what to delete.
# Make it so.
for remote in "${!branches[@]}"; do
[[ ${branches[$remote]} ]] || continue
git_push -f "$remote" ${branches[$remote]}
done
}
# Back up everything to your persoal remote.
backup_everything() {
local bc remote branches=() branch remotes=()
local -A touched_bcs
crowbar_remote_exists personal && remote_is_github personal || \
die "You must have a remote named personal to back things up." \
"Try running dev setup or dev remote add personal <personal repo urlbase>"
# Back up all the barclamps that are references as submodules for
# branches that this remote is "authoritative" for.
for bc in "$CROWBAR_DIR/barclamps/"*; do
[[ -d $bc/.git || -f $bc/.git ]] || continue
debug "Backing up barclamp ${bc##*/}"
in_barclamp "${bc##*/}" backup_repo
done
debug "Backing up Crowbar"
in_repo backup_repo
}
# Misnamed, this can pin a release, specific build,
# or a barclamp in a specific build.
# pin_release and unpin_release use this function to do all their work.
__pin_release() {
[[ $1 ]] || die "Must pass a release, release/build, or release/build/barclamp to pin."
local bc build ref
if barclamp_exists_in_build "$1"; then
build=${1%/*} bc=${1##*/}
ref="${2:-$(in_barclamp "$bc" git rev-parse HEAD)}"
barclamp_branch_for_build "$build" "$bc" "$ref" || return 1
elif build_exists "$1"; then
for bc in $(barclamps_in_build "$1"); do
__pin_release "$1/$bc" "$2" || return 1
done
elif release_exists "$1"; then
for build in $(builds_in_release "$1"); do
__pin_release "$1/$build" "$2" || return 1
done
else
echo "Don't know what $1 is!" >&2
return 1
fi
}
# Pin barclamps in a release to a specific tag or to their current HEAD.
pin_release() {
barclamps_are_clean || die "Crowbar must be clean before pinning $1"
if __pin_release "$@"; then
git commit -m "Pinned barclamps in $1 to ${2:-current HEAD}"
switch_release
return 0
else
in_repo git rm -r --cached releases/
in_repo git checkout HEAD -- releases
echo "Could not pin $1 to ${2:-current HEAD}, leaving things unchanged."
return 1
fi
}
# Reset barclamps in a release back to tracking the correct release branch.
unpin_release() {
[[ $1 ]] || die "Must pass a release, release/build, or release/build/barclamp to unpin."
local bc build
if barclamp_exists_in_build "$1"; then
build=${1%/*}
elif build_exists "$1"; then
build="$1"
elif release_exists "$1"; then
build="$1/master"
else
die "Don't know what $1 is!"
fi
pin_release "$1" "$(build_branch "$build")"
}
# Find branches in barclamps for a given release that are not synced,
# and show them.
find_unsynced_branches_for_release() {
# $1 = release
local bc rel line changes_found=false sha ref br build
local barclamps=() builds=()
rel="${1:-$(current_release)}"
if release_exists "$rel"; then
barclamps=($(barclamps_in_release "$rel"))
builds=($(builds_in_release "$rel"))
elif build_exists "$rel"; then
barclamps=($(barclamps_in_build "$rel"))
builds=("$rel")
else
die "Release $rel does not exist, cannot find unsynced changes for it!"
fi
for bc in "${barclamps[@]}"; do
local branches=()
local from to br
for build in "${builds[@]}"; do
__barclamp_exists_in_build "$rel/$build/$bc" || continue
ref=$(barclamp_branch_for_build "$rel/$build" "$bc")
[[ $ref = empty-branch ]] && \
die "Barclamp $bc should exist for $rel/$build, but it is set to empty-branch!"
break
done
upstream="$(in_barclamp "$bc" remote_for_branch "$ref")" || continue
case $changes_from in
remote) from="refs/heads/$ref" to="refs/remotes/$upstream/$ref";;
local) from="refs/remotes/$upstream/$ref" to="refs/heads/$ref";;
*) die "Cannot happen in find_unsynced_branches_for_release"
esac
in_barclamp "$bc" branches_synced '.' "$from" "$to" && continue
changes_found=true
printf "%s: %s -> %s\n" "$bc" "${from#refs/*/}" "${to#refs/*/}"
in_barclamp "$bc" git --no-pager log --oneline "${from#refs/*/}..${to#refs/*/}"
echo
done
if [[ $changes_found = false ]]; then
debug "No unsynced changes for release $rel"
return 1
fi
return 0
}
find_local_changed_branches_for_release() {
local changes_from="local"
find_unsynced_branches_for_release "$@"
}
find_remote_changed_branches_for_release() {
local changes_from="remote"
find_unsynced_branches_for_release "$@"
}
# Merge (or rebase) changes into the specificed branch from the corresponding
# branch on the specified remote.
merge_or_rebase_from_remote() {
# $1 = remote
# $2 = local branch
local remote branch rebase_temp
remote="$1" branch="$2" merge_temp="$branch-$btemp"
git rev-parse --verify -q "$remote/$branch" &>/dev/null || return 0
branches_synced "." "refs/heads/$branch" "refs/remotes/$remote/$branch" && return 0
quiet_checkout "$branch" || return 1
git branch -f --no-track "$merge_temp" "$branch"
if crowbar_remote_exists "$remote" && \
[[ ! $DEV_FROM_REMOTES ]]; then
if git rebase -p -q "$remote/$branch" "$branch" &>/dev/null; then
debug " Rebased $branch onto $remote/$branch"
return 0
else
git rebase --abort
git reset --hard "$merge_temp"
debug " Rebase failed, will try merge."
fi
fi
if git merge -q "$remote/$branch"; then
debug " Merged $remote/$branch merged into $branch"
return 0
fi
git merge --abort
git reset --hard "$merge_temp"
git branch -D "$merge_temp"
debug " Merge conflicts merging $remote/$branch into $branch"
return 1
}
# Handle merges across releases.
merge_releases() {
# $@ = releases to merge.
local rel branch start_ref build bc
bc="${repo##*/}"
start_ref=$(git rev-parse HEAD)
for rel in "$@"; do
[[ $rel ]] || continue
[[ $rel = $(current_release) ]] && continue
release_exists "$rel" || continue
for branch in $(barclamp_branches_for_release "$rel" "$bc"); do
git merge -q "$branch" && continue
git merge --abort
{
echo "Barclamp ${repo##*/}:"
echo " Merging releases $@ into $(current_release) failed."
echo " Dropping to a shell for you to fix up."
echo " If you want to abort this merge, exit the shell with 'exit 1'"
/bin/bash
} || {
git merge --abort
git branch "${head#refs/heads/}" "$start_ref"
debug "Barclamp ${repo##*/}: Merge of $br from $rel into $cur_rel failed."
return 1
}
done
done
}
# Merges in changes into all local branches from their upstreams.
# Assumes that upstream commits have already been fetched from the proper
# remotes by running dev fetch.
sync_repo() (
local branch head b rel bc remote ref repo="$1" res=0
shift
# $repo = dir to CD to initially.
# Assumes that remote has already been fetched.
cd "$repo"
# Repo is not clean, we will refuse to merge in any case.
git_is_clean || exit 1
# Merge upstream branches from our remotes
head=$(git symbolic-ref HEAD)
if [[ $head != refs/heads/* ]]; then
head=$(git rev-parse HEAD)
if [[ ! $head ]]; then
debug "Barclamp ${repo##*/}: Cannot find head commit."
return 1
fi
else
head=${head#refs/heads/}
fi
while read ref branch; do
branch=${branch#refs/heads/}
remote="$(remote_for_branch "$branch")" || continue
merge_or_rebase_from_remote "$remote" "$branch" && continue
res=1
done < <(git show-ref --heads)
quiet_checkout "${head#refs/heads/}"
if [[ $res = 1 ]]; then
debug "Merge conflicts detected when syncing barclamp ${repo##*/} with $remote."
debug "Please fix them up locally and retry the sync operation."
return 1
fi
if [[ $@ ]] && ! merge_releases "$@"; then
debug "Unfixed merge conflicts while merging releases."
return 2
fi
return 0
)
# Either unwind any merges/rebases performed as part of a sync or
# commit them.
unwind_or_commit_barclamp_syncs() {
# $1 = "unwind" or "commit"
local bc
for bc in "$CROWBAR_DIR/barclamps/"*; do
(
cd "$bc"
while read sha ref; do
[[ $ref = *-$btemp ]] || continue
ref=${ref#refs/heads/}
if [[ $1 = unwind ]]; then
debug "${bc##*/}: Unwinding last sync of ${ref%-$btemp}"
git branch -f "${ref%-$btemp}" "$ref"
fi
git branch -D "$ref"
done < <(git show-ref --heads)
)
done
}
# Merge all changes from our upstreams for all barclamps and the main Crowbar
# repository.
sync_everything() {
local unsynced_barclamps=()
local b u head res=0 ref branch rel
local btemp="$$-${RANDOM}"
barclamps_are_clean || exit 1
# Do barclamps first.
for b in "$CROWBAR_DIR/barclamps/"*; do
debug "Syncing barclamp ${b##*/}:"
sync_repo "$CROWBAR_DIR/barclamps/${b##*/}" "$@"
case $? in
1) unsynced_barclamps+=("${b##*/}");;
2)
debug "Merging releases aborted."
debug "Undoing any merges that succeeded:"
unwind_or_commit_barclamp_syncs unwind
return 1;;
esac
done
if [[ $unsynced_barclamps ]]; then
echo "Unable to sync:" >&2
for b in "${unsynced_barclamps[@]}"; do
printf " %s\n" "$b"
done
echo "Unwinding syncs that did succeed:"
unwind_or_commit_barclamp_syncs unwind
echo "Please fix things up and rerun sync."
return 1
fi
unwind_or_commit_barclamp_syncs commit
# Finished with barclamps, now for crowbar.
if in_repo git_is_clean; then
debug "Syncing crowbar"
if in_repo merge_or_rebase_from_remote "$(remote_for_branch master)" master; then
in_repo branch_exists "master-$btemp" && in_repo git branch -D "master-$btemp"
return 0
fi
echo "Please fix things up and do the merge manually."
return 1
else
debug "Main Crowbar is not clean, not updating it."
fi
}
dev_short_help() {
local cmd
echo "Available commands:"
echo
for cmd in $(for cmd in "${!DEV_SHORT_HELP[@]}"; do echo "$cmd"; done |sort); do
echo "${cmd}: ${DEV_SHORT_HELP[$cmd]}"
done
echo
echo 'For detailed help in a specific command, run dev help <command>.'
echo 'README.dev-and-workflow has a general overview of the dev tool.'
echo 'To see all the help at once, run dev help all'
}
dev_help () {
(
echo "$0: Development helper for Crowbar"
echo
if [[ $1 ]]; then
if [[ ${DEV_LONG_HELP[$1]} ]]; then
echo "$1: ${DEV_LONG_HELP[$1]}"
elif [[ $1 = all ]]; then
for cmd in $(for cmd in "${!DEV_LONG_HELP[@]}"; do echo "$cmd"; done |sort); do
echo "${cmd}: ${DEV_LONG_HELP[$cmd]}"
echo
done
else
echo "$1: No help for $1"
fi
else
dev_short_help
fi
) |less
}
# Tests to see if the given branch in a repo needs a pull request.
branch_needs_pull_req() {
# $1 = local branch
# $2 = (optional) target branch
local target="${2:-$1}"
local upstream="$to_remote/$target"
git rev-parse --verify -q "$upstream" &>/dev/null || return 1
branches_synced '.' "refs/remotes/$upstream" \
"refs/heads/$1" && return 1
return 0
}
# Push local updates for a release out to its "best" upstream, or
# whatever DEV_TO_REMOTES says to push to.
# This function tries to ensure that it will fail without doing anything
# if there will be any problems (due to authetication or whatever).
push_release() {
local rel_br release remotes remote bc res cmd build
local pushcmds=() to_remotes=()
local -A branches
release="$(current_release)"
if [[ $1 ]]; then
release_exists "$1" || \
die "Cannot push non-existent local release $1"
release="$1"
fi
local can_push=true
for bc in $(barclamps_in_release "$release"); do
for rel_br in $(barclamp_branches_for_release "$release" "$bc"); do
[[ $DEV_TO_REMOTES ]] && to_remotes=("${DEV_TO_REMOTES[@]}") || \
to_remotes=("$(in_barclamp "$bc" remote_for_branch "$rel_br")")
[[ $to_remotes ]] || to_remotes=("$(in_barclamp "$bc" origin_remote)")
for remote in "${to_remotes[@]}"; do
remote_available "$remote" || continue
if ! probe_barclamp_remote "$bc" "${DEV_REMOTE_URLBASE[$remote]}" &>/dev/null; then
debug "Barclamp $bc does not exist at $remote, skipping."
continue
fi
if ! res="$(in_barclamp "$bc" git push -nq "$remote" -- "${rel_br}:${rel_br}")"; then
debug "$bc: $rel_br -> $remote will fail."
echo "$res" >&2
exit 1
fi
debug "$bc: $rel_br -> $remote is OK."
pushcmds+=("in_barclamp $bc git_push $remote -- ${rel_br}:${rel_br}")
done
done
done
debug "Test passed, pushing branches."
for cmd in "${pushcmds[@]}"; do
if [[ $DRY_RUN = true ]]; then
echo "Would have run: $cmd"
else
$cmd || die "Could not push $rel_br to $remote after test push passed."
fi
done
debug "If you just pushed a new release, be sure and push the metadata as well."
}
# Make sure everything is up to date, and then figure out what
# barclamps and branches might need pull requests on Github.
# Once we have that figured out, print out a command line that can
# be used by dev pull-requests-gen to actually generate the pull requests.
pull_requests_prep() {
remote_available personal && remote_is_github personal || \
die "No personal remote available at Github. Cannot do pull requests."
barclamps_are_clean || exit 1
fetch_all && sync_everything || \
die "Unable to synchronize remotes for pull requests"
local barclamps_to_push=()
local to_remote="$(origin_remote)"
[[ $DEV_TO_REMOTES ]] && to_remote="$DEV_TO_REMOTES"
remote_is_github "$to_remote" || \
die "Cannot issue pull requests for $to_remote, it is not from github."
local branch bc build release=$(current_release)
local target_release="$release"
if [[ $1 = --merge ]]; then
target_release="$(find_best_parent "$release")" || \
die "No parent release to merge into!"
echo "Will generate pull requests for merge $release into $target_release" >&2
fi
for bc in $(barclamps_in_release $release); do
for branch in $(barclamp_branches_for_release "$release" "$bc"); do
local target_branch=$(barclamp_branches_for_release "$target_release" "$bc")
in_barclamp "$bc" branch_needs_pull_req "$branch" "$target_branch" || continue
barclamps_to_push+=("$bc")
break
done
done
[[ ${push_master} || ${barclamps_to_push} ]] || {
echo "Everything up to date, no pull requests are possible."
return 0
}
echo "Barclamps that might need pull requests: ${barclamps_to_push[*]-(none)}"
[[ ${push_master} ]] && echo "Crowbar needs push"
echo "Command to generate pull requests:"
echo -n "$0 pull-requests-gen --to $to_remote --release $release"
[[ $target_release != $release ]] && \
echo -n " --merge"
[[ ${push_master} ]] && \
echo -n " --branches master"
[[ ${barclamps_to_push[*]} ]] && \
echo -n " --barclamps ${barclamps_to_push[*]}"
echo
}
# Actaully generate a pull request by using make_pull_request to
# create the JSON blob that github needs, and then posting that to the
# right URL at Github.
do_pull_request() {
# $1 = url to POST to
# rest of args passed verbatim to make_pull_request helper.
local posturl="$1" lines
local -A res
shift
if [[ $DEBUG || $DRY_RUN ]]; then
make_pull_request "$@"
return
fi
lines="$( make_pull_request "$@" | \
curl_and_res -X POST --data @- \
-H "Content-Type: application/vnd.github.beta.raw+json" \
"$posturl")" || die "Error communicating with Github." \
"Please delete any pull requests that succeeded and try again."
. <(printf '%s' "$lines" | parse_yml_or_json - res) || {
die "Error parsing response from Github." \
"Response was:" \
"$lines"
"Please delete the pull requests that succeeded and try again."
}
if [[ ${res['number']} && ${res['html_url']} ]]; then
printf "Pull request %s created (%s)\n" \
"${res['number']}" "${res['html_url']}"
else
die "Pull request to $posturl with following params failed:" \
"$@" \
"Response was:" \
"$lines" \
"Please delete the pull requests that succeeded and try again."
fi
}
# Get the diffstat from the origin branch of the branch passed,
# or print an error message if there is no origin.
diffstat_from_upstream() {
local upstream=''
upstream="$to_remote/$1"
if git rev-parse --verify -q "$upstream" &>/dev/null; then
git diff --stat "$upstream" "$1"
else
echo "No origin to generate diffstat"
fi
}
# Simple helper for printing a short probably unique name for a branch.
git_ref() {
# $1 = branch
printf "%s-%s" "${1//\//-}" "$2"
}
# Make pull requests based on the command line args passed.
# These should follow the command line arguments that
# pull_requests_prep generated.
pull_requests_gen() {
# $@ = options to parse
dev_is_setup || die "You must run dev setup before trying to generate pull requests."
remote_available personal && remote_is_github personal || \
die "No personal remote available at Github. Cannot do pull requests."
local -A barclamps branches barclamp_branches bc_pulls br_pulls refs
local bc br bcr bcb title body option bc_name head release target_release
local merge_into_parent=false
local prc=0 n=1
local to_remote="$(origin_remote)"
[[ $DEV_TO_REMOTES ]] && to_remote="$DEV_TO_REMOTES"
local to_account="$(remote_github_account "$to_remote")" || \
die "Cannot issue pull requests for $to_remote, it is not from github."
# This is needed to make sure we can see parse_yml_or_json.
# It should go away at some point.
# Parse our options and validate them.
while [[ "$1" ]]; do
case $1 in
--branches)
shift
while [[ $1 && $1 != '--'* ]]; do
br="$1"
shift
in_repo branch_exists "$br" || \
die "$br is not a branch in Crowbar!"
in_repo git_remote_exists "$to_remote" || \
die "Cannot make pull request to $to_remote, we don't know about it."
br_pulls["$br"]="true"
done;;
--barclamps)
shift
while [[ $1 && $1 != '--'* ]]; do
bc="${1%%/*}"
br="${1#*/}"
[[ $bc = $br ]] && br="calculate"
barclamps["$bc"]="true"
barclamp_branches["$bc"]+=" $br"
shift
done;;
--release)
shift
release_exists "$1" || die "$1 is not a release"
release="$1"
shift;;
--merge) shift; merge_into_parent=true;;
*) die "Unknown option $1 to $0 pull-requests-gen!";;
esac
done
[[ $release ]] || release=$(current_release)
target_release=$release
if [[ $merge_into_parent = true ]]; then
target_release=$(find_best_parent "$release") || \
die "No parent release to merge into!"
fi
switch_release "$release"
# OK, sanity-check any barclamp branches we were passed.
for bc in "${!barclamps[@]}"; do
bcb=''
for br in ${barclamp_branches[$bc]}; do
[[ $br = calculate ]] && br=$(barclamp_branches_for_release "$release" "$bc")
in_barclamp "$bc" branch_exists "$br" || \
die "$br is not a branch in barclamp $bc!"
in_barclamp "$bc" git_remote_exists "$to_remote" || \
die "Cannot make pull request from barclamp $bc -- it is not present at remote $to_remote!"
is_in "$br" "$bcb" || bcb+=" $br"
done
barclamp_branches[$bc]="$bcb"
done
# Generate a very probably unique name for the pull request.
local pull_id="$(
(
for bc in "${!barclamps[@]}"; do
for bcb in ${barclamp_branches["$bc"]}; do
in_barclamp "$bc" git show-ref --hash "refs/heads/$bcb"
done
done
for br in "${!br_pulls[@]}"; do
in_repo git show-ref --hash "refs/heads/$br"
done
hostname -f
date -u '+%s.%N'
)|sha1sum |cut -d ' ' -f 1)"
for bc in "${!barclamps[@]}"; do
for bcb in ${barclamp_branches["$bc"]}; do
bc_pulls["$bc/$bcb"]="pull-req-$(in_barclamp "$bc" git_ref "$bcb" "$pull_id")"
done
done
# OK, now we know how many pull requests we have to issue.
prc=$((${#bc_pulls[@]} + ${#br_pulls[@]}))
# Get the common title and body for the pull request.
body="$(mktemp /tmp/crowbar-pull-req-XXXXXXXX)"
if [[ $DEBUG || $DRY_RUN ]]; then
title="Test"
echo "test" >> "$body"
else
echo "Enter a title for this pull request series."
echo "After you have entered a title, an editor will open, and you can"
echo "enter anything you want for the body of the pull requests."
read -p "Title: " title
if [[ $EDITOR ]]; then
$EDITOR "$body"
else
nano "$body"
fi
fi
# issue the pull requests for our barclamps.
for barclamp in "${!bc_pulls[@]}"; do
local bc=${barclamp%%/*}
local bc_base=${barclamp#*/}
in_barclamp "$bc" git_push personal "$bc_base:${bc_pulls[$barclamp]}"
local bc_target="$(barclamp_branches_for_release "$target_release" "$bc")"
local bc_head="$DEV_GITHUB_ID:${bc_pulls[$barclamp]}"
local bc_name=$(in_barclamp $bc git_url_for_remote $to_remote)
bc_name=${bc_name##*/}
bc_name=${bc_name##*:}
bc_name=${bc_name%.git}
do_pull_request \
"https://api.github.com/repos/$to_account/$bc_name/pulls" \
--title "$title [$n/$prc]" --base "$bc_target" --head "$bc_head" \
--body "@$body" \
--body "$(in_barclamp "$bc" diffstat_from_upstream "$bc_target")" \
--body "Crowbar-Pull-ID: $pull_id" \
--body "Crowbar-Release: $release"
n=$(($n + 1))
done
# Now, issue the requests for branches.
# Make sure they are ordered correctly.
for br in "${!br_pulls[@]}"; do
local br_pull_name="$(in_repo git_ref "$br" "$pull_id")"