diff --git a/README.md b/README.md
index f5e8942..1413f07 100644
--- a/README.md
+++ b/README.md
@@ -6,10 +6,56 @@ many repositories) for our CI pipelines.
# TOC
- [check_dependent_project](#check_dependent_project)
+ - [Usage](#check_dependent_project-usage)
+ - [Explanation](#check_dependent_project-explanation)
- [Implementation](#check_dependent_project-implementation)
# check_dependent_project
+## Usage
+
+Specify companions in the description of a pull request. For example, if you
+have a pull request which needs a Polkadot companion, say:
+
+```
+polkadot companion: [link]
+```
+
+The above tells the integration checks to test the pull request's branch with
+the specified PR rather than the default branch for that companion's repository.
+
+---
+
+On pull requests **which don't target master** you're able to specify the
+companion's branch in their description:
+
+```
+polkadot companion branch: [branch]
+```
+
+The above tells the script to use the specified branch from `${ORG}/polkadot`
+rather than the default branch for that companion's repository.
+
+Alternatively, it's also possible to provide a permanent override configuration
+through `--companion-overrides`. As an example:
+
+```bash
+check_dependent_project.sh \
+ --companion-overrides "
+ substrate: polkadot-v*
+ polkadot: release-v*
+ "
+```
+
+The above configures the script to use, for instance, the `release-v1.2`
+Polkadot branch for the companion in case the Substrate pull request is
+**targetting** the `polkadot-v1.2` branch - note how the suffix captured from
+the wildcard pattern, namely `1.2` from the pattern `*`, is correlated between
+those refs. This feature exists for release engineering purposes (more context
+in [issue 32](https://github.com/paritytech/pipeline-scripts/issues/32)).
+
+## Explanation
+
[check_dependent_project](./check_dependent_project.sh) implements the
[Companion Build System](https://github.com/paritytech/parity-processbot/issues/327)'s
cross-repository integration checks as a CI status. Currently the checks are
diff --git a/check_dependent_project.sh b/check_dependent_project.sh
index 0917ba7..f598467 100755
--- a/check_dependent_project.sh
+++ b/check_dependent_project.sh
@@ -24,6 +24,7 @@ set -eu -o pipefail
shopt -s inherit_errexit
. "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+. "$(dirname "${BASH_SOURCE[0]}")/github_graphql.sh"
get_arg required --org "$@"
org="$out"
@@ -37,12 +38,16 @@ github_api_token="$out"
get_arg optional --extra-dependencies "$@"
extra_dependencies="${out:-}"
+get_arg optional-many --companion-overrides "$@"
+companion_overrides=("${out[@]}")
+
set -x
this_repo_dir="$PWD"
this_repo="$(basename "$this_repo_dir")"
companions_dir="$this_repo_dir/companions"
extra_dependencies_dir="$this_repo_dir/extra_dependencies"
github_api="https://api.github.com"
+github_graphql_api="https://api.github.com/graphql"
org_github_prefix="https://github.com/$org"
org_crates_prefix="git+$org_github_prefix"
set +x
@@ -333,31 +338,48 @@ Both cases can be solved by merging master into $repo#$pr_number.
fi
}
+declare -A companion_branch_override
+companion_branch_override=()
+detect_companion_branch_override() {
+ local line="$1"
+ # detects the form "[repository] companion branch: [branch]"
+ if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+companion[[:space:]]+branch:[[:space:]]*([^[:space:]]+) ]]; then
+ companion_branch_override["${BASH_REMATCH[1]}"]="${BASH_REMATCH[2]}"
+ fi
+}
+
+declare -A pr_target_branch
+pr_target_branch=()
process_pr_description() {
local repo="$1"
local pr_number="$2"
- if ! [[ "$pr_number" =~ ^[[:digit:]]+$ ]]; then
- return
- fi
-
echo "Processing PR $repo#$pr_number"
+ local base_ref
local lines=()
while IFS= read -r line; do
- lines+=("$line")
+ if [ "${base_ref:-}" ]; then
+ lines+=("$line")
+ detect_companion_branch_override "$line"
+ else
+ base_ref="$line"
+ fi
done < <(curl \
-sSL \
-H "Authorization: token $github_api_token" \
"$github_api/repos/$org/$repo/pulls/$pr_number" | \
- jq -e -r ".body"
+ jq -e -r ".base.ref, .body"
)
# in case the PR has no body, jq should have printed "null" which effectively
# means lines will always be populated with something
+ # shellcheck disable=SC2128
if ! [ "$lines" ]; then
die "No lines were read for the description of PR $pr_number (some error probably occurred)"
fi
+ pr_target_branch["$repo"]="$base_ref"
+
for line in "${lines[@]}"; do
if [[ "$line" =~ [cC]ompanion:[[:space:]]*([^[:space:]]+) ]]; then
echo "Detected companion in the PR description of $repo#$pr_number: ${BASH_REMATCH[1]}"
@@ -372,35 +394,39 @@ patch_and_check_dependent() {
pushd "$dependent_repo_dir" >/dev/null
- # It is necessary to patch in extra dependencies which have already been
- # merged in previous steps of the Companion Build System's dependency chain.
- # For instance, consider the following dependency chain:
- # Substrate -> Polkadot -> Cumulus
- # When this script is running for Cumulus as the dependent, on Polkadot's
- # pipeline, it is necessary to patch the master of Substrate into this
- # script's branches because Substrate's master will contain the pull request
- # which was part of the dependency chain for this PR and was merged before
- # this script gets to run for the last time (after lockfile updates and before
- # merge).
- for extra_dependency in $extra_dependencies; do
- echo "Cloning extra dependency $extra_dependency to patch its default branch into $this_repo and $dependent"
- git clone \
- --depth=1 \
- "$org_github_prefix/$extra_dependency.git" \
- "$extra_dependencies_dir/$extra_dependency"
-
- echo "Patching extra dependency $extra_dependency into $this_repo_dir"
- diener patch \
- --target "$org_github_prefix/$extra_dependency" \
- --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \
- --path "$this_repo_dir/Cargo.toml"
-
- echo "Patching extra dependency $extra_dependency into $dependent"
- diener patch \
- --target "$org_github_prefix/$extra_dependency" \
- --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \
- --path Cargo.toml
- done
+ if [ "${has_overridden_dependent_ref:-}" ]; then
+ echo "Skipping extra_dependencies ($extra_dependencies) as the dependent repository's ref has been overridden"
+ else
+ # It is necessary to patch in extra dependencies which have already been
+ # merged in previous steps of the Companion Build System's dependency chain.
+ # For instance, consider the following dependency chain:
+ # Substrate -> Polkadot -> Cumulus
+ # When this script is running for Cumulus as the dependent, on Polkadot's
+ # pipeline, it is necessary to patch the master of Substrate into this
+ # script's branches because Substrate's master will contain the pull request
+ # which was part of the dependency chain for this PR and was merged before
+ # this script gets to run for the last time (after lockfile updates and before
+ # merge).
+ for extra_dependency in $extra_dependencies; do
+ echo "Cloning extra dependency $extra_dependency to patch its default branch into $this_repo and $dependent"
+ git clone \
+ --depth=1 \
+ "$org_github_prefix/$extra_dependency.git" \
+ "$extra_dependencies_dir/$extra_dependency"
+
+ echo "Patching extra dependency $extra_dependency into $this_repo_dir"
+ diener patch \
+ --target "$org_github_prefix/$extra_dependency" \
+ --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \
+ --path "$this_repo_dir/Cargo.toml"
+
+ echo "Patching extra dependency $extra_dependency into $dependent"
+ diener patch \
+ --target "$org_github_prefix/$extra_dependency" \
+ --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \
+ --path Cargo.toml
+ done
+ fi
# Patch this repository (the dependency) into the dependent for the sake of
# being able to test how the dependency graph will behave after the merge
@@ -444,38 +470,177 @@ patch_and_check_dependent() {
}
main() {
+ if ! [[ "$CI_COMMIT_REF_NAME" =~ ^[[:digit:]]+$ ]]; then
+ die "\"$CI_COMMIT_REF_NAME\" was not recognized as a pull request ref"
+ fi
+
# Set the user name and email to make merging work
git config --global user.name 'CI system'
git config --global user.email '<>'
git config --global pull.rebase false
- # Merge master into this branch so that we have a better expectation of the
- # integration still working after this PR lands.
- # Since master's HEAD is being merged here, at the start the dependency chain,
- # the same has to be done for all the companions because they might have
- # accompanying changes for the code being brought in.
- git fetch --force origin master
- git show-ref origin/master
- echo "Merge master into $this_repo#$CI_COMMIT_REF_NAME"
- git merge origin/master \
- --verbose \
- --no-edit \
- -m "Merge master into $this_repo#$CI_COMMIT_REF_NAME"
-
- discover_our_crates
-
# process_pr_description calls itself for each companion in the description on
# each detected companion PR, effectively considering all companion references
# on all PRs
process_pr_description "$this_repo" "$CI_COMMIT_REF_NAME"
+ # This PR might be targetting a custom ref (i.e. not master) through companion
+ # overrides from --companion-overrides or the PR's description, in which case
+ # it won't be proper to merge master (since it's not targetting master) before
+ # performing the companion checks
local dependent_repo_dir="$companions_dir/$dependent_repo"
if ! [ -e "$dependent_repo_dir" ]; then
- echo "Cloning $dependent_repo directly as it was not detected as a companion"
+ local dependent_clone_options=(
+ --depth=1
+ )
+
+ if [ "${pr_target_branch[$this_repo]}" == "master" ]; then
+ echo "Cloning dependent $dependent_repo directly as it was not detected as a companion"
+ elif [ "${companion_branch_override[$dependent_repo]:-}" ]; then
+ echo "Cloning dependent $dependent_repo with branch ${companion_branch_override[$dependent_repo]} from manual override"
+ dependent_clone_options+=("--branch" "${companion_branch_override[$dependent_repo]}")
+ has_overridden_dependent_ref=true
+ else
+ for override in "${companion_overrides[@]}"; do
+ echo "Processing companion override $override"
+
+ local this_repo_override this_repo_override_prefix dependent_repo_override dependent_repo_override_prefix
+ while IFS= read -r line; do
+ if [[ "$line" =~ ^[[:space:]]*$this_repo:[[:space:]]*(.*) ]]; then
+ this_repo_override="${BASH_REMATCH[1]}"
+ if [[ "$this_repo_override" =~ ^(.*)\* ]]; then
+ this_repo_override_prefix="${BASH_REMATCH[1]}"
+ fi
+ elif [[ "$line" =~ ^[[:space:]]*$dependent_repo:[[:space:]]*(.*) ]]; then
+ dependent_repo_override="${BASH_REMATCH[1]}"
+ if [[ "$dependent_repo_override" =~ ^(.*)\* ]]; then
+ dependent_repo_override_prefix="${BASH_REMATCH[1]}"
+ fi
+ fi
+ done < <(echo "$override")
+
+ if [[
+ ! ("${this_repo_override:-}") ||
+ ! ("${dependent_repo_override:-}")
+ ]]; then
+ continue
+ fi
+
+ echo "Detected override $this_repo_override for $this_repo and override $dependent_repo_override for $dependent_repo"
+
+ local base_ref_prefix="${this_repo_override_prefix:-$this_repo_override}"
+ if [ "${pr_target_branch[$this_repo]:0:${#base_ref_prefix}}" != "$base_ref_prefix" ]; then
+ continue
+ fi
+
+ local this_repo_override_suffix
+ if [ "${this_repo_override_prefix:-}" ]; then
+ this_repo_override_suffix="${pr_target_branch[$this_repo]:${#this_repo_override_prefix}}"
+ fi
+
+ dependent_clone_options+=("--branch")
+ local branch_name
+ if [[
+ ("${dependent_repo_override_prefix:-}") &&
+ ("${this_repo_override_suffix:-}")
+ ]]; then
+ branch_name="${dependent_repo_override_prefix}${this_repo_override_suffix}"
+
+ echo "Checking if $branch_name exists in $dependent_repo"
+ local response_code
+ response_code="$(curl \
+ -o /dev/null \
+ -sSL \
+ -H "Authorization: token $github_api_token" \
+ -w '%{response_code}' \
+ "$github_api/repos/$org/$dependent_repo/branches/$branch_name"
+ )"
+
+ # Sometimes the target branch found via override does not *yet* exist
+ # in the companion's repository because their release processes work
+ # differently; e.g. a release-v0.9.20 Polkadot branch might have a
+ # polkadot-v0.9.20 matching branch (notice the version) on Substrate,
+ # but not yet on Cumulus because their release approach is different.
+ # When that happens, the script has no choice other than *guess* the
+ # a replacement branch to be used for the inexistent branch.
+ if [ "$response_code" -eq 200 ]; then
+ echo "Branch $branch_name exists in $dependent_repo. Proceeding..."
+ else
+ echo "Branch $branch_name doesn't exist in $dependent_repo (status code $response_code)"
+ echo "Fetching the list of branches in $dependent_repo to find a suitable replacement..."
+
+ # The guessing for a replacement branch works by taking the most
+ # recently updated branch (ordered by commit date) which follows the
+ # pattern we've matched for the branch name. For example, if
+ # polkadot-v0.9.20 does not exist, instead use the latest (by commit
+ # date) branch following a "polkadot-v*" pattern, which happens to
+ # be polkadot-v0.9.19 as of this writing.
+ local replacement_branch_name
+ while IFS= read -r line; do
+ echo "Got candidate branch $line in $dependent_repo's refs"
+ if [ "${line:0:${#dependent_repo_override_prefix}}" == "$dependent_repo_override_prefix" ]; then
+ echo "Found candidate branch $line as the replacement of $branch_name"
+ replacement_branch_name="$line"
+ break
+ fi
+ done < <(ghgql_post \
+ "$github_graphql_api" \
+ "$github_api_token" \
+ "$(
+ ghgql_most_recent_branches_query \
+ "$org" \
+ "$dependent_repo" \
+ "$dependent_repo_override_prefix"
+ )" | jq -r '.data.repository.refs.edges[].node.name'
+ )
+
+ if [ "${replacement_branch_name:-}" ]; then
+ echo "Choosing branch $line as a replacement for $branch_name"
+ branch_name="$replacement_branch_name"
+ unset replacement_branch_name
+ else
+ die "Unable to find the replacement for inexistent branch $branch_name of $dependent_repo"
+ fi
+ fi
+ else
+ branch_name="$dependent_repo_override"
+ fi
+ dependent_clone_options+=("$branch_name")
+
+ echo "Setting up the clone of $dependent_repo with options: ${dependent_clone_options[*]}"
+ has_overridden_dependent_ref=true
+
+ break
+ done
+ fi
+
dependent_repo_dir="$this_repo_dir/$dependent_repo"
- git clone --depth=1 "$org_github_prefix/$dependent_repo.git" "$dependent_repo_dir"
+ # shellcheck disable=SC2068
+ git clone \
+ ${dependent_clone_options[@]} \
+ "$org_github_prefix/$dependent_repo.git" \
+ "$dependent_repo_dir"
fi
+ if [ "${has_overridden_dependent_ref:-}" ]; then
+ echo "Skipping master merge of $this_repo as the dependent repository's ref has been overridden"
+ else
+ # Merge master into this branch so that we have a better expectation of the
+ # integration still working after this PR lands.
+ # Since master's HEAD is being merged here, at the start the dependency chain,
+ # the same has to be done for all the companions because they might have
+ # accompanying changes for the code being brought in.
+ git fetch --force origin master
+ git show-ref origin/master
+ echo "Merge master into $this_repo#$CI_COMMIT_REF_NAME"
+ git merge origin/master \
+ --verbose \
+ --no-edit \
+ -m "Merge master into $this_repo#$CI_COMMIT_REF_NAME"
+ fi
+
+ discover_our_crates
+
patch_and_check_dependent "$dependent_repo" "$dependent_repo_dir"
}
main
diff --git a/github_graphql.sh b/github_graphql.sh
new file mode 100644
index 0000000..63bb7ff
--- /dev/null
+++ b/github_graphql.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+ghgql_most_recent_branches_query() {
+ local org="$1"
+ local repo="$2"
+ local refs_query="$3"
+
+ echo "
+ query {
+ repository(owner: \"$org\", name: \"$repo\") {
+ refs(
+ refPrefix: \"refs/heads/\",
+ first: 32,
+ query: \"$refs_query\",
+ orderBy: { field: TAG_COMMIT_DATE, direction: DESC }
+ ) {
+ edges {
+ node {
+ name
+ }
+ }
+ }
+ }
+ }
+ "
+}
+
+ghgql_post() {
+ local github_graphql_api="$1"
+ local github_api_token="$2"
+ local query="$3"
+
+ local req_body
+ req_body="{ \"query\": \"$(echo "$query" | tr -d '\n' | sed 's/"/\\"/g')\" }"
+
+ >&2 echo "
+Sending GraphQL body to $github_graphql_api
+Raw query: $query
+Request body: $req_body
+"
+
+ curl \
+ -sSL \
+ -H "Content-Type: application/json" \
+ -H "Authorization: bearer $github_api_token" \
+ -X POST \
+ -d "$req_body" \
+ "$github_graphql_api"
+}
diff --git a/utils.sh b/utils.sh
index eb8dfd5..c58faa5 100644
--- a/utils.sh
+++ b/utils.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
die() {
if [ "${1:-}" ]; then
>&2 echo "$1"
@@ -28,11 +30,13 @@ get_arg() {
local option_arg="$1"
shift
+ local args=("$@")
+
unset out
out=()
local get_next_arg
- for arg in "$@"; do
+ for arg in "${args[@]}"; do
if [ "${get_next_arg:-}" ]; then
out+=("$arg")
unset get_next_arg