diff --git a/README.md b/README.md index f5e8942..1413f07 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,56 @@ many repositories) for our CI pipelines. # TOC - [check_dependent_project](#check_dependent_project) + - [Usage](#check_dependent_project-usage) + - [Explanation](#check_dependent_project-explanation) - [Implementation](#check_dependent_project-implementation) # check_dependent_project +## Usage + +Specify companions in the description of a pull request. For example, if you +have a pull request which needs a Polkadot companion, say: + +``` +polkadot companion: [link] +``` + +The above tells the integration checks to test the pull request's branch with +the specified PR rather than the default branch for that companion's repository. + +--- + +On pull requests **which don't target master** you're able to specify the +companion's branch in their description: + +``` +polkadot companion branch: [branch] +``` + +The above tells the script to use the specified branch from `${ORG}/polkadot` +rather than the default branch for that companion's repository. + +Alternatively, it's also possible to provide a permanent override configuration +through `--companion-overrides`. As an example: + +```bash +check_dependent_project.sh \ + --companion-overrides " + substrate: polkadot-v* + polkadot: release-v* + " +``` + +The above configures the script to use, for instance, the `release-v1.2` +Polkadot branch for the companion in case the Substrate pull request is +**targetting** the `polkadot-v1.2` branch - note how the suffix captured from +the wildcard pattern, namely `1.2` from the pattern `*`, is correlated between +those refs. This feature exists for release engineering purposes (more context +in [issue 32](https://github.com/paritytech/pipeline-scripts/issues/32)). + +## Explanation + [check_dependent_project](./check_dependent_project.sh) implements the [Companion Build System](https://github.com/paritytech/parity-processbot/issues/327)'s cross-repository integration checks as a CI status. Currently the checks are diff --git a/check_dependent_project.sh b/check_dependent_project.sh index 0917ba7..f598467 100755 --- a/check_dependent_project.sh +++ b/check_dependent_project.sh @@ -24,6 +24,7 @@ set -eu -o pipefail shopt -s inherit_errexit . "$(dirname "${BASH_SOURCE[0]}")/utils.sh" +. "$(dirname "${BASH_SOURCE[0]}")/github_graphql.sh" get_arg required --org "$@" org="$out" @@ -37,12 +38,16 @@ github_api_token="$out" get_arg optional --extra-dependencies "$@" extra_dependencies="${out:-}" +get_arg optional-many --companion-overrides "$@" +companion_overrides=("${out[@]}") + set -x this_repo_dir="$PWD" this_repo="$(basename "$this_repo_dir")" companions_dir="$this_repo_dir/companions" extra_dependencies_dir="$this_repo_dir/extra_dependencies" github_api="https://api.github.com" +github_graphql_api="https://api.github.com/graphql" org_github_prefix="https://github.com/$org" org_crates_prefix="git+$org_github_prefix" set +x @@ -333,31 +338,48 @@ Both cases can be solved by merging master into $repo#$pr_number. fi } +declare -A companion_branch_override +companion_branch_override=() +detect_companion_branch_override() { + local line="$1" + # detects the form "[repository] companion branch: [branch]" + if [[ "$line" =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+companion[[:space:]]+branch:[[:space:]]*([^[:space:]]+) ]]; then + companion_branch_override["${BASH_REMATCH[1]}"]="${BASH_REMATCH[2]}" + fi +} + +declare -A pr_target_branch +pr_target_branch=() process_pr_description() { local repo="$1" local pr_number="$2" - if ! [[ "$pr_number" =~ ^[[:digit:]]+$ ]]; then - return - fi - echo "Processing PR $repo#$pr_number" + local base_ref local lines=() while IFS= read -r line; do - lines+=("$line") + if [ "${base_ref:-}" ]; then + lines+=("$line") + detect_companion_branch_override "$line" + else + base_ref="$line" + fi done < <(curl \ -sSL \ -H "Authorization: token $github_api_token" \ "$github_api/repos/$org/$repo/pulls/$pr_number" | \ - jq -e -r ".body" + jq -e -r ".base.ref, .body" ) # in case the PR has no body, jq should have printed "null" which effectively # means lines will always be populated with something + # shellcheck disable=SC2128 if ! [ "$lines" ]; then die "No lines were read for the description of PR $pr_number (some error probably occurred)" fi + pr_target_branch["$repo"]="$base_ref" + for line in "${lines[@]}"; do if [[ "$line" =~ [cC]ompanion:[[:space:]]*([^[:space:]]+) ]]; then echo "Detected companion in the PR description of $repo#$pr_number: ${BASH_REMATCH[1]}" @@ -372,35 +394,39 @@ patch_and_check_dependent() { pushd "$dependent_repo_dir" >/dev/null - # It is necessary to patch in extra dependencies which have already been - # merged in previous steps of the Companion Build System's dependency chain. - # For instance, consider the following dependency chain: - # Substrate -> Polkadot -> Cumulus - # When this script is running for Cumulus as the dependent, on Polkadot's - # pipeline, it is necessary to patch the master of Substrate into this - # script's branches because Substrate's master will contain the pull request - # which was part of the dependency chain for this PR and was merged before - # this script gets to run for the last time (after lockfile updates and before - # merge). - for extra_dependency in $extra_dependencies; do - echo "Cloning extra dependency $extra_dependency to patch its default branch into $this_repo and $dependent" - git clone \ - --depth=1 \ - "$org_github_prefix/$extra_dependency.git" \ - "$extra_dependencies_dir/$extra_dependency" - - echo "Patching extra dependency $extra_dependency into $this_repo_dir" - diener patch \ - --target "$org_github_prefix/$extra_dependency" \ - --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \ - --path "$this_repo_dir/Cargo.toml" - - echo "Patching extra dependency $extra_dependency into $dependent" - diener patch \ - --target "$org_github_prefix/$extra_dependency" \ - --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \ - --path Cargo.toml - done + if [ "${has_overridden_dependent_ref:-}" ]; then + echo "Skipping extra_dependencies ($extra_dependencies) as the dependent repository's ref has been overridden" + else + # It is necessary to patch in extra dependencies which have already been + # merged in previous steps of the Companion Build System's dependency chain. + # For instance, consider the following dependency chain: + # Substrate -> Polkadot -> Cumulus + # When this script is running for Cumulus as the dependent, on Polkadot's + # pipeline, it is necessary to patch the master of Substrate into this + # script's branches because Substrate's master will contain the pull request + # which was part of the dependency chain for this PR and was merged before + # this script gets to run for the last time (after lockfile updates and before + # merge). + for extra_dependency in $extra_dependencies; do + echo "Cloning extra dependency $extra_dependency to patch its default branch into $this_repo and $dependent" + git clone \ + --depth=1 \ + "$org_github_prefix/$extra_dependency.git" \ + "$extra_dependencies_dir/$extra_dependency" + + echo "Patching extra dependency $extra_dependency into $this_repo_dir" + diener patch \ + --target "$org_github_prefix/$extra_dependency" \ + --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \ + --path "$this_repo_dir/Cargo.toml" + + echo "Patching extra dependency $extra_dependency into $dependent" + diener patch \ + --target "$org_github_prefix/$extra_dependency" \ + --crates-to-patch "$extra_dependencies_dir/$extra_dependency" \ + --path Cargo.toml + done + fi # Patch this repository (the dependency) into the dependent for the sake of # being able to test how the dependency graph will behave after the merge @@ -444,38 +470,177 @@ patch_and_check_dependent() { } main() { + if ! [[ "$CI_COMMIT_REF_NAME" =~ ^[[:digit:]]+$ ]]; then + die "\"$CI_COMMIT_REF_NAME\" was not recognized as a pull request ref" + fi + # Set the user name and email to make merging work git config --global user.name 'CI system' git config --global user.email '<>' git config --global pull.rebase false - # Merge master into this branch so that we have a better expectation of the - # integration still working after this PR lands. - # Since master's HEAD is being merged here, at the start the dependency chain, - # the same has to be done for all the companions because they might have - # accompanying changes for the code being brought in. - git fetch --force origin master - git show-ref origin/master - echo "Merge master into $this_repo#$CI_COMMIT_REF_NAME" - git merge origin/master \ - --verbose \ - --no-edit \ - -m "Merge master into $this_repo#$CI_COMMIT_REF_NAME" - - discover_our_crates - # process_pr_description calls itself for each companion in the description on # each detected companion PR, effectively considering all companion references # on all PRs process_pr_description "$this_repo" "$CI_COMMIT_REF_NAME" + # This PR might be targetting a custom ref (i.e. not master) through companion + # overrides from --companion-overrides or the PR's description, in which case + # it won't be proper to merge master (since it's not targetting master) before + # performing the companion checks local dependent_repo_dir="$companions_dir/$dependent_repo" if ! [ -e "$dependent_repo_dir" ]; then - echo "Cloning $dependent_repo directly as it was not detected as a companion" + local dependent_clone_options=( + --depth=1 + ) + + if [ "${pr_target_branch[$this_repo]}" == "master" ]; then + echo "Cloning dependent $dependent_repo directly as it was not detected as a companion" + elif [ "${companion_branch_override[$dependent_repo]:-}" ]; then + echo "Cloning dependent $dependent_repo with branch ${companion_branch_override[$dependent_repo]} from manual override" + dependent_clone_options+=("--branch" "${companion_branch_override[$dependent_repo]}") + has_overridden_dependent_ref=true + else + for override in "${companion_overrides[@]}"; do + echo "Processing companion override $override" + + local this_repo_override this_repo_override_prefix dependent_repo_override dependent_repo_override_prefix + while IFS= read -r line; do + if [[ "$line" =~ ^[[:space:]]*$this_repo:[[:space:]]*(.*) ]]; then + this_repo_override="${BASH_REMATCH[1]}" + if [[ "$this_repo_override" =~ ^(.*)\* ]]; then + this_repo_override_prefix="${BASH_REMATCH[1]}" + fi + elif [[ "$line" =~ ^[[:space:]]*$dependent_repo:[[:space:]]*(.*) ]]; then + dependent_repo_override="${BASH_REMATCH[1]}" + if [[ "$dependent_repo_override" =~ ^(.*)\* ]]; then + dependent_repo_override_prefix="${BASH_REMATCH[1]}" + fi + fi + done < <(echo "$override") + + if [[ + ! ("${this_repo_override:-}") || + ! ("${dependent_repo_override:-}") + ]]; then + continue + fi + + echo "Detected override $this_repo_override for $this_repo and override $dependent_repo_override for $dependent_repo" + + local base_ref_prefix="${this_repo_override_prefix:-$this_repo_override}" + if [ "${pr_target_branch[$this_repo]:0:${#base_ref_prefix}}" != "$base_ref_prefix" ]; then + continue + fi + + local this_repo_override_suffix + if [ "${this_repo_override_prefix:-}" ]; then + this_repo_override_suffix="${pr_target_branch[$this_repo]:${#this_repo_override_prefix}}" + fi + + dependent_clone_options+=("--branch") + local branch_name + if [[ + ("${dependent_repo_override_prefix:-}") && + ("${this_repo_override_suffix:-}") + ]]; then + branch_name="${dependent_repo_override_prefix}${this_repo_override_suffix}" + + echo "Checking if $branch_name exists in $dependent_repo" + local response_code + response_code="$(curl \ + -o /dev/null \ + -sSL \ + -H "Authorization: token $github_api_token" \ + -w '%{response_code}' \ + "$github_api/repos/$org/$dependent_repo/branches/$branch_name" + )" + + # Sometimes the target branch found via override does not *yet* exist + # in the companion's repository because their release processes work + # differently; e.g. a release-v0.9.20 Polkadot branch might have a + # polkadot-v0.9.20 matching branch (notice the version) on Substrate, + # but not yet on Cumulus because their release approach is different. + # When that happens, the script has no choice other than *guess* the + # a replacement branch to be used for the inexistent branch. + if [ "$response_code" -eq 200 ]; then + echo "Branch $branch_name exists in $dependent_repo. Proceeding..." + else + echo "Branch $branch_name doesn't exist in $dependent_repo (status code $response_code)" + echo "Fetching the list of branches in $dependent_repo to find a suitable replacement..." + + # The guessing for a replacement branch works by taking the most + # recently updated branch (ordered by commit date) which follows the + # pattern we've matched for the branch name. For example, if + # polkadot-v0.9.20 does not exist, instead use the latest (by commit + # date) branch following a "polkadot-v*" pattern, which happens to + # be polkadot-v0.9.19 as of this writing. + local replacement_branch_name + while IFS= read -r line; do + echo "Got candidate branch $line in $dependent_repo's refs" + if [ "${line:0:${#dependent_repo_override_prefix}}" == "$dependent_repo_override_prefix" ]; then + echo "Found candidate branch $line as the replacement of $branch_name" + replacement_branch_name="$line" + break + fi + done < <(ghgql_post \ + "$github_graphql_api" \ + "$github_api_token" \ + "$( + ghgql_most_recent_branches_query \ + "$org" \ + "$dependent_repo" \ + "$dependent_repo_override_prefix" + )" | jq -r '.data.repository.refs.edges[].node.name' + ) + + if [ "${replacement_branch_name:-}" ]; then + echo "Choosing branch $line as a replacement for $branch_name" + branch_name="$replacement_branch_name" + unset replacement_branch_name + else + die "Unable to find the replacement for inexistent branch $branch_name of $dependent_repo" + fi + fi + else + branch_name="$dependent_repo_override" + fi + dependent_clone_options+=("$branch_name") + + echo "Setting up the clone of $dependent_repo with options: ${dependent_clone_options[*]}" + has_overridden_dependent_ref=true + + break + done + fi + dependent_repo_dir="$this_repo_dir/$dependent_repo" - git clone --depth=1 "$org_github_prefix/$dependent_repo.git" "$dependent_repo_dir" + # shellcheck disable=SC2068 + git clone \ + ${dependent_clone_options[@]} \ + "$org_github_prefix/$dependent_repo.git" \ + "$dependent_repo_dir" fi + if [ "${has_overridden_dependent_ref:-}" ]; then + echo "Skipping master merge of $this_repo as the dependent repository's ref has been overridden" + else + # Merge master into this branch so that we have a better expectation of the + # integration still working after this PR lands. + # Since master's HEAD is being merged here, at the start the dependency chain, + # the same has to be done for all the companions because they might have + # accompanying changes for the code being brought in. + git fetch --force origin master + git show-ref origin/master + echo "Merge master into $this_repo#$CI_COMMIT_REF_NAME" + git merge origin/master \ + --verbose \ + --no-edit \ + -m "Merge master into $this_repo#$CI_COMMIT_REF_NAME" + fi + + discover_our_crates + patch_and_check_dependent "$dependent_repo" "$dependent_repo_dir" } main diff --git a/github_graphql.sh b/github_graphql.sh new file mode 100644 index 0000000..63bb7ff --- /dev/null +++ b/github_graphql.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +ghgql_most_recent_branches_query() { + local org="$1" + local repo="$2" + local refs_query="$3" + + echo " + query { + repository(owner: \"$org\", name: \"$repo\") { + refs( + refPrefix: \"refs/heads/\", + first: 32, + query: \"$refs_query\", + orderBy: { field: TAG_COMMIT_DATE, direction: DESC } + ) { + edges { + node { + name + } + } + } + } + } + " +} + +ghgql_post() { + local github_graphql_api="$1" + local github_api_token="$2" + local query="$3" + + local req_body + req_body="{ \"query\": \"$(echo "$query" | tr -d '\n' | sed 's/"/\\"/g')\" }" + + >&2 echo " +Sending GraphQL body to $github_graphql_api +Raw query: $query +Request body: $req_body +" + + curl \ + -sSL \ + -H "Content-Type: application/json" \ + -H "Authorization: bearer $github_api_token" \ + -X POST \ + -d "$req_body" \ + "$github_graphql_api" +} diff --git a/utils.sh b/utils.sh index eb8dfd5..c58faa5 100644 --- a/utils.sh +++ b/utils.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + die() { if [ "${1:-}" ]; then >&2 echo "$1" @@ -28,11 +30,13 @@ get_arg() { local option_arg="$1" shift + local args=("$@") + unset out out=() local get_next_arg - for arg in "$@"; do + for arg in "${args[@]}"; do if [ "${get_next_arg:-}" ]; then out+=("$arg") unset get_next_arg