Refactored github status get/set functions/scripts to hide secrets (3.21)

build-scripts/bin/get-github-pull-request-info

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Args:
+#   $1 - repo identifier ("project/repo")
+#   $2 - PR identifier (the PR number)
+# Env:
+#   two github fine-grained personal access tokens are needed with read/write access to commit statuses
+#   $CFENGINE_PR_TOKEN_PATH - file path containing token associated with CFEngine github organization
+#   $NTHQ_PR_TOKEN_PATH     - file path containing token associated with NorthernTechHQ github organization
+#   see get-pr-token script along-side this script for details
+# Prints:
+#   $REPO_ID $PR_ID $PR_STATUSES_URL
+#   Where:
+#     $PR_STATUSES_URL - GH API URL to set PR's statuses
+# Returns:
+#   0 - success, 1 - error
+if [ -z "$1" ]; then echo "First argument, project, is required"; exit 1; fi
+if [ -z "$2" ]; then echo "Second argument, pull request number, is required"; exit 1; fi
+
+json_out="$(mktemp)"
+
+# curl 7.88 ish supports --header @file but apparently 7.52 (on bootstrap vm (deb-9)) does not, so compose a script
+curl_script_file="$(mktemp)"
+chmod 600 "$curl_script_file"
+echo -n "curl --insecure --fail --header \"Authorization: Bearer " > "$curl_script_file"
+
+_dir=$(readlink -e "$(dirname "$0")")
+"$_dir"/get-pr-token "$1" >> "$curl_script_file"
+echo "\" https://api.github.com/repos/$1/pulls/$2" >> "$curl_script_file"
+
+if ( # sub-shell to preserve original shell -/+x -/+e state
+  set +x # hide curl command below as it contains a secret! don't remove me!
+# uncomment the below to debug, warning: will reveal secrets in logs
+#  cat "$curl_script_file" >&2
+  bash "$curl_script_file" >"$json_out"
+); then
+  if command -v jq > /dev/null; then
+    URL=$(jq ".statuses_url" < "$json_out" | tr -d '"')
+    status=$?
+  else
+    URL=$(grep "statuses_url" "$json_out" | head -n1 | sed -r 's/\s+"statuses_url": "([^"]+)",/\1/')
+    status=$?
+  fi
+else
+  echo "Request failed. Response was $(cat "$json_out")" >&2
+  status=1
+fi
+
+# uncomment the below deletion of files for debugging
+rm "$curl_script_file"
+rm "$json_out"
+
+echo "$1 $2 $URL"
+exit $status

build-scripts/bin/get-pr-token

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# get a pr token from the path in an environment variable depending on github organization.
+# used by get-github-pull-request-info and set-github-status which in turn are used by bootstrap-tarballs and testing-pr jenkins job
+# Args:
+#   $1 - repository identifier ("organization/project")
+# Env:
+#   two github fine-grained personal access tokens are needed with read/write access to commit statuses
+#   $CFENGINE_PR_TOKEN_PATH - file path containing token associated with CFEngine github organization
+#   $NTHQ_PR_TOKEN_PATH     - file path containing token associated with NorthernTechHQ github organization
+( # hide commands as they may contain secrets or paths to secrets
+  set +x
+  if [ -z "$1" ]; then echo "Need repository identifier as first argument"; exit 1; fi
+  if [ -z "$CFENGINE_PR_TOKEN_PATH" ]; then echo "Env var CFENGINE_PR_TOKEN_PATH is required"; exit 1; fi
+  if [ ! -f "$CFENGINE_PR_TOKEN_PATH" ]; then echo "CFENGINE_PR_TOKEN_PATH file must exist"; exit 1; fi
+  if [ -z "$NTHQ_PR_TOKEN_PATH" ]; then echo "Env var NTHQ_PR_TOKEN_PATH is required"; exit 1; fi
+  if [ ! -f "$NTHQ_PR_TOKEN_PATH" ]; then echo "NTHQ_PR_TOKEN_PATH file must exist"; exit 1; fi
+)
+# debug the following sha256sum commands to help determine if the tokens are correct in jenkins builds
+#echo "sha256sum of CFENGINE_PR_TOKEN_PATH..." >&2
+#sha256sum "$CFENGINE_PR_TOKEN_PATH" >&2
+#echo "sha256sum of NTHQ_PR_TOKEN_PATH..." >&2
+#sha256sum "$NTHQ_PR_TOKEN_PATH" >&2
+if [ "${1%/*}" = "cfengine" ]; then
+  tr -d '\n' < "$CFENGINE_PR_TOKEN_PATH"
+elif [ "${1%/*}" = "NorthernTechHQ" ]; then
+  tr -d '\n' < "$NTHQ_PR_TOKEN_PATH"
+else
+  echo "$0 doesn't know about tokens for organization ${1%/*}"
+  exit 1
+fi

build-scripts/bin/set-github-status

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# GitHub reporting script
+# Args:
+#   Either:
+#     $1 - where to get repos and PRs info from
+#     $2 - what state to report to GitHub
+#     $3 - job spec (e.g. "ci/testing-pr/PACKAGES_HUB_x86_64_linux_redhat_7")
+#     $4 - description of the status
+#     $5 - URL to link from the status (e.g. $JOB_URL of the jenkins job)
+#   Or:
+#     $1 - where to get repos and PRs info from
+#     $2 - path to a JSON file ready to POST to GH
+# Env:
+#   see get-pr-token adjacent to this file
+
+PRs_file="$1"
+if [ -z "$PRs_file" ]; then
+    exit 1
+fi
+
+if [ $# = "2" ]; then
+    # just two args, check if it is a file we can read
+    if [ -r "$2" ]; then
+        JSON_file="$2"
+    else
+        "Path to a readable JSON file or status details required!"
+        exit 1
+    fi
+else
+    state="$2"
+    job_spec="$3"
+    description="$4"
+    job_url="$5"
+    if [ -z "$job_url" ]; then
+        job_url="https://ci.cfengine.com/"
+    fi
+
+    if [ -z "$state" ] || [ -z "$job_spec" ]; then
+        echo "Missing arguments"
+        exit 1
+    fi
+fi
+
+function set_status() {
+set -ex
+    # Actually set status at GitHub
+    # Args:
+    #   $1 - repo identifier (organization/project)
+    #   $2 - statuses API URL of the PR
+    # Env:
+
+    if [ -z "$1" ]; then
+      echo "Missing repo identifier (organization/project) as first argument"
+      exit 1
+    fi
+    if [ -z "$2" ]; then
+      echo "Missing pull request API URL as second argument"
+      exit 1
+    fi
+
+    # curl 7.88 ish supports --header @file but apparently 7.52 (on bootstrap vm (deb-9)) does not, so compose a script
+    curl_script_file="$(mktemp)"
+    chmod 600 "$curl_script_file"
+    echo -n "curl --fail --insecure -X POST --header \"Authorization: Bearer " > "$curl_script_file"
+    _dir=$(readlink -e "$(dirname "$0")")
+    "$_dir"/get-pr-token "$1" >> "$curl_script_file"
+    echo -n "\" $2 --data " >> "$curl_script_file"
+
+    if [ -n "$JSON_file" ]; then
+        (
+          set +x # hide secrets
+          echo "@$JSON_file" >> "$curl_script_file"
+        )
+    else
+        (
+          set +x # hide secrets
+          echo -n "@- <<EOF" >> "$curl_script_file"
+          echo -n "
+{
+  \"state\" : \"$state\",
+  \"target_url\" : \"$job_url\",
+  \"description\" : \"$description\",
+  \"context\" : \"$job_spec\"
+}
+EOF" >> "$curl_script_file"
+        )
+    fi
+
+# uncomment the below cat to see the curl_script_file location
+#    cat "$curl_script_file" >&2
+    bash "$curl_script_file"
+
+# uncomment the below file deletion to debug curl_script_file
+    rm "$curl_script_file"
+    return $?
+}
+
+while read -r line; do
+    # the PRs file has lines in the following format:
+    #   REPO_IDENTIFIER PR_ID PR_STATUS_API_URL
+    REPO_IDENTIFIER=$(echo "$line" | awk '{ print $1 };')
+    STATUS_URL=$(echo "$line" | awk '{ print $3 };')
+    set_status "$REPO_IDENTIFIER" "$STATUS_URL"
+done < "$PRs_file"

build-scripts/bootstrap-tarballs

@@ -1,40 +1,37 @@
 #!/bin/bash -x
 
+_dir=$(readlink -e "$(dirname "$0")")
+# refactored a few functions into single file scripts for easier development/debugging, see ENT-12741 and ENT-12595
+# Easier to add a path to a script than source a file of functions.
+export PATH="$_dir"/bin:$PATH
 . `dirname "$0"`/functions
 . detect-environment
 . compile-options
 . version
 
-get_GH_PR_info() {
-    # Args:
-    #   $1 - repo identifier ("project/repo")
-    #   $2 - PR identifier (the PR number)
-    # Env:
-    #   $GITHUB_STATUS_TOKEN - token for GitHub authentication
-    # Prints:
-    #   $REPO_ID $PR_ID $PR_STATUSES_URL
-    #   Where:
-    #     $PR_STATUSES_URL - GH API URL to set PR's statuses
-    # Returns:
-    #   0 - success, 1 - error
-    if [ -z "$1" ] || [ -z "$2" ] || [ -z "$GITHUB_STATUS_TOKEN" ]; then return 1; fi
-
-    if which jq > /dev/null; then
-        URL=$(curl -k -H "Authorization: token $GITHUB_STATUS_TOKEN" https://api.github.com/repos/$1/pulls/$2 |
-                  jq ".statuses_url" | tr -d '"')
-        status=$?
-    else
-        URL=$(curl -k -H "Authorization: token $GITHUB_STATUS_TOKEN" https://api.github.com/repos/$1/pulls/$2 |
-                  grep "statuses_url" | head -n1 | sed -r 's/\s+"statuses_url": "([^"]+)",/\1/')
-        status=$?
-    fi
-
-    echo "$1 $2 $URL"
-    return $status
-}
+mkdir -p $BASEDIR/output/tarballs
 
+# the first part of the script is not really critical
+set +e
 
-mkdir -p $BASEDIR/output/tarballs
+# Get information about PRs among the used revisions.
+# These PRs will have to be notified of build progress.
+for repo_spec in cfengine/buildscripts cfengine/core cfengine/masterfiles cfengine/enterprise cfengine/nova cfengine/mission-portal NorthernTechHQ/libntech; do
+  # remove organization/ from start of repo_spec
+  repo="${repo_spec#*/}"
+  rev_param_name="$(echo $repo | tr '[:lower:]-' '[:upper:]_')_REV"
+  revision="$(echo ${!rev_param_name})" || continue   # dereference
+
+  # remove "origin/" (if any)
+  revision="${revision##origin/}"
+  if expr "$revision" : "pull/" >/dev/null; then
+    pr_nr="$(echo $revision | cut -d/ -f2)"
+    get-github-pull-request-info "$repo_spec" "$pr_nr" >> $BASEDIR/output/PRs
+  fi
+done
+
+# now script failures should fail the script
+set -e
 
 cd $BASEDIR/core
 rm cfengine-3.*.tar.gz || true
@@ -109,20 +106,3 @@ if test -f "$BASEDIR/mission-portal/ldap/composer.json"; then
 fi
 )
 
-# the rest of the script is not really critical
-set +e
-
-# Get information about PRs among the used revisions.
-# These PRs will have to be notified of build progress.
-for repo in buildscripts core masterfiles enterprise nova mission-portal; do
-  rev_param_name="$(echo $repo | tr '[:lower:]-' '[:upper:]_')_REV"
-  revision="$(echo ${!rev_param_name})" || continue   # dereference
-
-  # remove "origin/" (if any)
-  revision="${revision##origin/}"
-  if expr "$revision" : "pull/" >/dev/null; then
-	repo_spec="cfengine/$repo"
-    pr_nr="$(echo $revision | cut -d/ -f2)"
-	get_GH_PR_info "$repo_spec" "$pr_nr" >> $BASEDIR/output/PRs
-  fi
-done