From a61af11f61bf85b34248eae2143b4c50df0d89af Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Thu, 6 Sep 2018 10:53:51 -0400 Subject: [PATCH 01/10] Adding release script to make my life easier after briefly making it harder --- .../release-utils/prepare-release-candidate | 306 ++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100755 dev-utilities/release-utils/prepare-release-candidate diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate new file mode 100755 index 0000000000..2399495b80 --- /dev/null +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -0,0 +1,306 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Creates and sets up the actual release artifacts into dev +# See https://cwiki.apache.org/confluence/display/METRON/Release+Process + +# define constants +CORE_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron.git" +DEV_REPO="https://dist.apache.org/repos/dist/dev/metron" +RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron" +PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git" +RC_PREFIX=rc + +function help { + echo " " + echo "usage: ${0}" + echo " -c/--current-version= The version of the current core release. [Required]" + echo " -v/--version= The version of the next core release. [Required]" + echo " -bc/--bro-current-version= The version of the next bro-kafka plugin release." + echo " -bv/--bro-version= The version of the next bro-kafka-plugin release." + echo " -n/--num-rc= The release candidate number, e.g. 1. [Required]" + echo " -p/--practice-run Do a practice run that doesn't push anything" + echo " -h/--help Usage information." + echo " " + echo "example: " + echo " prepare-release-candidate -v=0.6.0 -c=0.5.0 -r=1 -p -bc=0.1.0 -bv=0.2.0" + echo " " +} + +# print help, if the user just runs this without any args +if [ "$#" -eq 0 ]; then + help + exit 1 +fi + +# handle command line options +for i in "$@"; do + case $i in + # + # CORE VERSION: The core release version to validate; the 'next' release. + # + # + -v=*|--version=*) + CORE_VERSION="${i#*=}" + shift # past argument=value + ;; + + # + # CORE CURRENT VERSION: The core version of the current release. + # + # + -c=*|--current-version=*) + CORE_CURRENT_VERSION="${i#*=}" + shift # past argument=value + ;; + + # + # PLUGIN VERSION: The bro-kafka plugin version to validate; the 'next' release. + # + # + -bv=*|--version=*) + PLUGIN_VERSION="${i#*=}" + shift # past argument=value + ;; + + # + # PLUGIN CURRENT VERSION: The bro-kafka plugin version of the current release. + # + # + -bc=*|--current-version=*) + PLUGIN_CURRENT_VERSION="${i#*=}" + shift # past argument=value + ;; + + # + # NUM RC: The Release candidate number to create + # + # + -n=*|--num-rc=*) + RC_NUM="${i#*=}" + if [[ ! ${RC_NUM} =~ ^[0-9]+$ ]]; then + printf "Release candidate must be an integer, received %s\n" "$RC_NUM" + exit 1 + fi + RC="${RC_PREFIX}${RC_NUM}" + shift # past argument=value + ;; + + # + # PRACTICE RUN: If this is a practice run, don't push to remote Git + # + # -p + # --practice-run + # + -p|--practice-run) + PRACTICE_RUN=true + shift # past argument=value + ;; + + + # + # -h/--help + # + -h|--help) + help + exit 0 + shift # past argument with no value + ;; + + # + # Unknown option + # + *) + UNKNOWN_OPTION="${i#*=}" + echo "Error: unknown option: $UNKNOWN_OPTION" + help + exit 1 + ;; + esac +done + +# ensure all required values are set +if [ -z "$CORE_VERSION" ]; then + help + echo "Missing -v/--version is required" + exit 1 +fi + +if [ -z "$CORE_CURRENT_VERSION" ]; then + help + echo "Missing -c/--current-version is required" + exit 1 +fi + +if [ -z "$RC" ]; then + help + echo "Missing -r/--release-candidate is required" + exit 1 +fi + +# define default values +PRACTICE=false +TMPDIR="$HOME/tmp" +WORKDIR="$TMPDIR/metron-$CORE_VERSION" + +# warn the user if the working directory exists +if [ -d "$WORKDIR" ]; then + read -p " directory exists [$WORKDIR]. overwrite existing repo? [yN] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi +fi + +# Clear out the existing work directory +rm -rf "$WORKDIR" +mkdir "$WORKDIR" + +## Prepare everything for building the release artifacts + +# fetch the SVN repos +fetch_svn_repo () { + local SVN_REPO=$1 + local SVN_DIR=$2 + mkdir "$SVN_DIR" + cd "$SVN_DIR" + printf "Checking out repo: %s\n" "$SVN_REPO" "$(basename $SVN_DIR)" + svn co -q $SVN_REPO + cd "$(dirname $WORKDIR)" +} + +fetch_svn_repo "$DEV_REPO" "$WORKDIR/dev" +fetch_svn_repo "$RELEASE_REPO" "$WORKDIR/release" + +fetch_git_repo () { + GIT_REPO=$1 + GIT_DIR=$2 + printf "Checking out git repo: %s\n" "$GIT_REPO" + git clone $GIT_REPO "$GIT_DIR" + cd "$GIT_DIR" + git fetch --tags +} + +# fetch the repos and all tags +fetch_git_repo "$CORE_GIT_REPO" "$WORKDIR/core_git" +fetch_git_repo "$PLUGIN_GIT_REPO" "$WORKDIR/plugin_git" + +create_release_branch () { + local BRANCH_PREFIX=$1 + local BRANCH_VERSION=$2 + local GIT_DIR=$3 + + printf "Creating branch: %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" + cd "$GIT_DIR" + git checkout master + git checkout -b "${BRANCH_PREFIX}_${BRANCH_VERSION}" + + if [ "$PRACTICE_RUN" = true ]; then + printf "This is a practice run. Not running \n" "$BRANCH_PREFIX" "$BRANCH_VERSION" + else + printf "Pushing branch %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" + #git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" + fi +} + +# Create the core release branch +create_release_branch "Metron" "$CORE_VERSION" "$WORKDIR/core_git" +create_release_branch "Metron_bro-plugin-kafka" "$PLUGIN_VERSION" "$WORKDIR/plugin_git" + +# Create directory for release artifacts +ART_DIR="$WORKDIR/dev/metron/${CORE_VERSION}-RC${RC_NUM}" +mkdir "$ART_DIR" + +## Do the work of actually creating the release artifacts +create_release_artifacts () { + local VERSION=$1 + local CURRENT_VERSION=$2 + local GIT_DIR=$3 + local BRANCH_PREFIX=$4 + local TAG_PREFIX=$5 + local TAG_NAME=$6 + local TAG_POSTFIX=$7 + local CHANGES_POSTFIX=$8 + local ARTIFACT_PREFIX=$9 + local MAJOR_MINOR_ONLY=${10} + local ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}" + printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC" + cd "$GIT_DIR" + git checkout "${BRANCH_PREFIX}${VERSION}" && git pull + git tag "${ARTIFACT}" + + # Create the rc tarball from the tag + printf "Creating the RC tarball for tag %s\n" "$ARTIFACT" + git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz" + + # Create signing hash files + printf "Creating the SHA hash files\n" + gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512 + gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256 + + # Sign the release tarball + printf "Signing the release tarball\n" + gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz + + # Create the CHANGES file + # Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON" + # i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches + TAG_VERSION=$CURRENT_VERSION + if [ "$MAJOR_MINOR_ONLY" = true ]; then + TAG_VERSION=$(echo $CURRENT_VERSION | cut -d. -f-2) + printf "Stripped the point version from the current version. Tag is: %s\n" "$TAG_VERSION" + fi + + git log ${BRANCH_PREFIX}${VERSION} ^tags/${TAG_PREFIX}${TAG_VERSION}${TAG_POSTFIX} --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "CHANGES${CHANGES_POSTFIX}" + + # Setup the release artifacts + printf "Copying release artifacts\n" + cp "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR" + cp "${ARTIFACT}.tar.gz.sha512" "$ART_DIR" + cp "${ARTIFACT}.tar.gz.sha256" "$ART_DIR" + cp "${ARTIFACT}.tar.gz.asc" "$ART_DIR" + cp "CHANGES${CHANGES_POSTFIX}" "$ART_DIR" +} + +# Note that the seperator differs for core and the plugin ('-' vs '_') +read -s -p "Signing Key in 8-byte format (e.g. BADDCAFEDEADBEEF):" SIGNING_KEY +CORE_TAG_NAME=${CORE_CURRENT_VERSION}-release +CORE_PREFIX="apache-metron-" +create_release_artifacts "$CORE_VERSION" "$CORE_CURRENT_VERSION" "$WORKDIR/core_git" "Metron_" "$CORE_PREFIX" "$CORE_TAG_NAME" "-release" "" "$CORE_PREFIX" "false" + +PLUGIN_TAG_NAME=$(echo $PLUGIN_CURRENT_VERSION | cut -d. -f-2) +create_release_artifacts "$PLUGIN_VERSION" "$PLUGIN_CURRENT_VERSION" "$WORKDIR/plugin_git" "Metron_bro-plugin-kafka_" "" "$PLUGIN_TAG_NAME" "" ".bro-plugin" "apache-metron-bro-plugin-kafka_" "true" + +printf "Extracting LICENSE, NOTICE, and KEYS from tarball\n" # Only pull from core +cd ${ART_DIR} +CORE_ARTIFACT="apache-metron-${CORE_VERSION}-${RC}" +tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/LICENSE" +tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/KEYS" +tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/NOTICE" + +# Add the directory and commit to subversion +if [ "$PRACTICE_RUN" = true ]; then + printf "This is a practice run. Not running the following commands:\n" + printf "svn add %s-RC%s\n" "$CORE_VERSION" "$RC_NUM" + printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$CORE_VERSION" "$RC_NUM" +else + printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN" + #svn add "${CORE_VERSION}-RC{$RC_NUM}" + #svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" +fi + From 9543cb8ebd05a8c07e193c09c12593978c20d8f8 Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Thu, 6 Sep 2018 11:02:11 -0400 Subject: [PATCH 02/10] Removing commented out real code --- dev-utilities/release-utils/prepare-release-candidate | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index 2399495b80..30ab3448f9 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -214,7 +214,7 @@ create_release_branch () { printf "This is a practice run. Not running \n" "$BRANCH_PREFIX" "$BRANCH_VERSION" else printf "Pushing branch %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - #git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" + git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" fi } @@ -300,7 +300,7 @@ if [ "$PRACTICE_RUN" = true ]; then printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$CORE_VERSION" "$RC_NUM" else printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN" - #svn add "${CORE_VERSION}-RC{$RC_NUM}" - #svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" + svn add "${CORE_VERSION}-RC{$RC_NUM}" + svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" fi From d0fcb39cfda8a6afafc1c7c5258d922131a96312 Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Thu, 6 Sep 2018 11:28:10 -0400 Subject: [PATCH 03/10] Adding newline to printf --- dev-utilities/release-utils/prepare-release-candidate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index 30ab3448f9..ffdfac92e2 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -299,7 +299,7 @@ if [ "$PRACTICE_RUN" = true ]; then printf "svn add %s-RC%s\n" "$CORE_VERSION" "$RC_NUM" printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$CORE_VERSION" "$RC_NUM" else - printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN" + printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN\n" svn add "${CORE_VERSION}-RC{$RC_NUM}" svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" fi From 807906227e64a703c943495f63a73e988f6f5de6 Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Thu, 6 Sep 2018 18:35:58 -0400 Subject: [PATCH 04/10] moving to correct dir before SVN --- dev-utilities/release-utils/prepare-release-candidate | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index ffdfac92e2..da5a9fb7f4 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -300,6 +300,7 @@ if [ "$PRACTICE_RUN" = true ]; then printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$CORE_VERSION" "$RC_NUM" else printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN\n" + cd "$WORKDIR/dev/metron/" svn add "${CORE_VERSION}-RC{$RC_NUM}" svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" fi From 4111f4d629a26623d7f448aac41bdc0d2b3b0210 Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Fri, 7 Sep 2018 09:16:28 -0400 Subject: [PATCH 05/10] Updating RC Check to handle appopriately named plugin artifacts --- dev-utilities/release-utils/metron-rc-check | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-utilities/release-utils/metron-rc-check b/dev-utilities/release-utils/metron-rc-check index e3cc39d1c7..143ba85a28 100755 --- a/dev-utilities/release-utils/metron-rc-check +++ b/dev-utilities/release-utils/metron-rc-check @@ -181,7 +181,7 @@ fi if [ -n "$BRO" ]; then - METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" + METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc" echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY" @@ -214,12 +214,12 @@ fi if [ -n "$BRO" ]; then echo "Verifying Bro Kafka Plugin Assembly" - if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then + if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" ; then echo "[ERROR] failed to verify Bro Kafka Plugin Assembly" exit 1 fi - if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then + if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION-$RC.tar.gz" ; then echo "[ERROR] failed to unpack Bro Kafka Plugin Assembly" exit 1 fi From 4e585c831689067877f62c177eff6ee1fd9b7834 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Tue, 11 Sep 2018 16:19:43 -0400 Subject: [PATCH 06/10] Testing --- .../release-utils/prepare-release-candidate | 408 ++++++++---------- 1 file changed, 190 insertions(+), 218 deletions(-) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index da5a9fb7f4..57aed44d70 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -20,147 +20,119 @@ # See https://cwiki.apache.org/confluence/display/METRON/Release+Process # define constants -CORE_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron.git" +# Git repos +METRON_REPO_NAME="metron" +BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka" +METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git" +BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git" + DEV_REPO="https://dist.apache.org/repos/dist/dev/metron" RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron" -PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git" -RC_PREFIX=rc -function help { - echo " " - echo "usage: ${0}" - echo " -c/--current-version= The version of the current core release. [Required]" - echo " -v/--version= The version of the next core release. [Required]" - echo " -bc/--bro-current-version= The version of the next bro-kafka plugin release." - echo " -bv/--bro-version= The version of the next bro-kafka-plugin release." - echo " -n/--num-rc= The release candidate number, e.g. 1. [Required]" - echo " -p/--practice-run Do a practice run that doesn't push anything" - echo " -h/--help Usage information." - echo " " - echo "example: " - echo " prepare-release-candidate -v=0.6.0 -c=0.5.0 -r=1 -p -bc=0.1.0 -bv=0.2.0" - echo " " -} +RC_PREFIX=rc -# print help, if the user just runs this without any args -if [ "$#" -eq 0 ]; then - help - exit 1 +CONFIG_FILE=~/.metron-prepare-release-candidate +# does a config file already exist? +if [ -f $CONFIG_FILE ]; then + . $CONFIG_FILE + echo " ...using settings from $CONFIG_FILE" fi -# handle command line options -for i in "$@"; do - case $i in - # - # CORE VERSION: The core release version to validate; the 'next' release. - # - # - -v=*|--version=*) - CORE_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # CORE CURRENT VERSION: The core version of the current release. - # - # - -c=*|--current-version=*) - CORE_CURRENT_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # PLUGIN VERSION: The bro-kafka plugin version to validate; the 'next' release. - # - # - -bv=*|--version=*) - PLUGIN_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # PLUGIN CURRENT VERSION: The bro-kafka plugin version of the current release. - # - # - -bc=*|--current-version=*) - PLUGIN_CURRENT_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # NUM RC: The Release candidate number to create - # - # - -n=*|--num-rc=*) - RC_NUM="${i#*=}" - if [[ ! ${RC_NUM} =~ ^[0-9]+$ ]]; then - printf "Release candidate must be an integer, received %s\n" "$RC_NUM" - exit 1 - fi - RC="${RC_PREFIX}${RC_NUM}" - shift # past argument=value +#CORE_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron.git" +#PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git" + +# which repo? metron or metron-bro-plugin-kafka +echo " [1] ${METRON_REPO_NAME}" +echo " [2] ${BRO_PLUGIN_REPO_NAME}" +read -p " which repo? [1]: " INPUT +case "${INPUT}" in + [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2) + INPUT="${BRO_PLUGIN_UPSTREAM}" ;; + [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'') + INPUT="${METRON_UPSTREAM}" ;; + *) + echo "Invalid repo, provided \"${INPUT}\". Please choose between ${METRON_REPO_NAME} or ${BRO_PLUGIN_REPO_NAME}" + exit 1 ;; +esac +[ -n "$INPUT" ] && UPSTREAM=$INPUT - # - # PRACTICE RUN: If this is a practice run, don't push to remote Git - # - # -p - # --practice-run - # - -p|--practice-run) - PRACTICE_RUN=true - shift # past argument=value - ;; +CHOSEN_REPO=$(basename ${UPSTREAM%%.git}) +# Need the capitalized version of the repos some naming +CAPITAL_REPO="$(tr '[:lower:]' '[:upper:]' <<< ${CHOSEN_REPO:0:1})${CHOSEN_REPO:1}" +# apache id of committer (you) +if [ -z "$APACHE_NAME" ]; then + read -p " your apache userid [$APACHE_NAME]: " INPUT + [ -n "$INPUT" ] && APACHE_NAME=$INPUT - # - # -h/--help - # - -h|--help) - help - exit 0 - shift # past argument with no value - ;; + # write setting to config file + echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE +fi - # - # Unknown option - # - *) - UNKNOWN_OPTION="${i#*=}" - echo "Error: unknown option: $UNKNOWN_OPTION" - help - exit 1 - ;; - esac -done +# apache email addr of committer (you) +if [ -z "$APACHE_EMAIL" ]; then + APACHE_EMAIL=${APACHE_NAME}@apache.org + read -p " your apache email [$APACHE_EMAIL]: " INPUT + [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT -# ensure all required values are set -if [ -z "$CORE_VERSION" ]; then - help - echo "Missing -v/--version is required" - exit 1 + # write setting to config file, so it is not needed next time + echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE fi -if [ -z "$CORE_CURRENT_VERSION" ]; then - help - echo "Missing -c/--current-version is required" +# current version +read -p " current version: " CURRENT_VERSION +if [ -z "${CURRENT_VERSION}" ]; then + echo "Error: missing current version" exit 1 fi -if [ -z "$RC" ]; then - help - echo "Missing -r/--release-candidate is required" +# version that we're building an RC for +read -p " version being built: " VERSION +# TODO validate the version matches our style (x.y.z for metron or x.y for the plugin) +if [ -z "$VERSION" ]; then + echo "Error: missing version being built" exit 1 fi +# RC number we're building +getrcnum() { +read -p " release candidate number: " RC_NUM + if [[ "${RC_NUM}" =~ ^[0-9]+$ ]]; then + RC="${RC_PREFIX}${RC_NUM}" + return 0 + else + printf " Please enter an integer\n" + return 1 + fi +} +until getrcnum; do : ; done + +# Determine if this is a practice run or not. +getpractice() { + read -p " do a live run (push to remote repositories?) [y/n]" INPUT + case "${INPUT}" in + y) + PRACTICE="false" + return 0 ;; + n) + PRACTICE="true" + return 0 ;; + *) + printf "Please enter 'y' or 'n'\n" + return 1 + ;; + esac +} +until getpractice; do : ; done + # define default values -PRACTICE=false TMPDIR="$HOME/tmp" -WORKDIR="$TMPDIR/metron-$CORE_VERSION" +WORKDIR="$TMPDIR/${CHOSEN_REPO}-$VERSION" # warn the user if the working directory exists if [ -d "$WORKDIR" ]; then - read -p " directory exists [$WORKDIR]. overwrite existing repo? [yN] " -n 1 -r + read -p " directory exists [$WORKDIR]. overwrite existing directory? [y/n] " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1 @@ -173,7 +145,7 @@ mkdir "$WORKDIR" ## Prepare everything for building the release artifacts -# fetch the SVN repos +# Fetch the SVN repos. Always needed regardless of what's being released. fetch_svn_repo () { local SVN_REPO=$1 local SVN_DIR=$2 @@ -187,121 +159,121 @@ fetch_svn_repo () { fetch_svn_repo "$DEV_REPO" "$WORKDIR/dev" fetch_svn_repo "$RELEASE_REPO" "$WORKDIR/release" -fetch_git_repo () { - GIT_REPO=$1 - GIT_DIR=$2 - printf "Checking out git repo: %s\n" "$GIT_REPO" - git clone $GIT_REPO "$GIT_DIR" - cd "$GIT_DIR" - git fetch --tags -} - -# fetch the repos and all tags -fetch_git_repo "$CORE_GIT_REPO" "$WORKDIR/core_git" -fetch_git_repo "$PLUGIN_GIT_REPO" "$WORKDIR/plugin_git" - -create_release_branch () { - local BRANCH_PREFIX=$1 - local BRANCH_VERSION=$2 - local GIT_DIR=$3 - - printf "Creating branch: %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - cd "$GIT_DIR" - git checkout master - git checkout -b "${BRANCH_PREFIX}_${BRANCH_VERSION}" - - if [ "$PRACTICE_RUN" = true ]; then - printf "This is a practice run. Not running \n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - else - printf "Pushing branch %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" - fi -} - -# Create the core release branch -create_release_branch "Metron" "$CORE_VERSION" "$WORKDIR/core_git" -create_release_branch "Metron_bro-plugin-kafka" "$PLUGIN_VERSION" "$WORKDIR/plugin_git" +# Fetch the appropriate Git repo. Only need what we're releasing +GIT_DIR="$WORKDIR/${CHOSEN_REPO}" +printf "Checking out git repo: %s\n" "$UPSTREAM" +git clone $UPSTREAM "${GIT_DIR}" +cd "${GIT_DIR}" +git fetch --tags + +# Create the release branch in the Git repo +printf "Creating branch: %s_%s\n" "${CAPITAL_REPO}" "$VERSION" +cd "$GIT_DIR" +git checkout master +git checkout -b "${CAPITAL_REPO}_${VERSION}" + +if [ "${PRACTICE_RUN}" = true ]; then + printf "This is a practice run. Not running \n" "${CAPITAL_REPO}" "$VERSION" +else + printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION" + #git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" +fi # Create directory for release artifacts -ART_DIR="$WORKDIR/dev/metron/${CORE_VERSION}-RC${RC_NUM}" -mkdir "$ART_DIR" - -## Do the work of actually creating the release artifacts -create_release_artifacts () { - local VERSION=$1 - local CURRENT_VERSION=$2 - local GIT_DIR=$3 - local BRANCH_PREFIX=$4 - local TAG_PREFIX=$5 - local TAG_NAME=$6 - local TAG_POSTFIX=$7 - local CHANGES_POSTFIX=$8 - local ARTIFACT_PREFIX=$9 - local MAJOR_MINOR_ONLY=${10} - local ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}" - printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC" - cd "$GIT_DIR" - git checkout "${BRANCH_PREFIX}${VERSION}" && git pull - git tag "${ARTIFACT}" - - # Create the rc tarball from the tag - printf "Creating the RC tarball for tag %s\n" "$ARTIFACT" - git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz" - - # Create signing hash files - printf "Creating the SHA hash files\n" - gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512 - gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256 - - # Sign the release tarball - printf "Signing the release tarball\n" - gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz - - # Create the CHANGES file - # Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON" - # i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches - TAG_VERSION=$CURRENT_VERSION - if [ "$MAJOR_MINOR_ONLY" = true ]; then - TAG_VERSION=$(echo $CURRENT_VERSION | cut -d. -f-2) - printf "Stripped the point version from the current version. Tag is: %s\n" "$TAG_VERSION" +if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then + ART_DIR="$WORKDIR/dev/metron/${VERSION}-RC${RC_NUM}" +else + # We're using a sub module, so put it in it's own directory. + ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-RC${RC_NUM}" +fi +mkdir -p "$ART_DIR" + +# Setup various parameters we need for the release artifacts +if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then + CORE_PREFIX="apache-metron-" + BRANCH_PREFIX="${CAPITAL_REPO_NAME}" + TAG_PREFIX="${CORE_PREFIX}" + TAG_NAME="${CORE_TAG_NAME}" + TAG_POSTFIX="-release" + ARTIFACT_PREFIX="${CORE_PREFIX}" + + TAG_VERSION="${CURRENT_VERSION}" + TAG="${TAG_PREFIX}${TAG_VERSION}${TAG_POSTFIX}" +elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then + BRANCH_PREFIX="${CAPITAL_REPO_NAME}" + TAG_PREFIX="" + TAG_NAME="${PLUGIN_TAG_NAME}" + TAG_POSTFIX="" + ARTIFACT_PREFIX="apache-metron-bro-plugin-kafka_" + + TAG_VERSION="${CURRENT_VERSION}" + TAG_VERSION=$(echo ${CURRENT_VERSION} | cut -d. -f-2) + printf "Stripped the point version from the current version. Tag is: %s\n" "$TAG_VERSION" + TAG="${TAG_PREFIX}${TAG_VERSION}${TAG_POSTFIX}" + # Handle special tag case from prior release + if [ "${CURRENT_VERSION}" -eq "0.1" ]; then + TAG="0.1" fi +else + # If we ever add new modules, add them as needed. + printf "Unrecognized module: %s\n" "${CHOSEN_REPO}" + exit 1 +fi +ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}" - git log ${BRANCH_PREFIX}${VERSION} ^tags/${TAG_PREFIX}${TAG_VERSION}${TAG_POSTFIX} --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "CHANGES${CHANGES_POSTFIX}" +read -s -p "Signing Key in 8-byte format (e.g. BADDCAFEDEADBEEF):" SIGNING_KEY # TODO properly read this in, as above - # Setup the release artifacts - printf "Copying release artifacts\n" - cp "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR" - cp "${ARTIFACT}.tar.gz.sha512" "$ART_DIR" - cp "${ARTIFACT}.tar.gz.sha256" "$ART_DIR" - cp "${ARTIFACT}.tar.gz.asc" "$ART_DIR" - cp "CHANGES${CHANGES_POSTFIX}" "$ART_DIR" -} +## Do the work of actually creating the release artifacts +printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC" +cd "$GIT_DIR" +git checkout "${BRANCH_PREFIX}${VERSION}" && git pull +git tag "${ARTIFACT}" + +# Create the rc tarball from the tag +printf "Creating the RC tarball for tag %s\n" "$ARTIFACT" +git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz" + +# Create signing hash files +printf "Creating the SHA hash files\n" +gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512 +gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256 + +# Sign the release tarball +printf "Signing the release tarball\n" +gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz +if [[ $? -ne 0 ]]; then + exit 1 +fi -# Note that the seperator differs for core and the plugin ('-' vs '_') -read -s -p "Signing Key in 8-byte format (e.g. BADDCAFEDEADBEEF):" SIGNING_KEY -CORE_TAG_NAME=${CORE_CURRENT_VERSION}-release -CORE_PREFIX="apache-metron-" -create_release_artifacts "$CORE_VERSION" "$CORE_CURRENT_VERSION" "$WORKDIR/core_git" "Metron_" "$CORE_PREFIX" "$CORE_TAG_NAME" "-release" "" "$CORE_PREFIX" "false" +# Create the CHANGES file +# Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON" +# i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches +git log "${BRANCH_PREFIX}_${VERSION}" "^tags/${TAG}" --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "CHANGES" -PLUGIN_TAG_NAME=$(echo $PLUGIN_CURRENT_VERSION | cut -d. -f-2) -create_release_artifacts "$PLUGIN_VERSION" "$PLUGIN_CURRENT_VERSION" "$WORKDIR/plugin_git" "Metron_bro-plugin-kafka_" "" "$PLUGIN_TAG_NAME" "" ".bro-plugin" "apache-metron-bro-plugin-kafka_" "true" +# Setup the release artifacts +printf "Copying release artifacts\n" +mv "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR" +mv "${ARTIFACT}.tar.gz.sha512" "$ART_DIR" +mv "${ARTIFACT}.tar.gz.sha256" "$ART_DIR" +mv "${ARTIFACT}.tar.gz.asc" "$ART_DIR" +mv "CHANGES" "$ART_DIR" printf "Extracting LICENSE, NOTICE, and KEYS from tarball\n" # Only pull from core cd ${ART_DIR} -CORE_ARTIFACT="apache-metron-${CORE_VERSION}-${RC}" -tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/LICENSE" -tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/KEYS" -tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/NOTICE" +CORE_ARTIFACT="apache-metron-${VERSION}-${RC}" +tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/LICENSE" +tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/KEYS" +tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/NOTICE" # Add the directory and commit to subversion if [ "$PRACTICE_RUN" = true ]; then printf "This is a practice run. Not running the following commands:\n" - printf "svn add %s-RC%s\n" "$CORE_VERSION" "$RC_NUM" - printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$CORE_VERSION" "$RC_NUM" + printf "svn add %s-RC%s\n" "$VERSION" "$RC_NUM" + printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$VERSION" "$RC_NUM" else - printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN\n" - cd "$WORKDIR/dev/metron/" - svn add "${CORE_VERSION}-RC{$RC_NUM}" - svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" + printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n" + #cd "$WORKDIR/dev/metron/" + #svn add "${CORE_VERSION}-RC{$RC_NUM}" + #svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" fi From 0ff395ea9dc0a7c77d352598422131aeefefb38b Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Wed, 12 Sep 2018 16:48:17 -0400 Subject: [PATCH 07/10] fix --- .../release-utils/prepare-release-candidate | 457 +++++++++--------- 1 file changed, 230 insertions(+), 227 deletions(-) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index da5a9fb7f4..cd912f1eee 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -20,147 +20,144 @@ # See https://cwiki.apache.org/confluence/display/METRON/Release+Process # define constants -CORE_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron.git" +# Git repos + +set -eo pipefail + +METRON_REPO_NAME="metron" +BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka" +METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git" +BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git" + DEV_REPO="https://dist.apache.org/repos/dist/dev/metron" RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron" -PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git" -RC_PREFIX=rc -function help { - echo " " - echo "usage: ${0}" - echo " -c/--current-version= The version of the current core release. [Required]" - echo " -v/--version= The version of the next core release. [Required]" - echo " -bc/--bro-current-version= The version of the next bro-kafka plugin release." - echo " -bv/--bro-version= The version of the next bro-kafka-plugin release." - echo " -n/--num-rc= The release candidate number, e.g. 1. [Required]" - echo " -p/--practice-run Do a practice run that doesn't push anything" - echo " -h/--help Usage information." - echo " " - echo "example: " - echo " prepare-release-candidate -v=0.6.0 -c=0.5.0 -r=1 -p -bc=0.1.0 -bv=0.2.0" - echo " " -} +RC_PREFIX=rc +TAG_POSTFIX="-release" -# print help, if the user just runs this without any args -if [ "$#" -eq 0 ]; then - help - exit 1 +CONFIG_FILE=~/.metron-prepare-release-candidate +# does a config file already exist? +if [ -f $CONFIG_FILE ]; then + . $CONFIG_FILE + echo " ...using settings from $CONFIG_FILE" fi -# handle command line options -for i in "$@"; do - case $i in - # - # CORE VERSION: The core release version to validate; the 'next' release. - # - # - -v=*|--version=*) - CORE_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # CORE CURRENT VERSION: The core version of the current release. - # - # - -c=*|--current-version=*) - CORE_CURRENT_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # PLUGIN VERSION: The bro-kafka plugin version to validate; the 'next' release. - # - # - -bv=*|--version=*) - PLUGIN_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # PLUGIN CURRENT VERSION: The bro-kafka plugin version of the current release. - # - # - -bc=*|--current-version=*) - PLUGIN_CURRENT_VERSION="${i#*=}" - shift # past argument=value - ;; - - # - # NUM RC: The Release candidate number to create - # - # - -n=*|--num-rc=*) - RC_NUM="${i#*=}" - if [[ ! ${RC_NUM} =~ ^[0-9]+$ ]]; then - printf "Release candidate must be an integer, received %s\n" "$RC_NUM" - exit 1 - fi - RC="${RC_PREFIX}${RC_NUM}" - shift # past argument=value - ;; - - # - # PRACTICE RUN: If this is a practice run, don't push to remote Git - # - # -p - # --practice-run - # - -p|--practice-run) - PRACTICE_RUN=true - shift # past argument=value - ;; - - - # - # -h/--help - # - -h|--help) - help - exit 0 - shift # past argument with no value - ;; - - # - # Unknown option - # +# which repo? metron or metron-bro-plugin-kafka +getrepo() { + echo " [1] ${METRON_REPO_NAME}" + echo " [2] ${BRO_PLUGIN_REPO_NAME}" + read -p " which repo? [1]: " INPUT + case "${INPUT}" in + [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2) + INPUT="${BRO_PLUGIN_UPSTREAM}" ;; + [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'') + INPUT="${METRON_UPSTREAM}" ;; *) - UNKNOWN_OPTION="${i#*=}" - echo "Error: unknown option: $UNKNOWN_OPTION" - help - exit 1 - ;; + echo "Invalid repo, provided \"${INPUT}\". Please choose between ${METRON_REPO_NAME} or ${BRO_PLUGIN_REPO_NAME}" + return 1 + ;; esac -done + [ -n "$INPUT" ] && UPSTREAM=$INPUT + return 0 +} +until getrepo; do :; done -# ensure all required values are set -if [ -z "$CORE_VERSION" ]; then - help - echo "Missing -v/--version is required" - exit 1 -fi +CHOSEN_REPO=$(basename ${UPSTREAM%%.git}) +# Need the capitalized version of the repos some naming +CAPITAL_REPO="$(tr '[:lower:]' '[:upper:]' <<< ${CHOSEN_REPO:0:1})${CHOSEN_REPO:1}" -if [ -z "$CORE_CURRENT_VERSION" ]; then - help - echo "Missing -c/--current-version is required" - exit 1 +# apache id of committer (you) +if [ -z "${APACHE_NAME}" ]; then + read -p " your apache userid [${APACHE_NAME}]: " INPUT + [ -n "$INPUT" ] && APACHE_NAME=$INPUT + + # write setting to config file + echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE fi -if [ -z "$RC" ]; then - help - echo "Missing -r/--release-candidate is required" - exit 1 +# apache email addr of committer (you) +if [ -z "${APACHE_EMAIL}" ]; then + APACHE_EMAIL=${APACHE_NAME}@apache.org + read -p " your apache email [${APACHE_EMAIL}]: " INPUT + [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT + + # write setting to config file, so it is not needed next time + echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE fi +getcurrentversion() { + # version that we're building an RC for + read -p " current version: " CURRENT_VERSION + if ! [[ "${CURRENT_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf " Please enter a valid x.y.z version\n" + return 1 + fi + return 0 +} +until getcurrentversion; do : ; done + +# version that we're building an RC for +getversion() { + read -p " version being built: " VERSION + if ! [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf " Please enter a valid x.y.z version\n" + return 1 + fi + return 0 + } +until getversion; do : ; done + +# RC number we're building +getrcnum() { +read -p " release candidate number: " RC_NUM + if [[ "${RC_NUM}" =~ ^[0-9]+$ ]]; then + RC="${RC_PREFIX}${RC_NUM}" + return 0 + else + printf " Please enter an integer\n" + return 1 + fi +} +until getrcnum; do : ; done + + +getkey() { + read -s -p " signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): " SIGNING_KEY + printf "\n" + if ! [[ "${SIGNING_KEY}" =~ ^[A-F0-9]{16}$ ]]; then + printf " Please enter a valid signing key\n" + return 1 + fi + return 0 +} + +until getkey; do : ; done + +# Determine if this is a practice run or not. +getpractice() { + read -p " do a live run (push to remote repositories?) [y/n] " INPUT + case "${INPUT}" in + y) + PRACTICE_RUN=false + return 0 ;; + n) + PRACTICE_RUN=true + return 0 ;; + *) + printf "Please enter 'y' or 'n'\n" + return 1 + ;; + esac +} +until getpractice; do : ; done + # define default values -PRACTICE=false TMPDIR="$HOME/tmp" -WORKDIR="$TMPDIR/metron-$CORE_VERSION" +WORKDIR="$TMPDIR/${CHOSEN_REPO}-${VERSION}" # warn the user if the working directory exists if [ -d "$WORKDIR" ]; then - read -p " directory exists [$WORKDIR]. overwrite existing repo? [yN] " -n 1 -r + read -p " directory exists [$WORKDIR]. overwrite existing directory? [y/n] " -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1 @@ -173,7 +170,7 @@ mkdir "$WORKDIR" ## Prepare everything for building the release artifacts -# fetch the SVN repos +# Fetch the SVN repos. Always needed regardless of what's being released. fetch_svn_repo () { local SVN_REPO=$1 local SVN_DIR=$2 @@ -187,121 +184,127 @@ fetch_svn_repo () { fetch_svn_repo "$DEV_REPO" "$WORKDIR/dev" fetch_svn_repo "$RELEASE_REPO" "$WORKDIR/release" -fetch_git_repo () { - GIT_REPO=$1 - GIT_DIR=$2 - printf "Checking out git repo: %s\n" "$GIT_REPO" - git clone $GIT_REPO "$GIT_DIR" - cd "$GIT_DIR" - git fetch --tags -} - -# fetch the repos and all tags -fetch_git_repo "$CORE_GIT_REPO" "$WORKDIR/core_git" -fetch_git_repo "$PLUGIN_GIT_REPO" "$WORKDIR/plugin_git" - -create_release_branch () { - local BRANCH_PREFIX=$1 - local BRANCH_VERSION=$2 - local GIT_DIR=$3 - - printf "Creating branch: %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - cd "$GIT_DIR" - git checkout master - git checkout -b "${BRANCH_PREFIX}_${BRANCH_VERSION}" - - if [ "$PRACTICE_RUN" = true ]; then - printf "This is a practice run. Not running \n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - else - printf "Pushing branch %s_%s\n" "$BRANCH_PREFIX" "$BRANCH_VERSION" - git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" - fi -} - -# Create the core release branch -create_release_branch "Metron" "$CORE_VERSION" "$WORKDIR/core_git" -create_release_branch "Metron_bro-plugin-kafka" "$PLUGIN_VERSION" "$WORKDIR/plugin_git" +# Fetch the appropriate Git repo. Only need what we're releasing +GIT_DIR="$WORKDIR/${CHOSEN_REPO}" +printf "Checking out git repo: %s\n" "$UPSTREAM" +git clone $UPSTREAM "${GIT_DIR}" +cd "${GIT_DIR}" +git fetch --tags + +# Create the release branch in the Git repo +printf "Creating branch: %s_%s\n" "${CAPITAL_REPO}" "$VERSION" +cd "$GIT_DIR" +git checkout master +git checkout -b "${CAPITAL_REPO}_${VERSION}" + +if [ "${PRACTICE_RUN}" = true ]; then + printf "This is a practice run. Not running \n" "${CAPITAL_REPO}" "$VERSION" +else + printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION" + #git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" +fi # Create directory for release artifacts -ART_DIR="$WORKDIR/dev/metron/${CORE_VERSION}-RC${RC_NUM}" -mkdir "$ART_DIR" - -## Do the work of actually creating the release artifacts -create_release_artifacts () { - local VERSION=$1 - local CURRENT_VERSION=$2 - local GIT_DIR=$3 - local BRANCH_PREFIX=$4 - local TAG_PREFIX=$5 - local TAG_NAME=$6 - local TAG_POSTFIX=$7 - local CHANGES_POSTFIX=$8 - local ARTIFACT_PREFIX=$9 - local MAJOR_MINOR_ONLY=${10} - local ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}" - printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC" - cd "$GIT_DIR" - git checkout "${BRANCH_PREFIX}${VERSION}" && git pull - git tag "${ARTIFACT}" - - # Create the rc tarball from the tag - printf "Creating the RC tarball for tag %s\n" "$ARTIFACT" - git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz" - - # Create signing hash files - printf "Creating the SHA hash files\n" - gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512 - gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256 - - # Sign the release tarball - printf "Signing the release tarball\n" - gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz - - # Create the CHANGES file - # Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON" - # i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches - TAG_VERSION=$CURRENT_VERSION - if [ "$MAJOR_MINOR_ONLY" = true ]; then - TAG_VERSION=$(echo $CURRENT_VERSION | cut -d. -f-2) - printf "Stripped the point version from the current version. Tag is: %s\n" "$TAG_VERSION" +if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then + ART_DIR="$WORKDIR/dev/metron/${VERSION}-RC${RC_NUM}" +else + # We're using a sub module, so put it in it's own directory. + ART_DIR="$WORKDIR/dev/metron/${CHOSEN_REPO}/${VERSION}-RC${RC_NUM}" +fi +mkdir -p "$ART_DIR" + +# Setup various parameters we need for the release artifacts +if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then + CORE_PREFIX="apache-metron-" + ARTIFACT_PREFIX="${CORE_PREFIX}" + TAG_VERSION="${CURRENT_VERSION}" + TAG="${CORE_PREFIX}${TAG_VERSION}${TAG_POSTFIX}" +elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then + PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_" + ARTIFACT_PREFIX="${PLUGIN_PREFIX}" + TAG_VERSION="${CURRENT_VERSION}" + TAG_VERSION=$(echo ${CURRENT_VERSION} | cut -d. -f-2) + TAG="${PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}" + # Handle special tag case from prior release + if [ "${TAG_VERSION}" = "0.1" ]; then + TAG="0.1" fi +else + # If we ever add new modules, add them as needed. + printf "Unrecognized module: %s\n" "${CHOSEN_REPO}" + exit 1 +fi +ARTIFACT="${ARTIFACT_PREFIX}${VERSION}-${RC}" - git log ${BRANCH_PREFIX}${VERSION} ^tags/${TAG_PREFIX}${TAG_VERSION}${TAG_POSTFIX} --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "CHANGES${CHANGES_POSTFIX}" - - # Setup the release artifacts - printf "Copying release artifacts\n" - cp "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR" - cp "${ARTIFACT}.tar.gz.sha512" "$ART_DIR" - cp "${ARTIFACT}.tar.gz.sha256" "$ART_DIR" - cp "${ARTIFACT}.tar.gz.asc" "$ART_DIR" - cp "CHANGES${CHANGES_POSTFIX}" "$ART_DIR" -} - -# Note that the seperator differs for core and the plugin ('-' vs '_') -read -s -p "Signing Key in 8-byte format (e.g. BADDCAFEDEADBEEF):" SIGNING_KEY -CORE_TAG_NAME=${CORE_CURRENT_VERSION}-release -CORE_PREFIX="apache-metron-" -create_release_artifacts "$CORE_VERSION" "$CORE_CURRENT_VERSION" "$WORKDIR/core_git" "Metron_" "$CORE_PREFIX" "$CORE_TAG_NAME" "-release" "" "$CORE_PREFIX" "false" +## Do the work of actually creating the release artifacts +printf "Creating tentative git tag <%s%s-%s>. Do not push this tag until RC is ready for community review.\n" "${TAG_PREFIX}" "$VERSION" "$RC" +cd "$GIT_DIR" +git checkout "${CAPITAL_REPO}_${VERSION}" +# The branch only exists if this is not a practice run +if [ "${PRACTICE_RUN}" = false ]; then + git pull +fi +git tag "${ARTIFACT}" + +# Create the rc tarball from the tag +printf "Creating the RC tarball for tag %s\n" "$ARTIFACT" +git archive "--prefix=${ARTIFACT}/" "${ARTIFACT}" | gzip > "${ARTIFACT}.tar.gz" + +# Create signing hash files +printf "Creating the SHA hash files\n" +gpg --print-md SHA512 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha512 +gpg --print-md SHA256 ${ARTIFACT}.tar.gz > ${ARTIFACT}.tar.gz.sha256 + +# Sign the release tarball +printf "Signing the release tarball\n" +gpg -u ${SIGNING_KEY} --armor --output ${ARTIFACT}.tar.gz.asc --detach-sig ${ARTIFACT}.tar.gz +if [[ $? -ne 0 ]]; then + # gpg will print out an error on its own + exit 1 +fi -PLUGIN_TAG_NAME=$(echo $PLUGIN_CURRENT_VERSION | cut -d. -f-2) -create_release_artifacts "$PLUGIN_VERSION" "$PLUGIN_CURRENT_VERSION" "$WORKDIR/plugin_git" "Metron_bro-plugin-kafka_" "" "$PLUGIN_TAG_NAME" "" ".bro-plugin" "apache-metron-bro-plugin-kafka_" "true" +# Setup the release artifacts +printf "Copying release artifacts\n" +mv "${GIT_DIR}/${ARTIFACT}.tar.gz" "$ART_DIR" +mv "${ARTIFACT}.tar.gz.sha512" "$ART_DIR" +mv "${ARTIFACT}.tar.gz.sha256" "$ART_DIR" +mv "${ARTIFACT}.tar.gz.asc" "$ART_DIR" + +# Create the CHANGES file +# Do this by getting all commits in current branch that aren't in current release. Filter out any merges by making sure lines start with blankspace followed by "METRON" +# i.e. make sure the lines starts with a ticket number to avoid merge commits into feature branches +printf "Creating CHANGES file\n" +git log "${CAPITAL_REPO}_${VERSION}" "^tags/${TAG}" --no-merges | grep -E "^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" > "${ART_DIR}/CHANGES" +if [[ $? -ne 0 ]]; then + "Error creating CHANGES file" + exit 1 +fi printf "Extracting LICENSE, NOTICE, and KEYS from tarball\n" # Only pull from core cd ${ART_DIR} -CORE_ARTIFACT="apache-metron-${CORE_VERSION}-${RC}" -tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/LICENSE" -tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/KEYS" -tar --strip-components=1 -zxvf "${CORE_ARTIFACT}.tar.gz" "${CORE_ARTIFACT}/NOTICE" + +if [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then + # Bro's convention for the LICENSE is different, so the file is called COPYING + tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/COPYING" +else + tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/LICENSE" +fi + +# TODO figure out what to do for bro repo here. The KEYS file only needs to live in the /dist root, rather than in each sub repo. +# Should we have a separate process for adding to the KEYS file without doing a Metron release? +#tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/KEYS" +tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/NOTICE" # Add the directory and commit to subversion -if [ "$PRACTICE_RUN" = true ]; then +if [ "${PRACTICE_RUN}" = true ]; then printf "This is a practice run. Not running the following commands:\n" - printf "svn add %s-RC%s\n" "$CORE_VERSION" "$RC_NUM" - printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$CORE_VERSION" "$RC_NUM" + printf "svn add %s-RC%s\n" "$VERSION" "${RC_NUM}" + printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$VERSION" "${RC_NUM}" else - printf "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM} to dev SVN\n" - cd "$WORKDIR/dev/metron/" - svn add "${CORE_VERSION}-RC{$RC_NUM}" - svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" + printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n" + # TODO fix dirs for core vs plugin + #cd "$WORKDIR/dev/metron/" + #svn add "${CORE_VERSION}-RC{$RC_NUM}" + #svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" fi From 4be3dc9aa7c308d360241265c0c34b4eebad2988 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Mon, 8 Oct 2018 12:31:17 -0400 Subject: [PATCH 08/10] updating to handle branches not master and svn push. Needs testing --- .../release-utils/prepare-release-candidate | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index ddd3790d69..9ea3b2df28 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -24,6 +24,7 @@ set -eo pipefail # define constants # Git repos +# To add a new submodule, add the repo name, upstream Git repo, and update the getrepo() selection function METRON_REPO_NAME="metron" BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka" METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git" @@ -121,7 +122,7 @@ read -p " release candidate number: " RC_NUM } until getrcnum; do : ; done - +# Signing key getkey() { read -s -p " signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): " SIGNING_KEY printf "\n" @@ -152,6 +153,24 @@ getpractice() { } until getpractice; do : ; done +getbaserev() { + read -p " base revision branch or hash for release candidate (master if empty) " GIT_REF + GIT_REF=${GIT_REF:-master} + + # check to see if we were given branch + git rev-parse --verify ${GIT_REF} + if [[ $? -ne 0 ]]; then + # check to see if we were given hash + git cat-file -t ${GIT_REF} + if [[ $? -ne 0 ]]; then + "Provided SHA is not a valid git commit hash" + return 1 + fi + fi + return 0 +} +until getbaserev; do : ; done + # define default values TMPDIR="$HOME/tmp" WORKDIR="$TMPDIR/${CHOSEN_REPO}-${VERSION}" @@ -194,8 +213,9 @@ git fetch --tags # Create the release branch in the Git repo printf "Creating branch: %s_%s\n" "${CAPITAL_REPO}" "$VERSION" +printf "Using git rev: %s\n" ${GIT_REF} cd "$GIT_DIR" -git checkout master +git checkout ${GIT_REF} git checkout -b "${CAPITAL_REPO}_${VERSION}" if [ "${PRACTICE_RUN}" = true ]; then @@ -303,9 +323,9 @@ if [ "${PRACTICE_RUN}" = true ]; then printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$VERSION" "${RC_NUM}" else printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n" - # TODO fix dirs for core vs plugin - #cd "$WORKDIR/dev/metron/" - #svn add "${CORE_VERSION}-RC{$RC_NUM}" - #svn commit -m "Adding artifacts for Metron ${CORE_VERSION}-RC${RC_NUM}" + cd "$WORKDIR/dev/metron/" + COMMIT_DIR=$(basename ${ART_DIR}) + # svn add ${COMMIT_DIR} + #svn commit -m "Adding artifacts for Metron ${COMMIT_DIR}" fi From c54a00a6a3eadd3d44dfadf39f5802c2d3b1b2e8 Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Mon, 8 Oct 2018 14:10:19 -0400 Subject: [PATCH 09/10] Things seems working, bit of reorg and fix on bro tag --- .../release-utils/prepare-release-candidate | 96 ++++++++++--------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index 9ea3b2df28..b73ac3c279 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -24,7 +24,8 @@ set -eo pipefail # define constants # Git repos -# To add a new submodule, add the repo name, upstream Git repo, and update the getrepo() selection function +# To add a new submodule, add the repo name, upstream Git repo, and update the getrepo() selection function. +# if versioning of the submodule isn't x.y.z format, retrieval of the git tag must also be adjusted. METRON_REPO_NAME="metron" BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka" METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git" @@ -88,7 +89,7 @@ if [ -z "${APACHE_EMAIL}" ]; then fi getcurrentversion() { - # version that we're building an RC for + # currently released version. Used for CHANGES file read -p " current version: " CURRENT_VERSION if ! [[ "${CURRENT_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then printf " Please enter a valid x.y.z version\n" @@ -122,6 +123,41 @@ read -p " release candidate number: " RC_NUM } until getrcnum; do : ; done +# define default values +TMPDIR="$HOME/tmp" +WORKDIR="$TMPDIR/${CHOSEN_REPO}-${VERSION}" + +# warn the user if the working directory exists +if [ -d "$WORKDIR" ]; then + read -p " directory exists [$WORKDIR]. overwrite existing directory? [y/n] " -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi +fi + +# Clear out the existing work directory +rm -rf "$WORKDIR" +mkdir "$WORKDIR" + +getbaserev() { + read -p " base revision branch or hash for release candidate (master if empty) " GIT_REF + GIT_REF=${GIT_REF:-master} + + # check to see if we were given branch + git rev-parse -q --verify ${GIT_REF} >> /dev/null + if [[ $? -ne 0 ]]; then + # check to see if we were given hash + git cat-file -e ${GIT_REF} + if [[ $? -ne 0 ]]; then + "Unable to find git revision" + return 1 + fi + fi + return 0 +} +until getbaserev; do : ; done + # Signing key getkey() { read -s -p " signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): " SIGNING_KEY @@ -153,41 +189,6 @@ getpractice() { } until getpractice; do : ; done -getbaserev() { - read -p " base revision branch or hash for release candidate (master if empty) " GIT_REF - GIT_REF=${GIT_REF:-master} - - # check to see if we were given branch - git rev-parse --verify ${GIT_REF} - if [[ $? -ne 0 ]]; then - # check to see if we were given hash - git cat-file -t ${GIT_REF} - if [[ $? -ne 0 ]]; then - "Provided SHA is not a valid git commit hash" - return 1 - fi - fi - return 0 -} -until getbaserev; do : ; done - -# define default values -TMPDIR="$HOME/tmp" -WORKDIR="$TMPDIR/${CHOSEN_REPO}-${VERSION}" - -# warn the user if the working directory exists -if [ -d "$WORKDIR" ]; then - read -p " directory exists [$WORKDIR]. overwrite existing directory? [y/n] " -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi -fi - -# Clear out the existing work directory -rm -rf "$WORKDIR" -mkdir "$WORKDIR" - ## Prepare everything for building the release artifacts # Fetch the SVN repos. Always needed regardless of what's being released. @@ -222,7 +223,7 @@ if [ "${PRACTICE_RUN}" = true ]; then printf "This is a practice run. Not running \n" "${CAPITAL_REPO}" "$VERSION" else printf "Pushing branch %s_%s\n" "${CAPITAL_REPO}" "$VERSION" - #git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" + git push --set-upstream origin "${BRANCH_PREFIX}_${BRANCH_VERSION}" fi # Create directory for release artifacts @@ -244,7 +245,6 @@ elif [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then PLUGIN_PREFIX="apache-metron-bro-plugin-kafka_" ARTIFACT_PREFIX="${PLUGIN_PREFIX}" TAG_VERSION="${CURRENT_VERSION}" - TAG_VERSION=$(echo ${CURRENT_VERSION} | cut -d. -f-2) TAG="${PLUGIN_PREFIX}${TAG_VERSION}${TAG_POSTFIX}" # Handle special tag case from prior release if [ "${TAG_VERSION}" = "0.1" ]; then @@ -263,6 +263,7 @@ cd "$GIT_DIR" git checkout "${CAPITAL_REPO}_${VERSION}" # The branch only exists if this is not a practice run if [ "${PRACTICE_RUN}" = false ]; then + printf "Pulling latest state of branch\n" git pull fi git tag "${ARTIFACT}" @@ -317,15 +318,20 @@ fi tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/NOTICE" # Add the directory and commit to subversion +COMMIT_DIR=$(basename ${ART_DIR}) if [ "${PRACTICE_RUN}" = true ]; then printf "This is a practice run. Not running the following commands:\n" - printf "svn add %s-RC%s\n" "$VERSION" "${RC_NUM}" - printf "svn commit -m \"Adding artifacts for Metron %s-RC%s\"\n" "$VERSION" "${RC_NUM}" + printf "\n" ${COMMIT_DIR} + printf "" "${CHOSEN_REPO}" "${COMMIT_DIR}" else printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n" - cd "$WORKDIR/dev/metron/" - COMMIT_DIR=$(basename ${ART_DIR}) - # svn add ${COMMIT_DIR} - #svn commit -m "Adding artifacts for Metron ${COMMIT_DIR}" + # Metron goes in the root of the dir, submodules go in folder + if [ "${CHOSEN_REPO}" = "${METRON_REPO_NAME}" ]; then + cd "$WORKDIR/dev/metron/" + else + cd "$WORKDIR/dev/metron/${CHOSEN_REPO}" + fi + svn add ${COMMIT_DIR} + svn commit -m "Adding artifacts for ${CHOSEN_REPO} ${COMMIT_DIR}" fi From eb60a5a67f8ffc84949386e8a636f6776112c34f Mon Sep 17 00:00:00 2001 From: Justin Leet Date: Tue, 9 Oct 2018 09:15:33 -0400 Subject: [PATCH 10/10] slight cleanup and readme --- dev-utilities/release-utils/README.md | 129 ++++++++++++++++++ .../release-utils/prepare-release-candidate | 42 +++--- 2 files changed, 150 insertions(+), 21 deletions(-) create mode 100644 dev-utilities/release-utils/README.md diff --git a/dev-utilities/release-utils/README.md b/dev-utilities/release-utils/README.md new file mode 100644 index 0000000000..d1fd81623a --- /dev/null +++ b/dev-utilities/release-utils/README.md @@ -0,0 +1,129 @@ + + +# Release Tools + +This project contains tools to assist Apache Metron project committers. + +## Prepare Release Candidate + +This script automates the process of creating a release candidate from `apache/metron` or `apache/metron-bro-plugin-kafka`. The script will prompt for various information necessary. Ensure your signing key is setup per [Release Signing](https://www.apache.org/dev/release-signing.html) and [Apache GnuPGP Instructions](https://www.apache.org/dev/openpgp.html#gnupg) + +When prompted the `[value in brackets]` is used by default. To accept the default, simply press `enter`. If you would like to change the default, type it in and hit `enter` when done. + +In the following example, enter the appropriate information + +1. Execute the script. + + The first time the script is run, you will be prompted for additional information including your Apache username and Apache email. These values are persisted in `~/.metron-prepare-release-candidate`. Subsequent executions of the script will retrieve these values, rather than prompting you again for them. + + ``` + $ ./prepare-release-candidate + your apache userid []: leet + your apache email [leet@apache.org]: + ``` + +1. Select a repository we're creating an RC for. + + ``` + [1] metron + [2] metron-bro-plugin-kafka + which repo? [1]: 1 + ``` + +1. Enter the current version number. This will be the base for the CHANGES file + + ``` + current version: 0.6.0 + ``` + +1. Enter the version being built. + + ``` + version being built: 0.6.1 + ``` + +1. Enter the current RC number + + ``` + release candidate number: 1 + ``` + +1. Enter the branch we're releasing from. In most cases, this will be master, but for maintenance releases it can be another branch. + + ``` + base revision branch or hash for release candidate [master]: + ``` + +1. Enter the signing key id. + + ``` + signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): + ``` + +1. Enter if this is a practice run. In a practice run, nothing is pushed to SVN, but everything is setup and built otherwise. + + ``` + do a live run (push to remote repositories?) [y/n] + ``` + +1. Wait for all repos to be checked out to complete. There will be some additional work done, e.g. along with branch and tag creation. In a live run, you may be prompted for Git credentials to push a branch. + + ``` + Checking out repo: https://dist.apache.org/repos/dist/dev/metron + Checking out repo: dev + Checking out repo: https://dist.apache.org/repos/dist/release/metron + Checking out repo: release + Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git + Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'... + remote: Counting objects: 46146, done. + remote: Compressing objects: 100% (15568/15568), done. + remote: Total 46146 (delta 21513), reused 43696 (delta 19489) + Receiving objects: 100% (46146/46146), 56.00 MiB | 1.04 MiB/s, done. + Resolving deltas: 100% (21513/21513), done. + Creating branch: Metron_0.6.1 + Using git rev: master + Already on 'master' + Your branch is up to date with 'origin/master'. + Switched to a new branch 'Metron_0.6.1' + This is a practice run. Not running + Creating tentative git tag <0.6.1-rc1>. Do not push this tag until RC is ready for community review. + Already on 'Metron_0.6.1' + Creating the RC tarball for tag apache-metron-0.6.1-rc1 + Creating the SHA hash files + ``` + +1. Provide the passphrase to `gpg` to sign the artifacts. + + ``` + Signing the release tarball + Copying release artifacts + ``` + +1. Shortly afterwards the RC will be finalized. In a practice run, this will not be pushed back to SVN. + ``` + Creating CHANGES file + Extracting LICENSE, NOTICE, and KEYS from tarball + x LICENSE + x NOTICE + This is a practice run. Not running the following commands: + + + ``` + +At this point, all RC artifacts have been created. In a live run, these will have been pushed to the appropriate repositories and are ready for community review. diff --git a/dev-utilities/release-utils/prepare-release-candidate b/dev-utilities/release-utils/prepare-release-candidate index b73ac3c279..bedee9b744 100755 --- a/dev-utilities/release-utils/prepare-release-candidate +++ b/dev-utilities/release-utils/prepare-release-candidate @@ -45,6 +45,25 @@ if [ -f $CONFIG_FILE ]; then echo " ...using settings from $CONFIG_FILE" fi +# apache id of committer (you) +if [ -z "${APACHE_NAME}" ]; then + read -p " your apache userid [${APACHE_NAME}]: " INPUT + [ -n "$INPUT" ] && APACHE_NAME=$INPUT + + # write setting to config file + echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE +fi + +# apache email addr of committer (you) +if [ -z "${APACHE_EMAIL}" ]; then + APACHE_EMAIL=${APACHE_NAME}@apache.org + read -p " your apache email [${APACHE_EMAIL}]: " INPUT + [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT + + # write setting to config file, so it is not needed next time + echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE +fi + # which repo? metron or metron-bro-plugin-kafka getrepo() { echo " [1] ${METRON_REPO_NAME}" @@ -69,25 +88,6 @@ CHOSEN_REPO=$(basename ${UPSTREAM%%.git}) # Need the capitalized version of the repos some naming CAPITAL_REPO="$(tr '[:lower:]' '[:upper:]' <<< ${CHOSEN_REPO:0:1})${CHOSEN_REPO:1}" -# apache id of committer (you) -if [ -z "${APACHE_NAME}" ]; then - read -p " your apache userid [${APACHE_NAME}]: " INPUT - [ -n "$INPUT" ] && APACHE_NAME=$INPUT - - # write setting to config file - echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE -fi - -# apache email addr of committer (you) -if [ -z "${APACHE_EMAIL}" ]; then - APACHE_EMAIL=${APACHE_NAME}@apache.org - read -p " your apache email [${APACHE_EMAIL}]: " INPUT - [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT - - # write setting to config file, so it is not needed next time - echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE -fi - getcurrentversion() { # currently released version. Used for CHANGES file read -p " current version: " CURRENT_VERSION @@ -141,7 +141,7 @@ rm -rf "$WORKDIR" mkdir "$WORKDIR" getbaserev() { - read -p " base revision branch or hash for release candidate (master if empty) " GIT_REF + read -p " base revision branch or hash for release candidate [master]: " GIT_REF GIT_REF=${GIT_REF:-master} # check to see if we were given branch @@ -322,7 +322,7 @@ COMMIT_DIR=$(basename ${ART_DIR}) if [ "${PRACTICE_RUN}" = true ]; then printf "This is a practice run. Not running the following commands:\n" printf "\n" ${COMMIT_DIR} - printf "" "${CHOSEN_REPO}" "${COMMIT_DIR}" + printf "\n" "${CHOSEN_REPO}" "${COMMIT_DIR}" else printf "Adding artifacts for Metron ${VERSION}-RC${RC_NUM} to dev SVN\n" # Metron goes in the root of the dir, submodules go in folder