From 7587664a8c3d1f03a01276650a539ac7ef1aaf2c Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Tue, 25 Jul 2017 15:07:59 +0200 Subject: [PATCH 1/3] [FLINK-7281] Remove old and misleading release scripts/README --- tools/how_to_release.md | 34 ---------------------------------- tools/prepare_release.sh | 23 ----------------------- 2 files changed, 57 deletions(-) delete mode 100644 tools/how_to_release.md delete mode 100644 tools/prepare_release.sh diff --git a/tools/how_to_release.md b/tools/how_to_release.md deleted file mode 100644 index 70647367f8c30..0000000000000 --- a/tools/how_to_release.md +++ /dev/null @@ -1,34 +0,0 @@ - -# How to release a new version of Apache Flink - - -You need to have the following environment variables set: - -export OLD_VERSION=0.6-SNAPSHOT -export NEW_VERSION=0.6 -export GPG_KEY=XX -export GPG_PASSPHRASE=XX - -./prepare_release_files.sh - - - - - diff --git a/tools/prepare_release.sh b/tools/prepare_release.sh deleted file mode 100644 index 7accbea098138..0000000000000 --- a/tools/prepare_release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -echo "Updating version from $OLD_VERSION to $NEW_VERSION" - -find flink -name 'pom.xml' -type f -exec sed -i 's#$OLD_VERSION#$NEW_VERSION#' {} \; - From 663388725339f879521f6fbba7f07a6b327b1e70 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Tue, 25 Jul 2017 15:30:26 +0200 Subject: [PATCH 2/3] [FLINK-7281] Make quickstart versions depend on project versions --- .../src/main/resources/archetype-resources/pom.xml | 6 +++--- .../src/main/resources/archetype-resources/pom.xml | 6 +++--- pom.xml | 8 +++++--- tools/create_release_files.sh | 3 --- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index b04f75fc79666..54ab1b5b10bac 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -30,9 +30,9 @@ under the License. UTF-8 - 1.3-SNAPSHOT - 1.7.7 - 1.2.17 + @project.version@ + @slf4j.version@ + @log4j.version @scala.binary.version@ diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index 6a5eed09d03fe..6402d1d5b90f8 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -45,9 +45,9 @@ under the License. UTF-8 - 1.3-SNAPSHOT - 1.7.7 - 1.2.17 + @project.version@ + @slf4j.version@ + @log4j.version @scala.binary.version@ diff --git a/pom.xml b/pom.xml index 76aa6d73c2ff5..91c8cd3dad61b 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,8 @@ under the License. 18.0 2.3-custom 1.7 + 1.7.7 + 1.2.17 @@ -227,19 +229,19 @@ under the License. org.slf4j slf4j-api - 1.7.7 + ${slf4j.version} org.slf4j slf4j-log4j12 - 1.7.7 + ${slf4j.version} log4j log4j - 1.2.17 + ${log4j.version} diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh index f4c46735efab8..90e7245a29f3d 100755 --- a/tools/create_release_files.sh +++ b/tools/create_release_files.sh @@ -175,9 +175,6 @@ make_source_release() { #change version in all pom files find . -name 'pom.xml' -type f -exec perl -pi -e 's#'$OLD_VERSION'#'$NEW_VERSION'#' {} \; - #change version in quickstart archetypes - find . -name 'pom.xml' -type f -exec perl -pi -e 's#'$OLD_VERSION'#'$NEW_VERSION'#' {} \; - #change version of documentation cd docs perl -pi -e "s#^version: .*#version: ${NEW_VERSION}#" _config.yml From 6df9e3ec944a5d13945dbada30e9b88aef16a08b Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Thu, 27 Jul 2017 15:07:44 +0200 Subject: [PATCH 3/3] [FLINK-7290] Add modular release scripts --- tools/releasing/create_binary_release.sh | 104 +++++++++++++++++++++++ tools/releasing/create_release_branch.sh | 65 ++++++++++++++ tools/releasing/create_source_release.sh | 63 ++++++++++++++ tools/releasing/deploy_staging_jars.sh | 54 ++++++++++++ 4 files changed, 286 insertions(+) create mode 100755 tools/releasing/create_binary_release.sh create mode 100755 tools/releasing/create_release_branch.sh create mode 100755 tools/releasing/create_source_release.sh create mode 100755 tools/releasing/deploy_staging_jars.sh diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh new file mode 100755 index 0000000000000..7971d8c5934c9 --- /dev/null +++ b/tools/releasing/create_binary_release.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +RELEASE_VERSION=${RELEASE_VERSION:-1.3-SNAPSHOT} +SCALA_VERSION=none +HADOOP_VERSION=none +SKIP_GPG=${SKIP_GPG:-false} +MVN=${MVN:-mvn} + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +if [ "$(uname)" == "Darwin" ]; then + SHASUM="shasum -a 512" + MD5SUM="md5 -r" +else + SHASUM="sha512sum" + MD5SUM="md5sum" +fi + +########################### + +# build maven package, create Flink distribution, generate signature +make_binary_release() { + NAME=$1 + FLAGS=$2 + SCALA_VERSION=$3 + + echo "Creating binary release name: $NAME, flags: $FLAGS, SCALA_VERSION: ${SCALA_VERSION}" + dir_name="flink-$RELEASE_VERSION-bin-$NAME-scala_${SCALA_VERSION}" + + ./tools/change-scala-version.sh ${SCALA_VERSION} + + # enable release profile here (to check for the maven version) + $MVN clean package $FLAGS -DskipTests -Prelease -Dgpg.skip + + cd flink-dist/target/flink-*-bin/ + tar czf "${dir_name}.tgz" flink-* + + cp flink-*.tgz ../../../ + cd ../../../ + + # Sign md5 and sha the tgz + if [ "$SKIP_GPG" == "false" ] ; then + gpg --armor --detach-sig "${dir_name}.tgz" + fi + $MD5SUM "${dir_name}.tgz" > "${dir_name}.tgz.md5" + $SHASUM "${dir_name}.tgz" > "${dir_name}.tgz.sha" +} + +cd .. + + +if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then + make_binary_release "hadoop2" "" 2.10 + make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.10" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.10" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.10" + + make_binary_release "hadoop2" "" 2.11 + make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.11" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.11" +elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ] +then + make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.10" + make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11" +elif [ "$SCALA_VERSION" != none ] && [ "$HADOOP_VERSION" == "none" ] +then + make_binary_release "hadoop2" "" $SCALA_VERSION + make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "$SCALA_VERSION" + make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "$SCALA_VERSION" + make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "$SCALA_VERSION" +else + make_binary_release "hadoop2x" "-Dhadoop.version=$HADOOP_VERSION" "$SCALA_VERSION" +fi diff --git a/tools/releasing/create_release_branch.sh b/tools/releasing/create_release_branch.sh new file mode 100755 index 0000000000000..88f8342dc519c --- /dev/null +++ b/tools/releasing/create_release_branch.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +OLD_VERSION=${OLD_VERSION:-1.2-SNAPSHOT} +NEW_VERSION=${NEW_VERSION:-1.3-SNAPSHOT} +RELEASE_CANDIDATE=${RELEASE_CANDIDATE:-none} +MVN=${MVN:-mvn} + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +########################### + +cd .. + +target_branch=release-$NEW_VERSION +if [ "$RELEASE_CANDIDATE" != "none" ]; then + target_branch=$target_branch-$RELEASE_CANDIDATE +fi + +git checkout -b $target_branch + +#change version in all pom files +find . -name 'pom.xml' -type f -exec perl -pi -e 's#'$OLD_VERSION'#'$NEW_VERSION'#' {} \; + +#change version of documentation +cd docs +perl -pi -e "s#^version: .*#version: \"${NEW_VERSION}\"#" _config.yml +perl -pi -e "s#^version_short: .*#version_short: \"${NEW_VERSION}\"#" _config.yml +cd .. + +git commit -am "Commit for release $NEW_VERSION" + +RELEASE_HASH=`git rev-parse HEAD` +echo "Echo created release hash $RELEASE_HASH" + +echo "Done. Don't forget to commit the release version" diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh new file mode 100755 index 0000000000000..2b29527f9e8d7 --- /dev/null +++ b/tools/releasing/create_source_release.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +RELEASE_VERSION=${RELEASE_VERSION:-1.3-SNAPSHOT} +MVN=${MVN:-mvn} + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +if [ "$(uname)" == "Darwin" ]; then + SHASUM="shasum -a 512" + MD5SUM="md5 -r" +else + SHASUM="sha512sum" + MD5SUM="md5sum" +fi + +########################### + +cd .. + +echo "Creating source package" + +rsync -a \ + --exclude ".git" --exclude ".gitignore" --exclude ".gitattributes" --exclude ".travis.yml" \ + --exclude "deploysettings.xml" --exclude "CHANGELOG" --exclude ".github" --exclude "target" \ + --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \ + . flink-$RELEASE_VERSION + +tar czf flink-${RELEASE_VERSION}-src.tgz flink-$RELEASE_VERSION +gpg --armor --detach-sig flink-$RELEASE_VERSION-src.tgz +$MD5SUM flink-$RELEASE_VERSION-src.tgz > flink-$RELEASE_VERSION-src.tgz.md5 +$SHASUM flink-$RELEASE_VERSION-src.tgz > flink-$RELEASE_VERSION-src.tgz.sha + +rm -rf flink-$RELEASE_VERSION diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh new file mode 100755 index 0000000000000..96ef638b582d0 --- /dev/null +++ b/tools/releasing/deploy_staging_jars.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## +## Variables with defaults (if not overwritten by environment) +## +MVN=${MVN:-mvn} + +# fail immediately +set -o errexit +set -o nounset +# print command before executing +set -o xtrace + +CURR_DIR=`pwd` +if [[ `basename $CURR_DIR` != "tools" ]] ; then + echo "You have to call the script from the tools/ dir" + exit 1 +fi + +########################### + +cd .. + +echo "Deploying to repository.apache.org" + +echo "Deploying Scala 2.11 version" +cd tools && ./change-scala-version.sh 2.11 && cd .. + +$MVN clean deploy -Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10 + +# It is important to first deploy scala 2.11 and then scala 2.10 so that the quickstarts (which are independent of the scala version) +# are depending on scala 2.10. + +echo "Deploying Scala 2.10 version" +cd tools && ./change-scala-version.sh 2.10 && cd .. + +$MVN clean deploy -Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10