From b50fb2f0fe3481471ffc26360e01319f0187ee8d Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Wed, 26 Aug 2020 20:51:50 -0400 Subject: [PATCH 1/9] wip --- scripts/publish_prod.sh | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 scripts/publish_prod.sh diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh new file mode 100755 index 00000000..1baca478 --- /dev/null +++ b/scripts/publish_prod.sh @@ -0,0 +1,78 @@ +#!/bin/bash +set -e + +BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ $BRANCH != "master" ]; then + echo "Not on master, aborting" + exit 1 +fi + +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo 'AWS_ACCESS_KEY_ID not set. Are you using aws-vault?' + exit 1 +fi + +if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo 'AWS_SECRET_ACCESS_KEY not set. Are you using aws-vault?' + exit 1 +fi + +if [ -z "$AWS_SESSION_TOKEN" ]; then + echo 'AWS_SESSION_TOKEN not set. Are you using aws-vault?' + exit 1 +fi + +echo 'Checking Regions' +# ./scripts/list_layers.sh +echo "layer 1" + +# read -p "Do the list look good? " -n 1 -r +# echo +# if [[ ! $REPLY =~ ^[Yy]$ ]] +# then +# [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 +# fi +SEMVER_REGEX="(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$" +VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./scripts/file.txt) + +echo "$VERSION_LINE" +MINOR_VERSION_NUM=$(echo "$VERSION_LINE" | cut -d '.' -f 2) + +echo "$MINOR_VERSION_NUM" + +NEW_VERSION=$(($MINOR_VERSION_NUM + 1)) +echo $NEW_VERSION + +read -p "Does this Minor version LGTY? ${MINOR_VERSION_NUM}" -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 +fi + +echo "Replacing __version__ in ./datadog_lambda/__init__.py" +sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"\1.${NEW_VERSION}.\3\"/" ./scripts/file.txt + + + +# PY_FILE=$(cat "./datadog_lambda/__init__.py") + +# echo $PY_FILE +# echo "WTF" + +# NEW_VERSION=$(echo $PY_FILE | grep -o "$SEMVER_REGEX" ) +# NEW_VERSION=$(echo $PY_FILE | grep -o "[0-9]+.([0-9]+).[0-9]+") + +# echo $NEW_VERSION + +# echo 'Publishing to Node' +# yarn build +# yarn publish --new-version "$PACKAGE_VERSION" + +# echo 'Tagging Release' +# git tag "v$PACKAGE_VERSION" +# git push origin "refs/tags/v$PACKAGE_VERSION" + +# echo 'Publishing Lambda Layer' +# ./scripts/build_layers.sh +# ./scripts/publish_layers.sh \ No newline at end of file From 9cc92624e4b36291973a9d169a635957e4ed61de Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 09:16:57 -0400 Subject: [PATCH 2/9] Update module minor version to 22 --- datadog_lambda/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index 758d3fac..89798587 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = "2.21.0" +__version__ = "2.22.0" import os From 83ef6d7dd3e330e06a824d43224f5eb4d07012b6 Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 09:24:52 -0400 Subject: [PATCH 3/9] add publish script --- datadog_lambda/__init__.py | 2 +- scripts/publish_prod.sh | 103 ++++++++++++++++++------------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index 89798587..758d3fac 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = "2.22.0" +__version__ = "2.21.0" import os diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index 1baca478..e17ba5b5 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -2,77 +2,72 @@ set -e BRANCH=$(git rev-parse --abbrev-ref HEAD) +echo $BRANCH + if [ $BRANCH != "master" ]; then echo "Not on master, aborting" exit 1 fi -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo 'AWS_ACCESS_KEY_ID not set. Are you using aws-vault?' - exit 1 -fi +echo 'Checking AWS Regions' +aws-vault exec prod-engineering -- ./scripts/list_layers.sh -if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then - echo 'AWS_SECRET_ACCESS_KEY not set. Are you using aws-vault?' - exit 1 -fi +echo 'Checking USGov Regions' +AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/list_layers.sh -if [ -z "$AWS_SESSION_TOKEN" ]; then - echo 'AWS_SESSION_TOKEN not set. Are you using aws-vault?' - exit 1 +read -p "Do the list look good?(y/n) " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 fi -echo 'Checking Regions' -# ./scripts/list_layers.sh -echo "layer 1" - -# read -p "Do the list look good? " -n 1 -r -# echo -# if [[ ! $REPLY =~ ^[Yy]$ ]] -# then -# [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 -# fi -SEMVER_REGEX="(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\-[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$" -VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./scripts/file.txt) - -echo "$VERSION_LINE" +VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./datadog_lambda/__init__.py) MINOR_VERSION_NUM=$(echo "$VERSION_LINE" | cut -d '.' -f 2) - -echo "$MINOR_VERSION_NUM" - +echo "" +echo "Current version found: $MINOR_VERSION_NUM" +echo "" NEW_VERSION=$(($MINOR_VERSION_NUM + 1)) -echo $NEW_VERSION -read -p "Does this Minor version LGTY? ${MINOR_VERSION_NUM}" -n 1 -r +read -p "Ready to publish layers and update the minor version. $MINOR_VERSION_NUM to $NEW_VERSION?(y/n)" -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]] then [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 fi +echo "" echo "Replacing __version__ in ./datadog_lambda/__init__.py" -sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"\1.${NEW_VERSION}.\3\"/" ./scripts/file.txt - - - -# PY_FILE=$(cat "./datadog_lambda/__init__.py") - -# echo $PY_FILE -# echo "WTF" - -# NEW_VERSION=$(echo $PY_FILE | grep -o "$SEMVER_REGEX" ) -# NEW_VERSION=$(echo $PY_FILE | grep -o "[0-9]+.([0-9]+).[0-9]+") - -# echo $NEW_VERSION - -# echo 'Publishing to Node' -# yarn build -# yarn publish --new-version "$PACKAGE_VERSION" - -# echo 'Tagging Release' -# git tag "v$PACKAGE_VERSION" -# git push origin "refs/tags/v$PACKAGE_VERSION" +echo "" +sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"\1.${NEW_VERSION}.\3\"/" ./datadog_lambda/__init__.py + +git commit ./datadog_lambda/__init__.py -m "Update module minor version to ${NEW_VERSION}" + +echo "" +echo "Building layers..." +./scripts/build_layers.sh + +echo "" +echo "Publishing layers to AWS regions..." +aws-vault exec prod-engineering --./scripts/publish_layers.sh +echo "" +echo "Publishing layers to USGov regions..." +AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/publish_layers.sh + +echo "" +echo 'Pushing updates to github' +git push origin master +git push origin "refs/tags/v$NEW_VERSION" + +echo "" +echo "Publishing to https://pypi.org/project/datadog-lambda/" +./scripts/pypi.sh + +echo "" +echo "Now create a new release with the tag v${MINOR_VERSION_NUM} created" +echo "https://github.com/DataDog/datadog-lambda-python/releases/new" +echo "" +echo "Then publish a new serverless-plugin-datadog version with the new layer versions!" +echo "https://github.com/DataDog/devops/wiki/Datadog-Serverless-Plugin#release" +echo "" -# echo 'Publishing Lambda Layer' -# ./scripts/build_layers.sh -# ./scripts/publish_layers.sh \ No newline at end of file From 164af6668a33ac8db8e1de73f03ea5c6051e4371 Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 09:40:08 -0400 Subject: [PATCH 4/9] remove us gov cloud and remove aws-vault --- scripts/publish_prod.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index e17ba5b5..9a09b792 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -9,11 +9,23 @@ if [ $BRANCH != "master" ]; then exit 1 fi -echo 'Checking AWS Regions' -aws-vault exec prod-engineering -- ./scripts/list_layers.sh +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo 'AWS_ACCESS_KEY_ID not set. Are you using aws-vault?' + exit 1 +fi -echo 'Checking USGov Regions' -AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/list_layers.sh +if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo 'AWS_SECRET_ACCESS_KEY not set. Are you using aws-vault?' + exit 1 +fi + +if [ -z "$AWS_SESSION_TOKEN" ]; then + echo 'AWS_SESSION_TOKEN not set. Are you using aws-vault?' + exit 1 +fi + +echo 'Checking AWS Regions' +./scripts/list_layers.sh read -p "Do the list look good?(y/n) " -n 1 -r echo @@ -49,10 +61,7 @@ echo "Building layers..." echo "" echo "Publishing layers to AWS regions..." -aws-vault exec prod-engineering --./scripts/publish_layers.sh -echo "" -echo "Publishing layers to USGov regions..." -AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/publish_layers.sh +./scripts/publish_layers.sh echo "" echo 'Pushing updates to github' @@ -64,7 +73,7 @@ echo "Publishing to https://pypi.org/project/datadog-lambda/" ./scripts/pypi.sh echo "" -echo "Now create a new release with the tag v${MINOR_VERSION_NUM} created" +echo "Now create a new release with the tag v${NEW_VERSION} created" echo "https://github.com/DataDog/datadog-lambda-python/releases/new" echo "" echo "Then publish a new serverless-plugin-datadog version with the new layer versions!" From 20e4311cb730858afe217c4855e3dfc44802a8b8 Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 09:55:56 -0400 Subject: [PATCH 5/9] cr feedback --- scripts/publish_prod.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index 9a09b792..8dc10686 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -1,4 +1,7 @@ #!/bin/bash + +# Use with `aws-vault exec PROFILE -- ./publish_prod.sh + set -e BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -24,6 +27,17 @@ if [ -z "$AWS_SESSION_TOKEN" ]; then exit 1 fi +# Read the desired version +if [ -z "$1" ]; then + echo "Must specify a desired version number" + exit 1 +elif [[ ! $1 =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Must use a semantic version, e.g., 3.1.4" + exit 1 +else + NEW_VERSION=$1 +fi + echo 'Checking AWS Regions' ./scripts/list_layers.sh @@ -35,13 +49,10 @@ then fi VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./datadog_lambda/__init__.py) -MINOR_VERSION_NUM=$(echo "$VERSION_LINE" | cut -d '.' -f 2) -echo "" -echo "Current version found: $MINOR_VERSION_NUM" -echo "" -NEW_VERSION=$(($MINOR_VERSION_NUM + 1)) +CURRENT_VERSION=$(echo "$VERSION_LINE" | cut -d '"' -f 2) +echo "$CURRENT_VERSION" -read -p "Ready to publish layers and update the minor version. $MINOR_VERSION_NUM to $NEW_VERSION?(y/n)" -n 1 -r +read -p "Ready to publish layers and update the version from $CURRENT_VERSION to $NEW_VERSION?(y/n)" -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]] then @@ -51,9 +62,9 @@ fi echo "" echo "Replacing __version__ in ./datadog_lambda/__init__.py" echo "" -sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"\1.${NEW_VERSION}.\3\"/" ./datadog_lambda/__init__.py +sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"$NEW_VERSION\"/" ./datadog_lambda/__init__.py -git commit ./datadog_lambda/__init__.py -m "Update module minor version to ${NEW_VERSION}" +git commit ./datadog_lambda/__init__.py -m "Update module version to ${NEW_VERSION}" echo "" echo "Building layers..." @@ -77,6 +88,5 @@ echo "Now create a new release with the tag v${NEW_VERSION} created" echo "https://github.com/DataDog/datadog-lambda-python/releases/new" echo "" echo "Then publish a new serverless-plugin-datadog version with the new layer versions!" -echo "https://github.com/DataDog/devops/wiki/Datadog-Serverless-Plugin#release" echo "" From 07c19feff8bf913293c9db922c5d2fc050547b6f Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 10:22:28 -0400 Subject: [PATCH 6/9] make sure we use the minor version for tagging --- scripts/publish_prod.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index 8dc10686..8369bd3d 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -76,15 +76,16 @@ echo "Publishing layers to AWS regions..." echo "" echo 'Pushing updates to github' +MINOR_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2) git push origin master -git push origin "refs/tags/v$NEW_VERSION" +git push origin "refs/tags/v$MINOR_VERSION" echo "" echo "Publishing to https://pypi.org/project/datadog-lambda/" ./scripts/pypi.sh echo "" -echo "Now create a new release with the tag v${NEW_VERSION} created" +echo "Now create a new release with the tag v${MINOR_VERSION} created" echo "https://github.com/DataDog/datadog-lambda-python/releases/new" echo "" echo "Then publish a new serverless-plugin-datadog version with the new layer versions!" From f7897db49a9a988dd39c8239ed15862ed9fe3926 Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 10:23:15 -0400 Subject: [PATCH 7/9] small NIT --- scripts/publish_prod.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index 8369bd3d..d6ff1e2a 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Use with `aws-vault exec PROFILE -- ./publish_prod.sh +# Use with `aws-vault exec -- ./publish_prod.sh set -e From 1531d2dad7695ebba3571a3bf15653a98f18a616 Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 10:34:20 -0400 Subject: [PATCH 8/9] Update module version to 2.22.0 --- datadog_lambda/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index 758d3fac..89798587 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = "2.21.0" +__version__ = "2.22.0" import os From ac49ebb0b4005a54f62423dc7c4d1850c08d50e5 Mon Sep 17 00:00:00 2001 From: Sergio Prada Date: Thu, 27 Aug 2020 10:48:22 -0400 Subject: [PATCH 9/9] make last changes --- datadog_lambda/__init__.py | 2 +- scripts/publish_prod.sh | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/datadog_lambda/__init__.py b/datadog_lambda/__init__.py index 89798587..758d3fac 100644 --- a/datadog_lambda/__init__.py +++ b/datadog_lambda/__init__.py @@ -1,6 +1,6 @@ # The minor version corresponds to the Lambda layer version. # E.g.,, version 0.5.0 gets packaged into layer version 5. -__version__ = "2.22.0" +__version__ = "2.21.0" import os diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index d6ff1e2a..c5c49d7e 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -27,7 +27,6 @@ if [ -z "$AWS_SESSION_TOKEN" ]; then exit 1 fi -# Read the desired version if [ -z "$1" ]; then echo "Must specify a desired version number" exit 1 @@ -41,7 +40,7 @@ fi echo 'Checking AWS Regions' ./scripts/list_layers.sh -read -p "Do the list look good?(y/n) " -n 1 -r +read -p "Do the list look good? (y/n) " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]] then @@ -50,9 +49,8 @@ fi VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./datadog_lambda/__init__.py) CURRENT_VERSION=$(echo "$VERSION_LINE" | cut -d '"' -f 2) -echo "$CURRENT_VERSION" -read -p "Ready to publish layers and update the version from $CURRENT_VERSION to $NEW_VERSION?(y/n)" -n 1 -r +read -p "Ready to publish layers and update the version from $CURRENT_VERSION to $NEW_VERSION? (y/n)" -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]] then @@ -80,6 +78,17 @@ MINOR_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2) git push origin master git push origin "refs/tags/v$MINOR_VERSION" + +echo 'Checking AWS Regions Again...' +./scripts/list_layers.sh + + +read -p "Do regions look good? Ready to publish $NEW_VERSION to Pypi? (y/n)" -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 +fi echo "" echo "Publishing to https://pypi.org/project/datadog-lambda/" ./scripts/pypi.sh