From af4e1ee6dcf0acad7b778324b7e91f299d788068 Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:26:45 -0800 Subject: [PATCH 01/12] try reading new env-file param --- bin/compile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bin/compile b/bin/compile index fc9075a3b..5cf832de7 100755 --- a/bin/compile +++ b/bin/compile @@ -7,8 +7,22 @@ set -o pipefail # don't ignore exit codes when piping output # Configure directories build_dir=$1 cache_dir=$2 +env_file=$3 + bp_dir=$(cd $(dirname $0); cd ..; pwd) +if [ "$env_file" ]; then + status "before:" + env + + status "Here's your env_file:" + status $env_file + source $env_file + + status "after:" + env +fi + # Load some convenience functions like status(), echo(), and indent() source $bp_dir/bin/common.sh From e9a658eae6a46d403b93ccd88f87adc915994d88 Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:28:33 -0800 Subject: [PATCH 02/12] debug --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 5cf832de7..06f332341 100755 --- a/bin/compile +++ b/bin/compile @@ -2,7 +2,7 @@ set -e # fail fast set -o pipefail # don't ignore exit codes when piping output -# set -x # enable debugging +set -x # enable debugging # Configure directories build_dir=$1 @@ -16,7 +16,7 @@ if [ "$env_file" ]; then env status "Here's your env_file:" - status $env_file + status "$env_file" source $env_file status "after:" From 03d6e3ba087955d10966be5310ebedc63526438d Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:31:39 -0800 Subject: [PATCH 03/12] don't forget those utility functions --- bin/compile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/compile b/bin/compile index 06f332341..fcbc00b2d 100755 --- a/bin/compile +++ b/bin/compile @@ -11,6 +11,12 @@ env_file=$3 bp_dir=$(cd $(dirname $0); cd ..; pwd) +# Load some convenience functions like status(), echo(), and indent() +source $bp_dir/bin/common.sh + +# Output npm debug info on error +trap cat_npm_debug_log ERR + if [ "$env_file" ]; then status "before:" env @@ -23,12 +29,6 @@ if [ "$env_file" ]; then env fi -# Load some convenience functions like status(), echo(), and indent() -source $bp_dir/bin/common.sh - -# Output npm debug info on error -trap cat_npm_debug_log ERR - # Look in package.json's engines.node field for a semver range semver_range=$(cat $build_dir/package.json | $bp_dir/vendor/jq -r .engines.node) From 5d4673813b4c8d26bcd5999800b093c365f9886f Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:34:40 -0800 Subject: [PATCH 04/12] subshell --- bin/compile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/compile b/bin/compile index fcbc00b2d..0495cae3a 100755 --- a/bin/compile +++ b/bin/compile @@ -17,6 +17,7 @@ source $bp_dir/bin/common.sh # Output npm debug info on error trap cat_npm_debug_log ERR +( if [ "$env_file" ]; then status "before:" env @@ -28,6 +29,7 @@ if [ "$env_file" ]; then status "after:" env fi +) # Look in package.json's engines.node field for a semver range semver_range=$(cat $build_dir/package.json | $bp_dir/vendor/jq -r .engines.node) From 506996e33e03eb71193593dc85bfdf7885f196d5 Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:36:53 -0800 Subject: [PATCH 05/12] bashin --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 0495cae3a..0f9f06804 100755 --- a/bin/compile +++ b/bin/compile @@ -24,7 +24,7 @@ if [ "$env_file" ]; then status "Here's your env_file:" status "$env_file" - source $env_file + export $(cat $env_file) status "after:" env From 2f499530ba6613a506ec561e60c603a73692386f Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:39:08 -0800 Subject: [PATCH 06/12] experiment --- bin/compile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 0f9f06804..04bd69fc4 100755 --- a/bin/compile +++ b/bin/compile @@ -17,19 +17,17 @@ source $bp_dir/bin/common.sh # Output npm debug info on error trap cat_npm_debug_log ERR -( if [ "$env_file" ]; then status "before:" env status "Here's your env_file:" status "$env_file" - export $(cat $env_file) + export $(egrep -v "^(PATH|foo)" $env_file) status "after:" env fi -) # Look in package.json's engines.node field for a semver range semver_range=$(cat $build_dir/package.json | $bp_dir/vendor/jq -r .engines.node) From 1e9b489019d7b40f289bd1c4370a63ad03583990 Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 11:40:22 -0800 Subject: [PATCH 07/12] experiment --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 04bd69fc4..1944002bf 100755 --- a/bin/compile +++ b/bin/compile @@ -23,7 +23,7 @@ if [ "$env_file" ]; then status "Here's your env_file:" status "$env_file" - export $(egrep -v "^(PATH|foo)" $env_file) + export $(egrep -v "^(PATH)" $env_file) status "after:" env From c56b58c8348a042a86a1b620bd2c5ec9735852c0 Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 13 Dec 2013 12:18:52 -0800 Subject: [PATCH 08/12] experiment --- bin/compile | 28 ++++++++++++---------------- bin/test | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/bin/compile b/bin/compile index 1944002bf..083ab2775 100755 --- a/bin/compile +++ b/bin/compile @@ -2,7 +2,7 @@ set -e # fail fast set -o pipefail # don't ignore exit codes when piping output -set -x # enable debugging +# set -x # enable debugging # Configure directories build_dir=$1 @@ -17,18 +17,6 @@ source $bp_dir/bin/common.sh # Output npm debug info on error trap cat_npm_debug_log ERR -if [ "$env_file" ]; then - status "before:" - env - - status "Here's your env_file:" - status "$env_file" - export $(egrep -v "^(PATH)" $env_file) - - status "after:" - env -fi - # Look in package.json's engines.node field for a semver range semver_range=$(cat $build_dir/package.json | $bp_dir/vendor/jq -r .engines.node) @@ -81,9 +69,17 @@ elif test -d $cache_dir/node_modules; then npm prune 2>&1 | indent fi -# Make npm output to STDOUT instead of its default STDERR -status "Installing dependencies" -npm install --userconfig $build_dir/.npmrc --production 2>&1 | indent +# Scope config var availability only to `npm install` +( + if [ "$env_file" ]; then + status "Importing application config vars" + export $(egrep -v "^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH)" "$env_file") + fi + + status "Installing dependencies" + # Make npm output to STDOUT instead of its default STDERR + npm install --userconfig $build_dir/.npmrc --production 2>&1 | indent +) status "Caching node_modules directory for future builds" rm -rf $cache_dir diff --git a/bin/test b/bin/test index 996e33bdc..a4e9d9794 100755 --- a/bin/test +++ b/bin/test @@ -104,6 +104,23 @@ testProcfileAbsentNpmStartAbsent() { assertCapturedSuccess } +testProcfileAbsentNpmStartPresent() { + compile "procfile-absent-npm-start-present" + assertCaptured "No Procfile found; Adding npm start to new Procfile" + assertFile "web: npm start" "Procfile" + assertCapturedSuccess +} + +testEnvfileNotImported() { + compile "stable-node" + assertNotCaptured "Importing application config vars" +} + +testEnvfileImported() { + compileWithEnv "stable-node" + assertCaptured "Importing application config vars" +} + # Pending Tests # testNodeBinariesAddedToPath() { @@ -132,6 +149,8 @@ testProcfileAbsentNpmStartAbsent() { # assertCapturedError 1 "not found among available versions" # } + + # Utils pushd $(dirname 0) >/dev/null @@ -159,6 +178,13 @@ compile() { capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} } +compileWithEnv() { + compile_dir=$(mktmpdir) + env_file="FOO=1;BAR=2;" + cp -r ${bp_dir}/test/$1/. ${compile_dir} + capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir) $env_file} +} + assertFile() { assertEquals "$1" "$(cat ${compile_dir}/$2)" } From 8b280a265627634237d44fff9a03e85be8b3b05f Mon Sep 17 00:00:00 2001 From: zeke Date: Fri, 20 Dec 2013 11:52:16 -0800 Subject: [PATCH 09/12] Fix the new envfile tests. Thanks @tt --- bin/test | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/bin/test b/bin/test index 370799b68..53c96f812 100755 --- a/bin/test +++ b/bin/test @@ -114,11 +114,15 @@ testProcfileAbsentNpmStartPresent() { testEnvfileNotImported() { compile "stable-node" assertNotCaptured "Importing application config vars" + assertCapturedSuccess } testEnvfileImported() { - compileWithEnv "stable-node" + env_file=$(mktemp) + echo "FOO=1" > $env_file + compile "stable-node" "$(mktmpdir)" $env_file assertCaptured "Importing application config vars" + assertCapturedSuccess } # Utils @@ -145,14 +149,7 @@ compile_dir="" compile() { compile_dir=$(mktmpdir) cp -r ${bp_dir}/test/$1/. ${compile_dir} - capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} -} - -compileWithEnv() { - compile_dir=$(mktmpdir) - env_file="FOO=1;BAR=2;" - cp -r ${bp_dir}/test/$1/. ${compile_dir} - capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir) $env_file} + capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3 } assertFile() { From 36ad8a3bea11105c7cf150fcdda6cee004482927 Mon Sep 17 00:00:00 2001 From: zeke Date: Tue, 21 Jan 2014 11:54:09 -0800 Subject: [PATCH 10/12] update compile script to work with new buildpack-env-dir API --- bin/common.sh | 13 +++++++++++++ bin/compile | 8 ++++---- bin/test | 11 ++++++----- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/bin/common.sh b/bin/common.sh index 49b790a55..4a9d17e1d 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -28,3 +28,16 @@ indent() { cat_npm_debug_log() { test -f $build_dir/npm-debug.log && cat $build_dir/npm-debug.log } + +export_env_dir() { + env_dir=$1 + whitelist_regex=${2:-''} + blacklist_regex=${3:-'^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH)$'} + if [ -d "$env_dir" ]; then + for e in $(ls $env_dir); do + echo "$e" | grep -E "$whitelist_regex" | grep -qvE "$blacklist_regex" && + export "$e=$(cat $env_dir/$e)" + : + done + fi +} diff --git a/bin/compile b/bin/compile index 9a2f20830..816c6c91b 100755 --- a/bin/compile +++ b/bin/compile @@ -7,7 +7,7 @@ set -o pipefail # don't ignore exit codes when piping output # Configure directories build_dir=$1 cache_dir=$2 -env_file=$3 +env_dir=$3 bp_dir=$(cd $(dirname $0); cd ..; pwd) @@ -71,9 +71,9 @@ fi # Scope config var availability only to `npm install` ( - if [ "$env_file" ]; then - status "Importing application config vars" - export $(egrep -v "^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH)" "$env_file") + if [ -d "$env_dir" ]; then + status "Exporting config vars to environment" + export_env_dir $env_dir fi status "Installing dependencies" diff --git a/bin/test b/bin/test index e623bb35a..977d1ee29 100755 --- a/bin/test +++ b/bin/test @@ -117,11 +117,12 @@ testEnvfileNotImported() { assertCapturedSuccess } -testEnvfileImported() { - env_file=$(mktemp) - echo "FOO=1" > $env_file - compile "stable-node" "$(mktmpdir)" $env_file - assertCaptured "Importing application config vars" +testEnvDirExported() { + env_dir=$(mktmpdir) + echo "chicken" > $env_dir/birds + echo "koi" > $env_dir/fish + compile "stable-node" "$(mktmpdir)" $env_dir + assertCaptured "Exporting config vars to environment" assertCapturedSuccess } From 9e1e554b60f62a0b4765150a96ca85f92ff54ee5 Mon Sep 17 00:00:00 2001 From: zeke Date: Tue, 21 Jan 2014 12:05:35 -0800 Subject: [PATCH 11/12] update buildpack-env-dir tests --- bin/test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/test b/bin/test index 977d1ee29..b2cafc9ff 100755 --- a/bin/test +++ b/bin/test @@ -111,9 +111,9 @@ testProcfileAbsentNpmStartPresent() { assertCapturedSuccess } -testEnvfileNotImported() { +testEnvDirNotImported() { compile "stable-node" - assertNotCaptured "Importing application config vars" + assertNotCaptured "Exporting config vars to environment" assertCapturedSuccess } From 1e46a9b9af70d562b4777740a1681de50ff5df23 Mon Sep 17 00:00:00 2001 From: zeke Date: Tue, 21 Jan 2014 13:22:10 -0800 Subject: [PATCH 12/12] Cache everything one level deeper (inside $cache_dir/node) to avoid clobbering cached artifacts from other buildpacks. Store node-version textfile in new .heroku directory, in the slug _and_ the cache. --- bin/compile | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/bin/compile b/bin/compile index 9a2f20830..177c5bc13 100755 --- a/bin/compile +++ b/bin/compile @@ -61,12 +61,18 @@ if test -d $build_dir/node_modules; then status "Found existing node_modules directory; skipping cache" status "Rebuilding any native dependencies" npm rebuild 2>&1 | indent -elif test -d $cache_dir/node_modules; then +elif test -d $cache_dir/node/node_modules; then status "Restoring node_modules directory from cache" - cp -r $cache_dir/node_modules $build_dir/ + cp -r $cache_dir/node/node_modules $build_dir/ status "Pruning cached dependencies not specified in package.json" npm prune 2>&1 | indent + + if test -f $cache_dir/node/.heroku/node-version && [ $(cat $cache_dir/node/.heroku/node-version) != "$node_version" ]; then + status "Node version changed since last build; rebuilding dependencies" + npm rebuild 2>&1 | indent + fi + fi # Scope config var availability only to `npm install` @@ -81,10 +87,26 @@ fi npm install --userconfig $build_dir/.npmrc --production 2>&1 | indent ) -status "Caching node_modules directory for future builds" -rm -rf $cache_dir/node_modules -mkdir -p $cache_dir -test -d $build_dir/node_modules && cp -r $build_dir/node_modules $cache_dir/ +# Persist goodies like node-version in the slug +mkdir -p $build_dir/.heroku + +# Save resolved node version in the slug for later reference +echo $node_version > $build_dir/.heroku/node-version + +# Purge node-related cached content, being careful not to purge the top-level +# cache, for the sake of heroku-buildpack-multi apps. +rm -rf $cache_dir/node_modules # (for apps still on the older caching strategy) +rm -rf $cache_dir/node +mkdir -p $cache_dir/node + +# If app has a node_modules directory, cache it. +if test -d $build_dir/node_modules; then + status "Caching node_modules directory for future builds" + cp -r $build_dir/node_modules $cache_dir/node +fi + +# Copy goodies to the cache +cp -r $build_dir/.heroku $cache_dir/node status "Cleaning up node-gyp and npm artifacts" rm -rf "$build_dir/.node-gyp"