From 338bbde10516705f07293b9aaf0c87b3ba6d4309 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Wed, 25 May 2022 16:02:49 -0400 Subject: [PATCH 01/18] Don't asset_checksum on client .env secrets --- bin/asset_checksum | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/bin/asset_checksum b/bin/asset_checksum index e9bd2ce847..976c5df008 100755 --- a/bin/asset_checksum +++ b/bin/asset_checksum @@ -10,14 +10,6 @@ if [ "$ASSETS_PREFIX" = "" ]; then export ASSETS_PREFIX=qa-warehouse-staging-ecs fi -# Get client secrets, store in separate secrets file for hashing later. -if [ "$SECRET_ARN" != "" ]; then - bin/download_secrets.rb > .env.secret -else - echo 'No $SECRET_ARN!' - exit 1 -fi - # Pull down our client theme files. ASSETS_BUCKET_NAME=openpath-ecs-assets UPDATE_ONLY=false ./bin/sync_app_assets.rb > /dev/null 2>&1 @@ -33,4 +25,4 @@ tar -cf - --sort=name `# Force sorting` \ --no-acls `# No ACLs` \ --mode="go-rwx,u-rw" `# Normalize permissions` \ --exclude=.keep,*.rbc,**.orig,*.swp `# Exclude a few files we know we don't care about` \ - app/assets .env.secret | md5sum | cut -d ' ' -f 1 + app/assets | md5sum | cut -d ' ' -f 1 From 4bd52cbdc2cfdcf5f37e2208079f631b6765e5bb Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Wed, 25 May 2022 16:03:32 -0400 Subject: [PATCH 02/18] Move downloading secrets to lib --- config/deploy/docker/assets/download_secrets.rb | 16 ++-------------- config/deploy/docker/lib/aws_sdk_helpers.rb | 11 +++++++++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/config/deploy/docker/assets/download_secrets.rb b/config/deploy/docker/assets/download_secrets.rb index dcfca8a5a2..914565be84 100755 --- a/config/deploy/docker/assets/download_secrets.rb +++ b/config/deploy/docker/assets/download_secrets.rb @@ -1,16 +1,4 @@ #!/usr/bin/env ruby +require_relative '../config/deploy/docker/lib/aws_sdk_helpers' -begin - require 'aws-sdk-secretsmanager' - - define_method(:client) { Aws::SecretsManager::Client.new } - - resp = client.get_secret_value( - secret_id: ENV.fetch('SECRET_ARN') - ) - - puts resp.to_h[:secret_string] -rescue Aws::Errors::MissingCredentialsError - STDERR.puts "Need credentials to sync secrets (or run on a server/container with a role)" - puts "ERROR=secretsyncfailed" -end +puts AwsSdkHelpers::Helpers.get_secret(ENV.fetch('SECRET_ARN')) diff --git a/config/deploy/docker/lib/aws_sdk_helpers.rb b/config/deploy/docker/lib/aws_sdk_helpers.rb index 83188d1457..ac9b281221 100644 --- a/config/deploy/docker/lib/aws_sdk_helpers.rb +++ b/config/deploy/docker/lib/aws_sdk_helpers.rb @@ -111,5 +111,16 @@ def _spot_capacity_provider_name def _on_demand_capacity_provider_name @_on_demand_capacity_provider_name ||= AwsSdkHelpers::Helpers.get_capacity_provider_name('OnDemand') end + + def self.get_secret(secret_arn) + resp = AwsSdkHelpers::ClientMethods.secretsmanager.get_secret_value( + secret_id: secret_arn, + ) + + return resp.to_h[:secret_string] + rescue Aws::Errors::MissingCredentialsError + warn 'Need credentials to sync secrets (or run on a server/container with a role)' + puts 'ERROR=secretsyncfailed' + end end end From be2064031cb94822a6d3be08c3668ed723996cc3 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Wed, 25 May 2022 16:03:50 -0400 Subject: [PATCH 03/18] Pull down remote secrets.yml --- config/deploy/docker/assets/entrypoint.sh | 6 +++--- config/deploy/docker/lib/command_args.rb | 24 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/config/deploy/docker/assets/entrypoint.sh b/config/deploy/docker/assets/entrypoint.sh index 62ca6e2956..8d6e45dcb7 100755 --- a/config/deploy/docker/assets/entrypoint.sh +++ b/config/deploy/docker/assets/entrypoint.sh @@ -30,13 +30,13 @@ echo 'Setting Timezone' cp /usr/share/zoneinfo/$TIMEZONE /app/etc-localtime echo $TIMEZONE > /etc/timezone -echo 'Syncing the assets from s3...' +echo 'Syncing the client assets from s3...' T1=`date +%s` ./bin/sync_app_assets.rb T2=`date +%s` echo "...sync_app_assets 1 took $(expr $T2 - $T1) seconds" -echo 'Pulling down assets from S3...' +echo 'Clobbering assets...' T1=`date +%s` bundle exec rake assets:clobber && mkdir -p ./public/assets T2=`date +%s` @@ -48,8 +48,8 @@ T2=`date +%s` echo "...checksumming took $(expr $T2 - $T1) seconds" echo "Using ASSET_CHECKSUM [${ASSET_CHECKSUM}]" +echo 'Pulling down the compiled assets...' # Using ASSETS_PREFIX from .env and ASSET_CHECKSUM from above. cd ./public/assets -# Pull down the compiled assets. Using ASSETS_PREFIX from .env and ASSET_CHECKSUM from above. bundle exec ../../bin/wait_for_compiled_assets.rb T1=`date +%s` ASSETS_PREFIX="${ASSETS_PREFIX}/${ASSET_CHECKSUM}" ASSETS_BUCKET_NAME=openpath-precompiled-assets UPDATE_ONLY=true ../../bin/sync_app_assets.rb diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 06a8524d81..a58eb58302 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -7,7 +7,29 @@ class CommandArgs def initialize path = Pathname.new(__FILE__).join('..', '..', 'assets', 'secret.deploy.values.yml') - config = YAML.load_file(path) + local_config = YAML.load_file(path) + remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) + remote_config = YAML.safe_load(remote_config_text) + + if local_config != remote_config + puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' + unsure = $stdin.readline + if unsure.chomp.downcase.match?(/y(es)?/) + File.write(path, remote_config_text) + config = remote_config + else + tmppath = "/tmp/remote-secrets-#{Time.now.to_i}.yml" + File.write(tmppath, remote_config_text) + puts "Okay. Remote config saved to #{tmppath} for your convenience. (Local is at #{path})" + exit + end + elsif remote_config.nil? + config = local_config + puts "[WARN] ❗ Remote secrets.yml not found, using local: #{path}" + else + config = remote_config + end + defaults = config['_global_defaults'] || {} config.each_key do |key| config.delete(key) if key.match?(/^_/) From cd20d6a88c48efa54bfcadc769cb944d66578586 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 14:54:56 -0400 Subject: [PATCH 04/18] fix path in download_secrets.rb? --- config/deploy/docker/assets/download_secrets.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/docker/assets/download_secrets.rb b/config/deploy/docker/assets/download_secrets.rb index 914565be84..e4d619c90f 100755 --- a/config/deploy/docker/assets/download_secrets.rb +++ b/config/deploy/docker/assets/download_secrets.rb @@ -1,4 +1,4 @@ #!/usr/bin/env ruby -require_relative '../config/deploy/docker/lib/aws_sdk_helpers' +require_relative '../lib/aws_sdk_helpers' puts AwsSdkHelpers::Helpers.get_secret(ENV.fetch('SECRET_ARN')) From cc0ce18905946bbcba1fcbd96efdff980893c863 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 15:06:16 -0400 Subject: [PATCH 05/18] SECRETS_YML_SECRET_ARN --- .github/workflows/asset_compilation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asset_compilation.yml b/.github/workflows/asset_compilation.yml index c14a4cd697..de2a816324 100644 --- a/.github/workflows/asset_compilation.yml +++ b/.github/workflows/asset_compilation.yml @@ -117,10 +117,10 @@ jobs: - name: 'App setup' env: - SECRETS_FILE_SECRET_ARN: ${{ secrets.SECRETS_FILE_SECRET_ARN }} + SECRETS_YML_SECRET_ARN: ${{ secrets.SECRETS_YML_SECRET_ARN }} run: | # cp config/database.yml.ci config/database.yml - SECRET_ARN=$SECRETS_FILE_SECRET_ARN bundle exec bin/download_secrets.rb > config/deploy/docker/assets/secret.deploy.values.yml + SECRET_ARN=$SECRETS_YML_SECRET_ARN bundle exec bin/download_secrets.rb > config/deploy/docker/assets/secret.deploy.values.yml - name: Generate checksum and compile env: From ff65dd1666b57023e6f0b0dc0b24b5c40e7908c8 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 15:33:33 -0400 Subject: [PATCH 06/18] shuffle stuff --- .github/workflows/asset_compilation.yml | 13 +++++++------ config/deploy/docker/lib/command_args.rb | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/asset_compilation.yml b/.github/workflows/asset_compilation.yml index de2a816324..4b8cce6eda 100644 --- a/.github/workflows/asset_compilation.yml +++ b/.github/workflows/asset_compilation.yml @@ -115,15 +115,16 @@ jobs: bundle config set --local without 'production staging development' bundle install --jobs 10 --retry 3 - - name: 'App setup' - env: - SECRETS_YML_SECRET_ARN: ${{ secrets.SECRETS_YML_SECRET_ARN }} - run: | - # cp config/database.yml.ci config/database.yml - SECRET_ARN=$SECRETS_YML_SECRET_ARN bundle exec bin/download_secrets.rb > config/deploy/docker/assets/secret.deploy.values.yml + # - name: 'App setup' + # env: + # SECRETS_YML_SECRET_ARN: ${{ secrets.SECRETS_YML_SECRET_ARN }} + # run: | + # # cp config/database.yml.ci config/database.yml + # SECRET_ARN=$SECRETS_YML_SECRET_ARN bundle exec bin/download_secrets.rb > config/deploy/docker/assets/secret.deploy.values.yml - name: Generate checksum and compile env: + SECRETS_YML_SECRET_ARN: ${{ secrets.SECRETS_YML_SECRET_ARN }} DATABASE_ADAPTER: postgresql DATABASE_HOST: postgres DATABASE_PASS: postgres diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index a58eb58302..e859adc572 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -11,7 +11,7 @@ def initialize remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) remote_config = YAML.safe_load(remote_config_text) - if local_config != remote_config + if local_config != remote_config && local_config.present? puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' unsure = $stdin.readline if unsure.chomp.downcase.match?(/y(es)?/) From 7647b9f5e060dfe7c8deaec7fbe23c6044aebecb Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 15:36:48 -0400 Subject: [PATCH 07/18] handle secrets yml path doesn't exist --- config/deploy/docker/lib/command_args.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index e859adc572..d32512e265 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -7,7 +7,7 @@ class CommandArgs def initialize path = Pathname.new(__FILE__).join('..', '..', 'assets', 'secret.deploy.values.yml') - local_config = YAML.load_file(path) + local_config = File.exist?(path) ? YAML.load_file(path) : false remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) remote_config = YAML.safe_load(remote_config_text) From 18dacff8d6e724994f96b0a80aca849b9290084e Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 15:52:00 -0400 Subject: [PATCH 08/18] debug --- config/deploy/docker/lib/command_args.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index d32512e265..1746b2f0c1 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -9,6 +9,14 @@ def initialize path = Pathname.new(__FILE__).join('..', '..', 'assets', 'secret.deploy.values.yml') local_config = File.exist?(path) ? YAML.load_file(path) : false remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) + + puts "remote_config_text is class #{remote_config_text.class}" + resp = AwsSdkHelpers::ClientMethods.secretsmanager.get_secret_value( + secret_id: ENV['SECRETS_YML_SECRET_ARN'], + ) + puts "resp keys: #{resp.to_h.keys}" + puts "resp[:secret_string] is class #{resp.to_h[:secret_string]}" + remote_config = YAML.safe_load(remote_config_text) if local_config != remote_config && local_config.present? From 2002b1ee0ca09149ce8ef4aefbeb4554fe59c57d Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 15:56:20 -0400 Subject: [PATCH 09/18] aha? --- config/deploy/docker/lib/command_args.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 1746b2f0c1..654263a4a3 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -10,14 +10,7 @@ def initialize local_config = File.exist?(path) ? YAML.load_file(path) : false remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) - puts "remote_config_text is class #{remote_config_text.class}" - resp = AwsSdkHelpers::ClientMethods.secretsmanager.get_secret_value( - secret_id: ENV['SECRETS_YML_SECRET_ARN'], - ) - puts "resp keys: #{resp.to_h.keys}" - puts "resp[:secret_string] is class #{resp.to_h[:secret_string]}" - - remote_config = YAML.safe_load(remote_config_text) + remote_config = YAML.safe_load(remote_config_text, [Symbol]) if local_config != remote_config && local_config.present? puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' From 1931af28be3abec62a85087700e4a1ec586fa382 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 16:01:35 -0400 Subject: [PATCH 10/18] yaml safe_load is evil --- config/deploy/docker/lib/command_args.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 654263a4a3..19c6d862c4 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -10,7 +10,7 @@ def initialize local_config = File.exist?(path) ? YAML.load_file(path) : false remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) - remote_config = YAML.safe_load(remote_config_text, [Symbol]) + remote_config = YAML.safe_load(remote_config_text, [Symbol], aliases: true) if local_config != remote_config && local_config.present? puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' From 803cc8234ecc6dadc606c4f04799bb6a3db8020c Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 16:50:26 -0400 Subject: [PATCH 11/18] why is dependency management in rails such a travesty --- .github/workflows/asset_compilation.yml | 7 ------- bin/download_secrets.rb | 5 ++++- .../docker/assets/Dockerfile.open-path-warehouse.base | 1 - config/deploy/docker/assets/download_secrets.rb | 4 ---- config/deploy/docker/lib/command_args.rb | 3 +++ 5 files changed, 7 insertions(+), 13 deletions(-) mode change 120000 => 100755 bin/download_secrets.rb delete mode 100755 config/deploy/docker/assets/download_secrets.rb diff --git a/.github/workflows/asset_compilation.yml b/.github/workflows/asset_compilation.yml index 4b8cce6eda..2b80b4c3d7 100644 --- a/.github/workflows/asset_compilation.yml +++ b/.github/workflows/asset_compilation.yml @@ -115,13 +115,6 @@ jobs: bundle config set --local without 'production staging development' bundle install --jobs 10 --retry 3 - # - name: 'App setup' - # env: - # SECRETS_YML_SECRET_ARN: ${{ secrets.SECRETS_YML_SECRET_ARN }} - # run: | - # # cp config/database.yml.ci config/database.yml - # SECRET_ARN=$SECRETS_YML_SECRET_ARN bundle exec bin/download_secrets.rb > config/deploy/docker/assets/secret.deploy.values.yml - - name: Generate checksum and compile env: SECRETS_YML_SECRET_ARN: ${{ secrets.SECRETS_YML_SECRET_ARN }} diff --git a/bin/download_secrets.rb b/bin/download_secrets.rb deleted file mode 120000 index 34cdfefd5b..0000000000 --- a/bin/download_secrets.rb +++ /dev/null @@ -1 +0,0 @@ -../config/deploy/docker/assets/download_secrets.rb \ No newline at end of file diff --git a/bin/download_secrets.rb b/bin/download_secrets.rb new file mode 100755 index 0000000000..914565be84 --- /dev/null +++ b/bin/download_secrets.rb @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +require_relative '../config/deploy/docker/lib/aws_sdk_helpers' + +puts AwsSdkHelpers::Helpers.get_secret(ENV.fetch('SECRET_ARN')) diff --git a/config/deploy/docker/assets/Dockerfile.open-path-warehouse.base b/config/deploy/docker/assets/Dockerfile.open-path-warehouse.base index 7c08eb2599..cd6b4fd176 100644 --- a/config/deploy/docker/assets/Dockerfile.open-path-warehouse.base +++ b/config/deploy/docker/assets/Dockerfile.open-path-warehouse.base @@ -14,7 +14,6 @@ WORKDIR /app COPY Gemfile Gemfile.lock Rakefile config.ru package.json ./ COPY bin ./bin COPY public ./public -COPY config/deploy/docker/assets/download_secrets.rb ./bin COPY config/deploy/docker/lib ./bin COPY config/deploy/docker/assets/deploy_tasks.open-path-warehouse.sh ./bin/deploy_tasks.sh diff --git a/config/deploy/docker/assets/download_secrets.rb b/config/deploy/docker/assets/download_secrets.rb deleted file mode 100755 index e4d619c90f..0000000000 --- a/config/deploy/docker/assets/download_secrets.rb +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env ruby -require_relative '../lib/aws_sdk_helpers' - -puts AwsSdkHelpers::Helpers.get_secret(ENV.fetch('SECRET_ARN')) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 19c6d862c4..993efdf6d7 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -1,11 +1,14 @@ # frozen_string_literal: true require 'yaml' +require 'dotenv' class CommandArgs attr_accessor :deployments def initialize + Dotenv.load('.env', '.env.local') + path = Pathname.new(__FILE__).join('..', '..', 'assets', 'secret.deploy.values.yml') local_config = File.exist?(path) ? YAML.load_file(path) : false remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) From a368dca7d9daedf07752e4d88cd6713baf84bc41 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 18:30:49 -0400 Subject: [PATCH 12/18] avoid calling .present? on false --- config/deploy/docker/lib/command_args.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 993efdf6d7..0b685c2504 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -15,7 +15,7 @@ def initialize remote_config = YAML.safe_load(remote_config_text, [Symbol], aliases: true) - if local_config != remote_config && local_config.present? + if local_config&.present? && local_config != remote_config puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' unsure = $stdin.readline if unsure.chomp.downcase.match?(/y(es)?/) From 328dc28869c234011214681a66d1f84044383394 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 18:49:46 -0400 Subject: [PATCH 13/18] .present? => .empty? because no active_support --- config/deploy/docker/lib/command_args.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 0b685c2504..387efde248 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -15,7 +15,7 @@ def initialize remote_config = YAML.safe_load(remote_config_text, [Symbol], aliases: true) - if local_config&.present? && local_config != remote_config + if local_config&.empty? && local_config != remote_config puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' unsure = $stdin.readline if unsure.chomp.downcase.match?(/y(es)?/) From 42db849d3bf5030b7231bbd5e25a69f24079e556 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 20:02:22 -0400 Subject: [PATCH 14/18] Refactor asset_compiler to be more efficient? --- config/deploy/docker/lib/asset_compiler.rb | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/config/deploy/docker/lib/asset_compiler.rb b/config/deploy/docker/lib/asset_compiler.rb index 50aa5878e1..aad59330f2 100644 --- a/config/deploy/docker/lib/asset_compiler.rb +++ b/config/deploy/docker/lib/asset_compiler.rb @@ -30,16 +30,6 @@ def self.compiled_assets_s3_path(target_group_name, checksum) end def run! - time_me name: 'Secrets download' do - system(`SECRET_ARN=#{@secret_arn.shellescape} bin/download_secrets.rb > .env`) - end - - Dotenv.load('.env') - - time_me name: 'Clobberin\'' do - system('source .env; rake assets:clobber') # TODO: don't call out to rake like this, it's inefficient - end - checksum = '' time_me name: 'Checksumming' do checksum = `SECRET_ARN=#{@secret_arn.shellescape} ASSETS_PREFIX=#{@target_group_name.shellescape} bin/asset_checksum`.split(' ')[-1] @@ -57,6 +47,12 @@ def run! return end + time_me name: 'Secrets download' do + system(`SECRET_ARN=#{@secret_arn.shellescape} bin/download_secrets.rb > .env`) + end + + Dotenv.load('.env') + time_me name: 'Compiling assets' do system('source .env; rake --quiet assets:precompile > /dev/null 2>&1') # TODO: don't call out to rake like this, it's inefficient end From 7461549ceca3b0d8b2f18010ad286abbba5b1bf5 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 20:07:36 -0400 Subject: [PATCH 15/18] fix command_args conditional thingy --- config/deploy/docker/lib/command_args.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index 387efde248..b987971a46 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -10,12 +10,18 @@ def initialize Dotenv.load('.env', '.env.local') path = Pathname.new(__FILE__).join('..', '..', 'assets', 'secret.deploy.values.yml') - local_config = File.exist?(path) ? YAML.load_file(path) : false - remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) + if File.exist?(path) + local_config = YAML.load_file(path) + local_config = nil if local_config.empty? + else + local_config = nil + end + + remote_config_text = AwsSdkHelpers::Helpers.get_secret(ENV['SECRETS_YML_SECRET_ARN']) remote_config = YAML.safe_load(remote_config_text, [Symbol], aliases: true) - if local_config&.empty? && local_config != remote_config + if !local_config.nil? && local_config != remote_config puts 'Local secrets.yml differs from remote config, would you like to pull down the remote version? This will overwrite your local file. [y/N]' unsure = $stdin.readline if unsure.chomp.downcase.match?(/y(es)?/) From 2e32e50d1dd87f52b497fdfbd0ee300642deba38 Mon Sep 17 00:00:00 2001 From: Andrew Zito Date: Thu, 26 May 2022 20:12:47 -0400 Subject: [PATCH 16/18] add some docs --- docs/asset_precompilation.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/asset_precompilation.md diff --git a/docs/asset_precompilation.md b/docs/asset_precompilation.md new file mode 100644 index 0000000000..ac6b9ac721 --- /dev/null +++ b/docs/asset_precompilation.md @@ -0,0 +1,30 @@ +## Asset Checksumming + +`bin/asset_checksum` + +In order to make it easy to tell if the assets need to be recompiled or if the cached versions can be used, we generate a checksum based on the content of `app/assets` (including client theme files). The goal is for the checksum to be guaranteed to change if a change is made to the source assets that will alter the compiled output, *and* guaranteed *not* to change otherwise. + +*Note: Because the checksum when stored in "cache" (S3) will be namespaced to client/environment, it is not necessary to hash any of the client ENV secrets (CLIENT or RAILS_ENV would have been the only ones affecting compiled output).* + +## The GitHub Actions Workflow + +`.github/workflows/asset_compilation.yml` + +This workflow iterates through every client (for both environments). It uses a [matrix](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) to iterate over a list of anonymized identifiers (`gha_staging_load_1`, `gha_production_load_3`), which are used to preserve client anonymity in an open source code base. Each identifier is passed to `bin/compile_assets.rb`. + +## The Asset Compiler + +`bin/compile_assets.rb` +`config/deploy/docker/lib/asset_compiler.rb` + +`bin/compile_assets.rb` is analogous to `bin/deploy.rb`. It uses `config/deploy/docker/lib/command_args.rb` to pull down the secrets.yml file from AWS Secrets Manager (the credentials are stored in the GitHub [repository secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) and are not loaded for non-internal workflow runs). It takes the anonymized identifier passed by the workflow and finds it in the list of client group identifiers at the bottom of the secrets.yml file, where it is mapped to real client identifying information. Currently each anonymized "group" has only 1 client, in order to maximize speed of the parallel asset compilation. The client information is passed to `AssetCompiler.run!` + +`config/deploy/docker/lib/asset_compiler.rb` is analogous to `config/deploy/docker/lib/deployer.rb`. It starts by performing an asset_checksum and checking if that checksum has been stored in S3 yet. If it has, we don't need to do anything else, since the compiled output of the source assets would match the stored compiled output. If the checksum hasn't been stored, then there has been a change to the assets. In this case we pull down the client ENV secrets to bootstrap the client environment, and run asset precompilation via rake. These assets are then uploaded to S3 under the client, environment, and checksum. + +## The Deployed Containers + +`config/deploy/docker/assets/entrypoint.sh` + +When a container spins up, the Docker entrypoint script generates an asset_checksum and pulls down the assets from S3. If the assets don't exist yet, the script will wait (`bin/wait_for_compiled_assets.rb`) and check again every 60 seconds. Note that this will happen in the deploy container before any of the application containers are spun up, meaning that you should be able to catch missing assets before the deploy goes through. + +**NOTE:** If you make a change in the remote client theme files, you will need to ensure that the GitHub Actions workflow runs at least once to pick up that change. Otherwise the checksum generated in the entrypoint won't match the last stored checksum and the waiting will never finish. From 5689e4b6417721318580fc29740ec1ffe1b5eb8d Mon Sep 17 00:00:00 2001 From: Elliot Anders Date: Fri, 27 May 2022 17:23:12 -0700 Subject: [PATCH 17/18] Allow for continuing with a local file after reviewing diff --- config/deploy/docker/lib/command_args.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/config/deploy/docker/lib/command_args.rb b/config/deploy/docker/lib/command_args.rb index b987971a46..72bb622e7f 100644 --- a/config/deploy/docker/lib/command_args.rb +++ b/config/deploy/docker/lib/command_args.rb @@ -31,7 +31,12 @@ def initialize tmppath = "/tmp/remote-secrets-#{Time.now.to_i}.yml" File.write(tmppath, remote_config_text) puts "Okay. Remote config saved to #{tmppath} for your convenience. (Local is at #{path})" - exit + pp `diff #{path} #{tmppath}` + puts 'continue anyway? [y/N]' + unsure = $stdin.readline + exit unless unsure.chomp.downcase.match?(/y(es)?/) + config = local_config + puts "[WARN] ❗ Using local config: #{path}" end elsif remote_config.nil? config = local_config From 787bc1a474c43a9e80b94e8d1f1ea949170b8454 Mon Sep 17 00:00:00 2001 From: Elliot Anders Date: Fri, 27 May 2022 17:26:15 -0700 Subject: [PATCH 18/18] Allow for continuing with a local file after reviewing diff --- config/brakeman.ignore | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/config/brakeman.ignore b/config/brakeman.ignore index d80784484f..c763456540 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -1,5 +1,25 @@ { "ignored_warnings": [ + { + "warning_type": "Command Injection", + "warning_code": 14, + "fingerprint": "023c724508b4c341ebb8aae1c407e91e113c05bccffe0dca58702f276fbe1dd0", + "check_name": "Execute", + "message": "Possible command injection", + "file": "config/deploy/docker/lib/command_args.rb", + "line": 34, + "link": "https://brakemanscanner.org/docs/warning_types/command_injection/", + "code": "`diff #{path} #{tmppath}`", + "render_path": null, + "location": { + "type": "method", + "class": "CommandArgs", + "method": "initialize" + }, + "user_input": "path", + "confidence": "Medium", + "note": "" + }, { "warning_type": "File Access", "warning_code": 16, @@ -254,7 +274,7 @@ { "type": "template", "name": "homeless_summary_report/warehouse_reports/reports/show", - "line": 25, + "line": 27, "file": "drivers/homeless_summary_report/app/views/homeless_summary_report/warehouse_reports/reports/show.haml", "rendered": { "name": "homeless_summary_report/warehouse_reports/reports/_counts_table", @@ -1821,7 +1841,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/homeless_summary_report/app/models/homeless_summary_report/report.rb", - "line": 1035, + "line": 986, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "clients.send(variant).send(\"spm_#{field}\").average(\"spm_#{field}\")", "render_path": null, @@ -2564,6 +2584,6 @@ "note": "" } ], - "updated": "2022-05-14 15:14:48 +0000", + "updated": "2022-05-28 00:26:04 +0000", "brakeman_version": "5.0.4" }