From a2855efe40a680c854e2806ced207f493e0acbea Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Mon, 10 Jun 2019 18:42:51 +0530 Subject: [PATCH 1/8] Add a 'backend' property to the configuaration.rb --- lib/gemstash/configuration.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gemstash/configuration.rb b/lib/gemstash/configuration.rb index 0ec7631f..dcf512e7 100644 --- a/lib/gemstash/configuration.rb +++ b/lib/gemstash/configuration.rb @@ -19,7 +19,8 @@ class Configuration puma_threads: 16, puma_workers: 1, cache_expiration: 30 * 60, - cache_max_size: 500 + cache_max_size: 500, + backend: "local" }.freeze DEFAULT_FILE = File.expand_path("~/.gemstash/config.yml").freeze From e13ba50616a8dabac3907574df0918f0f8a257cc Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Mon, 17 Jun 2019 19:05:24 +0530 Subject: [PATCH 2/8] Implement cli option for the backend property --- lib/gemstash/cli/setup.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/gemstash/cli/setup.rb b/lib/gemstash/cli/setup.rb index 5d703bdd..1b277936 100644 --- a/lib/gemstash/cli/setup.rb +++ b/lib/gemstash/cli/setup.rb @@ -26,6 +26,7 @@ def run ask_database ask_protected_fetch ask_timeout + ask_bachend check_cache check_storage check_database @@ -122,6 +123,11 @@ def ask_timeout @config[:fetch_timeout] = timeout.to_i end + def ask_bachend + say_current_config(:backend, "Current backend") + @config[:backend] = ask_with_default("What backend?", %w[local s3], "local") + end + def check_cache try("cache") { gemstash_env.cache_client.alive! } end From b1ca7040163e8d765b85d9f6ad3385016a2e8d36 Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Sun, 30 Jun 2019 22:52:52 +0530 Subject: [PATCH 3/8] Fix 'bachend' typo in setup.rb --- lib/gemstash/cli/setup.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gemstash/cli/setup.rb b/lib/gemstash/cli/setup.rb index 1b277936..db374de8 100644 --- a/lib/gemstash/cli/setup.rb +++ b/lib/gemstash/cli/setup.rb @@ -26,7 +26,7 @@ def run ask_database ask_protected_fetch ask_timeout - ask_bachend + ask_backend check_cache check_storage check_database @@ -123,7 +123,7 @@ def ask_timeout @config[:fetch_timeout] = timeout.to_i end - def ask_bachend + def ask_backend say_current_config(:backend, "Current backend") @config[:backend] = ask_with_default("What backend?", %w[local s3], "local") end From 80033ecf427b1115d25cf67fd1a7764b37440fbb Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Mon, 8 Jul 2019 00:36:21 +0530 Subject: [PATCH 4/8] Add aws-sdk to the Gemfile --- Gemfile | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 Gemfile diff --git a/Gemfile b/Gemfile old mode 100644 new mode 100755 index 1cf516df..f8819dcc --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,8 @@ source "https://rubygems.org" +gem "aws-sdk" + # Declare Ruby version so that activesupport gem can notice that ruby RUBY_VERSION From dc80fb136de22b811b65e8e01a6783bb969d3f85 Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Mon, 8 Jul 2019 10:05:17 +0530 Subject: [PATCH 5/8] Implement authorization trigger for AWS S3 --- lib/gemstash/cli/setup.rb | 14 ++++++++++++++ lib/gemstash/configuration.rb | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/gemstash/cli/setup.rb b/lib/gemstash/cli/setup.rb index db374de8..86fc37ba 100644 --- a/lib/gemstash/cli/setup.rb +++ b/lib/gemstash/cli/setup.rb @@ -3,6 +3,7 @@ require "gemstash" require "fileutils" require "yaml" +require "aws-sdk" module Gemstash class CLI @@ -126,6 +127,19 @@ def ask_timeout def ask_backend say_current_config(:backend, "Current backend") @config[:backend] = ask_with_default("What backend?", %w[local s3], "local") + check_aws_authorization() unless @config[:backend] == "local" + end + + def check_aws_authorization + begin + @cli.say "Checking authorization to Aws" + s3 = Aws::S3::Resource.new(region: 'us-west-2') + @config[:s3_metadata] = { authorized: true }.merge(Gemstash::Configuration::DEFAULTS[:s3_metadata]) + rescue Aws::Sigv4::Errors::MissingCredentialsError => e + @cli.say "Aws authorization failed: #{e.message}" + @cli.say "Please refer to https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html" + @config[:s3_metadata] = { authorized: false }.merge(Gemstash::Configuration::DEFAULTS[:s3_metadata]) + end end def check_cache diff --git a/lib/gemstash/configuration.rb b/lib/gemstash/configuration.rb index dcf512e7..47ee566c 100644 --- a/lib/gemstash/configuration.rb +++ b/lib/gemstash/configuration.rb @@ -20,7 +20,8 @@ class Configuration puma_workers: 1, cache_expiration: 30 * 60, cache_max_size: 500, - backend: "local" + backend: "local", + s3_metadata: { authorized: false, bucket_name: nil } }.freeze DEFAULT_FILE = File.expand_path("~/.gemstash/config.yml").freeze From dc42915090ee679666794cd924e4483f4b55202a Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Mon, 8 Jul 2019 10:45:53 +0530 Subject: [PATCH 6/8] Implement first version of ask_s3_metadata method --- lib/gemstash/cli/setup.rb | 19 ++++++++++++++++--- lib/gemstash/configuration.rb | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/gemstash/cli/setup.rb b/lib/gemstash/cli/setup.rb index 86fc37ba..4fce8e88 100644 --- a/lib/gemstash/cli/setup.rb +++ b/lib/gemstash/cli/setup.rb @@ -126,20 +126,33 @@ def ask_timeout def ask_backend say_current_config(:backend, "Current backend") + previous_backend = @config[:backend] @config[:backend] = ask_with_default("What backend?", %w[local s3], "local") - check_aws_authorization() unless @config[:backend] == "local" + check_aws_authorization(previous_backend) unless @config[:backend] == "local" end - def check_aws_authorization + def check_aws_authorization(previous_backend) begin @cli.say "Checking authorization to Aws" - s3 = Aws::S3::Resource.new(region: 'us-west-2') + s3 = Aws::S3::Resource.new() @config[:s3_metadata] = { authorized: true }.merge(Gemstash::Configuration::DEFAULTS[:s3_metadata]) rescue Aws::Sigv4::Errors::MissingCredentialsError => e @cli.say "Aws authorization failed: #{e.message}" @cli.say "Please refer to https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html" + @config[:backend] = previous_backend @config[:s3_metadata] = { authorized: false }.merge(Gemstash::Configuration::DEFAULTS[:s3_metadata]) end + ask_s3_bucket_metadata() unless @config[:s3_metadata][:authorized] == false + end + + def ask_s3_bucket_metadata + bucket_name = @cli.ask "What is the target S3 bucket?" + bucket = Aws::S3::Bucket.new(bucket_name) + if bucket.exists?() + @config[:s3_metadata] = { bucket_name: bucket_name }.merge(@config[:s3_metadata]) + else + @cli.say "S3 bucket: #{bucket_name} does not exist!" + end end def check_cache diff --git a/lib/gemstash/configuration.rb b/lib/gemstash/configuration.rb index 47ee566c..6807617f 100644 --- a/lib/gemstash/configuration.rb +++ b/lib/gemstash/configuration.rb @@ -21,7 +21,7 @@ class Configuration cache_expiration: 30 * 60, cache_max_size: 500, backend: "local", - s3_metadata: { authorized: false, bucket_name: nil } + s3_metadata: {} }.freeze DEFAULT_FILE = File.expand_path("~/.gemstash/config.yml").freeze From c5f855bdea4efd8ce9dcd61fdafa6387249700c8 Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Mon, 8 Jul 2019 12:48:01 +0530 Subject: [PATCH 7/8] Implement support for creating a new S3 bucket A new S3 bucket will be created when the specified bucket is not owned. --- lib/gemstash/cli/setup.rb | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/gemstash/cli/setup.rb b/lib/gemstash/cli/setup.rb index 4fce8e88..c3f3acac 100644 --- a/lib/gemstash/cli/setup.rb +++ b/lib/gemstash/cli/setup.rb @@ -126,12 +126,11 @@ def ask_timeout def ask_backend say_current_config(:backend, "Current backend") - previous_backend = @config[:backend] @config[:backend] = ask_with_default("What backend?", %w[local s3], "local") - check_aws_authorization(previous_backend) unless @config[:backend] == "local" + check_aws_authorization() unless @config[:backend] == "local" end - def check_aws_authorization(previous_backend) + def check_aws_authorization begin @cli.say "Checking authorization to Aws" s3 = Aws::S3::Resource.new() @@ -139,19 +138,26 @@ def check_aws_authorization(previous_backend) rescue Aws::Sigv4::Errors::MissingCredentialsError => e @cli.say "Aws authorization failed: #{e.message}" @cli.say "Please refer to https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html" - @config[:backend] = previous_backend @config[:s3_metadata] = { authorized: false }.merge(Gemstash::Configuration::DEFAULTS[:s3_metadata]) + @cli.say "Falling back to default backend: #{Gemstash::Configuration::DEFAULTS[:backend]}" + @config[:backend] = Gemstash::Configuration::DEFAULTS[:backend] end ask_s3_bucket_metadata() unless @config[:s3_metadata][:authorized] == false end def ask_s3_bucket_metadata bucket_name = @cli.ask "What is the target S3 bucket?" - bucket = Aws::S3::Bucket.new(bucket_name) - if bucket.exists?() + begin + s3 = Aws::S3::Client.new() + s3.create_bucket(bucket: bucket_name) + @config[:s3_metadata] = { bucket_name: bucket_name }.merge(@config[:s3_metadata]) + rescue Aws::S3::Errors::BucketAlreadyExists => e + @cli.say "S3 bucket creation failed: #{e.message}" + @cli.say "Falling back to default backend: #{Gemstash::Configuration::DEFAULTS[:backend]}" + @config[:backend] = Gemstash::Configuration::DEFAULTS[:backend] + rescue Aws::S3::Errors::BucketAlreadyOwnedByYou => e + @cli.say "Note: #{e.message}" @config[:s3_metadata] = { bucket_name: bucket_name }.merge(@config[:s3_metadata]) - else - @cli.say "S3 bucket: #{bucket_name} does not exist!" end end From 1dae69a93f84435d59dc7f7f1fa7e4f62668d0c8 Mon Sep 17 00:00:00 2001 From: Gayashan Bombuwala Date: Tue, 9 Jul 2019 14:12:22 +0530 Subject: [PATCH 8/8] Fix style issues that cause the build failure --- lib/gemstash/cli/setup.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/gemstash/cli/setup.rb b/lib/gemstash/cli/setup.rb index c3f3acac..020cb90f 100644 --- a/lib/gemstash/cli/setup.rb +++ b/lib/gemstash/cli/setup.rb @@ -127,13 +127,13 @@ def ask_timeout def ask_backend say_current_config(:backend, "Current backend") @config[:backend] = ask_with_default("What backend?", %w[local s3], "local") - check_aws_authorization() unless @config[:backend] == "local" + check_aws_authorization unless @config[:backend] == "local" end def check_aws_authorization begin @cli.say "Checking authorization to Aws" - s3 = Aws::S3::Resource.new() + Aws::S3::Resource.new @config[:s3_metadata] = { authorized: true }.merge(Gemstash::Configuration::DEFAULTS[:s3_metadata]) rescue Aws::Sigv4::Errors::MissingCredentialsError => e @cli.say "Aws authorization failed: #{e.message}" @@ -142,14 +142,14 @@ def check_aws_authorization @cli.say "Falling back to default backend: #{Gemstash::Configuration::DEFAULTS[:backend]}" @config[:backend] = Gemstash::Configuration::DEFAULTS[:backend] end - ask_s3_bucket_metadata() unless @config[:s3_metadata][:authorized] == false + ask_s3_bucket_metadata unless @config[:s3_metadata][:authorized] == false end def ask_s3_bucket_metadata bucket_name = @cli.ask "What is the target S3 bucket?" begin - s3 = Aws::S3::Client.new() - s3.create_bucket(bucket: bucket_name) + s3_client = Aws::S3::Client.new + s3_client.create_bucket(bucket: bucket_name) @config[:s3_metadata] = { bucket_name: bucket_name }.merge(@config[:s3_metadata]) rescue Aws::S3::Errors::BucketAlreadyExists => e @cli.say "S3 bucket creation failed: #{e.message}"