Introduce Amazon ObjectStorageManager (S3)

Basic stubs and test for management of Amazon S3

app/models/manageiq/providers/amazon/cloud_manager.rb

@@ -23,6 +23,12 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana
 
   include ManageIQ::Providers::Amazon::ManagerMixin
 
+  has_many :storage_managers,
+           :foreign_key => :parent_ems_id,
+           :class_name  => "ManageIQ::Providers::StorageManager",
+           :autosave    => true,
+           :dependent   => :destroy
+
   has_one :network_manager,
           :foreign_key => :parent_ems_id,
           :class_name  => "ManageIQ::Providers::Amazon::NetworkManager",
@@ -41,7 +47,19 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana
            :to        => :network_manager,
            :allow_nil => true
 
-  before_create :ensure_managers
+  has_one :s3_manager,
+          :foreign_key => :parent_ems_id,
+          :class_name  => "ManageIQ::Providers::Amazon::ObjectStorageManager",
+          :autosave    => true,
+          :dependent   => :destroy
+
+  delegate :cloud_object_store_containers,
+             :cloud_object_store_objects,
+             :to        => :s3_manager,
+             :allow_nil => true
+
+  before_create :ensure_managers,
+                :ensure_s3_manager
 
   supports :provisioning
   supports :regions
@@ -54,6 +72,13 @@ def ensure_managers
     network_manager.provider_region = provider_region
   end
 
+  def ensure_s3_manager
+    build_s3_manager unless s3_manager
+    s3_manager.name            = "#{name} S3 Manager"
+    s3_manager.zone_id         = zone_id
+    s3_manager.provider_region = provider_region
+  end
+
   def self.ems_type
     @ems_type ||= "ec2".freeze
   end

app/models/manageiq/providers/amazon/object_storage_manager.rb

@@ -0,0 +1,37 @@
+class ManageIQ::Providers::Amazon::ObjectStorageManager < ManageIQ::Providers::ObjectStorageManager
+  require_nested :RefreshParser
+  require_nested :RefreshWorker
+  require_nested :Refresher
+
+  delegate :authentication_check,
+           :authentication_status,
+           :authentications,
+           :authentication_for_summary,
+           :zone,
+           :connect,
+           :verify_credentials,
+           :with_provider_connection,
+           :address,
+           :ip_address,
+           :hostname,
+           :default_endpoint,
+           :endpoints,
+           :to        => :parent_manager,
+           :allow_nil => true
+
+  def self.ems_type
+    @ems_type ||= "s3_bucket".freeze
+  end
+
+  def self.description
+    @description ||= "Amazon S3 Bucket".freeze
+  end
+
+  def description
+    @description ||= "Amazon S3 Bucket".freeze
+  end
+
+  def self.hostname_required?
+    false
+  end
+end

app/models/manageiq/providers/amazon/object_storage_manager/refresh_parser.rb

@@ -0,0 +1,53 @@
+class ManageIQ::Providers::Amazon::ObjectStorageManager::RefreshParser
+  include ManageIQ::Providers::Amazon::RefreshHelperMethods
+
+  def initialize(ems, options = nil)
+    @ems        = ems
+    @connection = ems.connect(:service => :S3)
+    @data       = {}
+    @data_index = {}
+    @options    = options || {}
+  end
+
+  def ems_inv_to_hashes
+    log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]"
+
+    $aws_log.info("#{log_header}...")
+    object_store
+
+    $aws_log.info("#{log_header}...Complete")
+
+    @data
+  end
+
+  def object_store
+    # TODO: change
+    buckets = @connection.client.list_buckets.buckets
+    process_collection(buckets, :cloud_object_store_containers) { |c| parse_container(c) }
+  end
+
+  def parse_container(bucket)
+    uid = bucket.name
+
+    new_result = {
+      :ems_ref     => uid,
+      :key         => bucket.name
+    }
+    return uid, new_result
+  end
+
+  def parse_object(obj, bucket)
+    uid = obj.key
+
+    new_result = {
+      :ems_ref        => uid,
+      :etag           => obj.etag,
+      :last_modified  => obj.last_modified,
+      :content_length => obj.size,
+      :key            => obj.key,
+      #:content_type   => obj.content_type,
+      :container      => bucket
+    }
+    return uid, new_result
+  end
+end

app/models/manageiq/providers/amazon/object_storage_manager/refresh_worker.rb

@@ -0,0 +1,11 @@
+class ManageIQ::Providers::Amazon::ObjectStorageManager::RefreshWorker < ::MiqEmsRefreshWorker
+  require_nested :Runner
+
+  def self.ems_class
+    ManageIQ::Providers::Amazon::ObjectStorageManager
+  end
+
+  def self.settings_name
+    :ems_refresh_worker_amazon_s3
+  end
+end

app/models/manageiq/providers/amazon/object_storage_manager/refresh_worker/runner.rb

@@ -0,0 +1,3 @@
+class ManageIQ::Providers::Amazon::ObjectStorageManager::RefreshWorker::Runner <
+  ManageIQ::Providers::BaseManager::RefreshWorker::Runner
+end

app/models/manageiq/providers/amazon/object_storage_manager/refresher.rb

@@ -0,0 +1,8 @@
+class ManageIQ::Providers::Amazon::ObjectStorageManager::Refresher <
+  ManageIQ::Providers::BaseManager::Refresher
+  include ::EmsRefresh::Refreshers::EmsRefresherMixin
+
+  def parse_legacy_inventory(ems)
+    ManageIQ::Providers::Amazon::ObjectStorageManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+  end
+end

spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb

@@ -33,7 +33,7 @@ def expected_table_counts
       :cloud_subnet                  => 10,
       :custom_attribute              => 0,
       :disk                          => 11,
-      :ext_management_system         => 2,
+      :ext_management_system         => 3,
       :firewall_rule                 => 101,
       :flavor                        => 56,
       :floating_ip                   => 13,

spec/models/manageiq/providers/amazon/aws_stubs.rb

@@ -30,6 +30,7 @@ def test_counts(scaling = nil)
       :security_group_count                            => scaling * 20,
       :inbound_firewall_rule_per_security_group_count  => scaling * 5,
       :outbound_firewall_rule_per_security_group_count => scaling * 5,
+      :s3_buckets_count                                => scaling * 5
     }
   end
 
@@ -344,6 +345,17 @@ def mocked_load_balancers
     mocked_lbs
   end
 
+  def mocked_s3_buckets
+    mocked_s3_buckets = []
+    test_counts[:s3_buckets_count].times do |i|
+      mocked_s3_buckets << {
+        :name          => "bucket_id_#{i}",
+        :creation_date => Time.now
+      }
+    end
+    mocked_s3_buckets
+  end
+
   def mocked_instance_health
     mocked_instance_healths = []
     expected_table_counts[:load_balancer_pool_member].times do |i|

spec/models/manageiq/providers/amazon/cloud_manager/refresher_other_region_spec.rb

@@ -34,7 +34,7 @@
   end
 
   def assert_table_counts
-    expect(ExtManagementSystem.count).to eq(2)
+    expect(ExtManagementSystem.count).to eq(3)
     expect(Flavor.count).to eq(56)
     expect(AvailabilityZone.count).to eq(3)
     expect(FloatingIp.count).to eq(3)

spec/models/manageiq/providers/amazon/cloud_manager/stubbed_refresher_spec.rb

@@ -132,7 +132,7 @@ def expected_table_counts(disconnect = nil)
 
     {
       :auth_private_key                  => test_counts[:key_pair_count],
-      :ext_management_system             => 2,
+      :ext_management_system             => 3,
       # TODO(lsmola) collect all flavors for original refresh
       :flavor                            => @dto_settings[:dto_refresh] ? 57 : 56,
       :availability_zone                 => 5,

spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb

@@ -111,7 +111,7 @@ def expected_table_counts
 
     {
       :auth_private_key                  => 0,
-      :ext_management_system             => 2,
+      :ext_management_system             => 3,
       :flavor                            => 0,
       :availability_zone                 => 0,
       :vm_or_template                    => 0,