Merge pull request ManageIQ#12062 from mkanoor/rest_api_git_refresh_a…

…nd_import

Rest API Git refresh and import

app/services/git_based_domain_import_service.rb

@@ -30,6 +30,29 @@ def queue_import(git_repo_id, branch_or_tag, tenant_id)
     MiqTask.generic_action_with_callback(task_options, queue_options)
   end
 
+  def queue_refresh_and_import(git_url, ref, ref_type, tenant_id)
+    import_options = {
+      "git_url"   => git_url,
+      "ref"       => ref,
+      "ref_type"  => ref_type,
+      "tenant_id" => tenant_id
+    }
+
+    task_options = {
+      :action => "Refresh and import git repository",
+      :userid => User.current_user.userid
+    }
+
+    queue_options = {
+      :class_name  => "MiqAeDomain",
+      :method_name => "import_git_url",
+      :role        => "git_owner",
+      :args        => [import_options]
+    }
+
+    MiqTask.generic_action_with_callback(task_options, queue_options)
+  end
+
   def import(git_repo_id, branch_or_tag, tenant_id)
     task_id = queue_import(git_repo_id, branch_or_tag, tenant_id)
     task = MiqTask.wait_for_taskid(task_id)

lib/miq_automation_engine/models/miq_ae_domain.rb

@@ -6,6 +6,7 @@ class MiqAeDomain < MiqAeNamespace
   VALID_SOURCES  = [SYSTEM_SOURCE, REMOTE_SOURCE, USER_SOURCE, USER_LOCKED_SOURCE].freeze
   LOCKED_SOURCES = [SYSTEM_SOURCE, REMOTE_SOURCE, USER_LOCKED_SOURCE].freeze
   EDITABLE_PROPERTIES_FOR_REMOTES = [:priority, :enabled].freeze
+  AUTH_KEYS = %w(userid password).freeze
 
   default_scope { where(:parent_id => nil).where(arel_table[:name].not_eq("$")) }
   validates_inclusion_of :parent_id, :in => [nil], :message => 'should be nil for Domain'
@@ -19,10 +20,6 @@ class MiqAeDomain < MiqAeNamespace
   belongs_to :git_repository
   validates_inclusion_of :source, :in => VALID_SOURCES
 
-  BRANCH = 'branch'.freeze
-  TAG    = 'tag'.freeze
-  DEFAULT_BRANCH = 'origin/master'.freeze
-
   EXPORT_EXCLUDE_KEYS = [/^id$/, /^(?!tenant).*_id$/, /^created_on/, /^updated_on/,
                          /^updated_by/, /^reserved$/, /^commit_message/,
                          /^commit_time/, /^commit_sha/, /^ref$/, /^ref_type$/,
@@ -112,16 +109,12 @@ def available_version
     version_field.try(:fetch_path, 'field', 'default_value')
   end
 
-  def self.import_git_repo(options)
-    git_repo = GitRepository.find(options['git_repository_id'])
-    raise "Git repository with id #{options['git_repository_id']} not found" unless git_repo
+  def self.import_git_url(options)
+    MiqAeGitImport.new(options).import
+  end
 
-    MiqAeDomain.find_by(:name => options['domain']).try(:destroy) if options['domain']
-    import_options(git_repo, options)
-    domain = Array.wrap(MiqAeImport.new(options['domain'] || '*', options).import).first
-    raise MiqAeException::DomainNotFound, "Import of domain failed" unless domain
-    domain.update_git_info(git_repo, options['ref'], options['ref_type'])
-    domain
+  def self.import_git_repo(options)
+    MiqAeGitImport.new(options).import
   end
 
   def update_git_info(git_repo, ref, ref_type)
@@ -148,9 +141,9 @@ def latest_ref_info
     raise MiqAeException::InvalidDomain, "Not Git enabled" unless git_enabled?
     raise "No branch or tag selected for this domain" if ref.nil? && ref_type.nil?
     case ref_type
-    when BRANCH
+    when MiqAeGitImport::BRANCH
       git_repository.branch_info(ref)
-    when TAG
+    when MiqAeGitImport::TAG
       git_repository.tag_info(ref)
     end
   end
@@ -189,23 +182,6 @@ def about_file_name
     File.join(MiqAeDatastore::DATASTORE_DIRECTORY, "#{about.fqname}#{CLASS_DIR_SUFFIX}", CLASS_YAML_FILENAME) if about
   end
 
-  def self.import_options(git_repo, options)
-    options['git_dir'] = git_repo.directory_name
-    options['preview'] ||= false
-    options['ref'] ||= DEFAULT_BRANCH
-    options['ref_type'] ||= BRANCH
-    options['ref_type'] = options['ref_type'].downcase
-
-    case options['ref_type']
-    when BRANCH
-      options['branch'] = options['ref']
-    when TAG
-      options['tag'] = options['ref']
-    end
-  end
-
-  private_class_method :import_options
-
   def self.reset_priority_of_system_domains
     domains = MiqAeDomain.where('source = ? AND name <> ?',
                                 SYSTEM_SOURCE,  MiqAeDatastore::MANAGEIQ_DOMAIN).order('name DESC')

lib/miq_automation_engine/models/miq_ae_git_import.rb

@@ -0,0 +1,83 @@
+class MiqAeGitImport
+  AUTH_KEYS = %w(userid password).freeze
+  BRANCH = 'branch'.freeze
+  TAG    = 'tag'.freeze
+  DEFAULT_BRANCH = 'origin/master'.freeze
+
+  def initialize(options)
+    @options = options
+    @preview = options['preview']
+  end
+
+  def import
+    pre_import
+    @options['git_dir'] = @git_repo.directory_name
+    MiqAeDomain.find_by(:name => @options['domain']).try(:destroy) if @options['domain'] && !@preview
+    result = MiqAeYamlImportGitfs.new(@options['domain'] || '*', @options).import
+    domain = Array.wrap(result).first
+    post_import(domain) unless @preview
+    domain
+  end
+
+  private
+
+  def pre_import
+    repo_from_id if @options['git_repository_id']
+    create_repo unless @git_repo
+    default_import_options
+    validate_refs
+  end
+
+  def post_import(domain)
+    if domain
+      domain.update_git_info(@git_repo, @options['ref'], @options['ref_type'])
+    else
+      raise MiqAeException::DomainNotFound, "Import of domain failed"
+    end
+  end
+
+  def repo_from_id
+    @git_repo = GitRepository.find(@options['git_repository_id'])
+    raise "Git repository with id #{@options['git_repository_id']} not found" unless @git_repo
+  end
+
+  def create_repo
+    @git_repo = GitRepository.find_or_create_by(:url => @options['git_url'])
+    @git_repo.update_attributes(:verify_ssl => @options['verify_ssl'] || OpenSSL::SSL::VERIFY_PEER)
+    if @options['userid'] && @options['password']
+      @git_repo.update_authentication(:default => @options.slice(*AUTH_KEYS))
+    end
+    @git_repo.refresh
+  end
+
+  def default_import_options
+    @options['ref'] ||= DEFAULT_BRANCH
+    @options['ref_type'] ||= BRANCH
+    @options['ref_type'] = @options['ref_type'].downcase
+
+    case @options['ref_type']
+    when BRANCH
+      @options['branch'] = @options['ref']
+    when TAG
+      @options['tag'] = @options['ref']
+    else
+      raise ArgumentError, "Invalid reference type #{@options['ref_type']} should be branch or tag"
+    end
+  end
+
+  def validate_refs
+    match = nil
+    case @options['ref_type']
+    when BRANCH
+      other_name = "origin/#{@options['ref']}"
+      match = @git_repo.git_branches.detect { |branch| branch.name.casecmp(@options['ref']) == 0 }
+      match ||= @git_repo.git_branches.detect { |branch| branch.name.casecmp(other_name) == 0 }
+    when TAG
+      match = @git_repo.git_tags.detect { |tag| tag.name.casecmp(@options['ref']) == 0 }
+    end
+    unless match
+      raise ArgumentError, "#{@options['ref_type'].titleize} #{@options['ref']} doesn't exist in repository"
+    end
+    @options['ref'] = match.name
+  end
+end

lib/miq_automation_engine/models/miq_ae_import.rb

@@ -8,6 +8,8 @@ def self.new(domain, options)
       MiqAeYamlImportConsolidated.new(domain, options)
     elsif options['git_dir'].present?
       MiqAeYamlImportGitfs.new(domain, options)
+    elsif options['git_url'].present? || options['git_repository_id'].present?
+      MiqAeGitImport.new(options)
     end
   end
 end

lib/tasks/evm_automate.rake

@@ -126,9 +126,9 @@ namespace :evm do
     desc 'Import automate model information from an export folder or zip file. '
     task :import => :environment do
       begin
-        raise "Must specify domain for import:" if ENV['DOMAIN'].blank?
-        if ENV['YAML_FILE'].blank? && ENV['IMPORT_DIR'].blank? && ENV['ZIP_FILE'].blank?
-          raise 'Must specify either a directory with exported automate model or a zip file'
+        raise "Must specify domain for import:" if ENV['DOMAIN'].blank? && ENV['GIT_URL'].blank?
+        if ENV['YAML_FILE'].blank? && ENV['IMPORT_DIR'].blank? && ENV['ZIP_FILE'].blank? && ENV['GIT_URL'].blank?
+          raise 'Must specify either a directory with exported automate model or a zip file or a http based git url'
         end
         preview        = ENV['PREVIEW'] ||= 'true'
         raise 'Preview must be true or false' unless %w{true false}.include?(preview)
@@ -150,6 +150,16 @@ namespace :evm do
         elsif ENV['YAML_FILE'].present?
           puts "Importing automate domain: #{ENV['DOMAIN']} from file #{ENV['YAML_FILE']}"
           import_options['yaml_file']   = ENV['YAML_FILE']
+        elsif ENV['GIT_URL'].present?
+          puts "Importing automate domain from url #{ENV['GIT_URL']}"
+          ENV['DOMAIN'] = nil
+          import_options['git_url'] = ENV['GIT_URL']
+          import_options['overwrite'] = true
+          import_options['userid'] = ENV['USERID']
+          import_options['password'] = ENV['PASSWORD']
+          import_options['ref'] = ENV['REF'] || MiqAeGitImport::DEFAULT_BRANCH
+          import_options['ref_type'] = ENV['REF_TYPE'] || MiqAeGitImport::BRANCH
+          import_options['verify_ssl'] = ENV['verify_ssl'] || OpenSSL::SSL::VERIFY_PEER
         end
         %w(SYSTEM ENABLED).each do |name|
           if ENV[name].present?

spec/lib/miq_automation_engine/models/miq_ae_domain_spec.rb

@@ -291,16 +291,24 @@
     let(:commit_sha) { "abcd" }
     let(:branch_name) { "b1" }
     let(:tag_name) { "t1" }
-    let(:dom1) { FactoryGirl.create(:miq_ae_git_domain, :tenant => @user.current_tenant) }
+    let(:domain_name) { "BB8" }
+    let(:url) { "http://www.example.com/x/y" }
+    let(:dom1) do
+      FactoryGirl.create(:miq_ae_git_domain,
+                         :tenant => @user.current_tenant,
+                         :name   => domain_name)
+    end
     let(:dom2) { FactoryGirl.create(:miq_ae_domain) }
-    let(:repo) { FactoryGirl.create(:git_repository, :url => "http://www.example.com/x/y") }
+    let(:repo) { FactoryGirl.create(:git_repository, :url => url) }
     let(:git_import) { instance_double('MiqAeYamlImportGitfs') }
     let(:info) { {'commit_time' => commit_time, 'commit_message' => commit_message, 'commit_sha' => commit_sha} }
     let(:new_info) { {'commit_time' => commit_time_new, 'commit_message' => "BB-8", 'commit_sha' => "def"} }
     let(:commit_hash) do
       {'commit_message' => commit_message, 'commit_time' => commit_time,
-       'commit_sha' => commit_sha, 'ref' => branch_name, 'ref_type' => MiqAeDomain::BRANCH}
+       'commit_sha' => commit_sha, 'ref' => branch_name, 'ref_type' => MiqAeGitImport::BRANCH}
     end
+    let(:branch) { FactoryGirl.create(:git_branch, :name => branch_name) }
+    let(:tag) { FactoryGirl.create(:git_tag, :name => tag_name) }
 
     it "check if a git domain is locked" do
       expect(dom1.editable?(@user)).to be_falsey
@@ -314,65 +322,26 @@
     it "git info" do
       expect(repo).to receive(:branch_info).with(branch_name).and_return(info)
 
-      dom1.update_git_info(repo, branch_name, MiqAeDomain::BRANCH)
+      dom1.update_git_info(repo, branch_name, MiqAeGitImport::BRANCH)
       dom1.reload
       expect(dom1.attributes).to have_attributes(commit_hash)
     end
 
-    it "import a domain" do
-      domain_name = dom1.name
-      expect(MiqAeImport).to receive(:new).with(any_args).and_return(git_import)
-      expect_any_instance_of(GitRepository).to receive(:branch_info).with(branch_name).and_return(info)
-      allow(git_import).to receive(:import) { MiqAeDomain.create(:name => domain_name) }
-      options = {'domain' => domain_name, 'git_repository_id' => repo.id,
-                 'tenant_id' => @user.current_tenant.id, 'ref' => branch_name,
-                 'ref_type' => 'BrancH'}
-      dom1 = MiqAeDomain.import_git_repo(options)
-      expect(dom1.attributes).to have_attributes(commit_hash)
-    end
-
-    it "import domain embedded in git repository" do
-      domain_name = "ASTERIX"
-      expect(MiqAeImport).to receive(:new).with(any_args).and_return(git_import)
-      expect_any_instance_of(GitRepository).to receive(:branch_info).with(branch_name).and_return(info)
-      allow(git_import).to receive(:import) { [MiqAeDomain.create(:name => domain_name)] }
-      options = {'git_repository_id' => repo.id,
-                 'tenant_id' => @user.current_tenant.id, 'ref' => branch_name,
-                 'ref_type' => "BRANCH" }
-      dom1 = MiqAeDomain.import_git_repo(options)
-
-      expect(dom1.attributes).to have_attributes(commit_hash)
-    end
-
-    it "import a domain fails" do
-      expect(MiqAeImport).to receive(:new).with(any_args).and_return(git_import)
-      allow(git_import).to receive(:import) { nil }
-      expect do
-        options = {'git_repository_id' => repo.id,
-                   'tenant_id' => @user.current_tenant.id, 'ref' => branch_name}
-        MiqAeDomain.import_git_repo(options)
-      end.to raise_error(MiqAeException::DomainNotFound)
-    end
-
-    it "import without git_repository_id" do
-      expect { MiqAeDomain.import_git_repo({}) }.to raise_error(ActiveRecord::RecordNotFound)
-    end
-
     it "git repo changed for non git domain" do
       expect { dom2.git_repo_changed? }.to raise_error(MiqAeException::InvalidDomain)
     end
 
     it "git repo branch changed" do
       expect_any_instance_of(GitRepository).to receive(:branch_info).with(branch_name).twice.and_return(new_info)
       dom1.update_attributes(:ref => branch_name, :git_repository => repo,
-                             :ref_type => MiqAeDomain::BRANCH, :commit_sha => commit_sha)
+                             :ref_type => MiqAeGitImport::BRANCH, :commit_sha => commit_sha)
       expect(dom1.git_repo_changed?).to be_truthy
       expect(dom1.latest_ref_info).to have_attributes(new_info)
     end
 
     it "git repo tag changed" do
       expect(repo).to receive(:tag_info).with(tag_name).twice.and_return(new_info)
-      dom1.update_attributes(:ref => tag_name, :ref_type => MiqAeDomain::TAG,
+      dom1.update_attributes(:ref => tag_name, :ref_type => MiqAeGitImport::TAG,
                              :git_repository => repo, :commit_sha => commit_sha)
       expect(dom1.git_repo_changed?).to be_truthy
       expect(dom1.latest_ref_info).to have_attributes(new_info)