Merge pull request #76 from SUSE/custom-repos

Added custom repositories - Card 4721

Gemfile

@@ -38,22 +38,11 @@ gem 'railties', '~> 5.1.3'
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   gem 'byebug', platforms: %i[mri mingw x64_mingw]
-  gem 'rspec-rails', '~> 3.5'
   gem 'scc-codestyle'
-  gem 'factory_girl_rails'
-  gem 'shoulda-matchers'
-  gem 'ffaker'
-  gem 'rspec-its'
   gem 'gettext', require: false # needed for gettext_i18n_rails tasks
   gem 'ruby_parser', require: false # needed for gettext_i18n_rails tasks
   gem 'gettext_test_log'
   gem 'memory_profiler'
-  gem 'webmock'
-
-  # Branch that contains fixes for recording and playing back Typhoeus requests with on_headers and on_body callbacks
-  # Hopefully it will get merged some time soon :-)
-  gem 'vcr', github: 'vcr/vcr', ref: '0ce29d08d492792ffecbec468e70a638e9e9f140'
-  gem 'coveralls', require: false
 end
 
 group :development do
@@ -66,9 +55,17 @@ group :development do
 end
 
 group :test do
+  gem 'rspec-rails', '~> 3.5'
+  gem 'factory_girl_rails'
+  gem 'ffaker'
+  gem 'rspec-its'
   gem 'fakefs', require: 'fakefs/safe'
+  gem 'shoulda-matchers'
+  gem 'webmock'
   gem 'fuubar'
   gem 'timecop'
+  gem 'vcr', '~> 4.0'
+  gem 'coveralls', require: false
 end
 
 gem 'simplecov', require: false, group: :test

Gemfile.lock

@@ -1,10 +1,3 @@
-GIT
-  remote: https://github.com/vcr/vcr.git
-  revision: 0ce29d08d492792ffecbec468e70a638e9e9f140
-  ref: 0ce29d08d492792ffecbec468e70a638e9e9f140
-  specs:
-    vcr (3.0.3)
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -216,6 +209,7 @@ GEM
       unf_ext
     unf_ext (0.0.7.4)
     unicode-display_width (1.3.0)
+    vcr (4.0.0)
     versionist (1.5.0)
       activesupport (>= 3)
       railties (>= 3)
@@ -269,9 +263,9 @@ DEPENDENCIES
   timecop
   typhoeus (~> 1.1, >= 1.1.2)
   tzinfo-data
-  vcr!
+  vcr (~> 4.0)
   versionist
   webmock
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

app/models/product.rb

@@ -69,4 +69,8 @@ def change_repositories_mirroring!(conditions, mirroring_enabled)
     repositories.where(conditions).update_all(mirroring_enabled: mirroring_enabled)
   end
 
+  def service
+    Service.find_or_create_by(product_id: id)
+  end
+
 end

app/models/repository.rb

@@ -7,17 +7,38 @@ class Repository < ApplicationRecord
 
   scope :only_installer_updates, -> { unscope(where: :installer_updates).where(installer_updates: true) }
   scope :only_mirrored, -> { where(mirroring_enabled: true) }
+  scope :only_custom, -> { where(scc_id: nil) }
+  scope :only_scc, -> { where.not(scc_id: nil) }
 
   validates :name, presence: true
   validates :external_url, presence: true
   validates :local_path, presence: true
 
-  # Mangles remote repo URL to make a nicer local path, see specs for examples
-  def self.make_local_path(url)
-    uri = URI(url)
-    path = uri.path.to_s
-    path.gsub!(%r{^/repo}, '') if (uri.hostname == 'updates.suse.com')
-    path
+  before_destroy :ensure_destroy_possible
+
+  class << self
+
+    def remove_suse_repos_without_tokens!
+      where(auth_token: nil).where('external_url LIKE ?', 'https://updates.suse.com%').delete_all
+    end
+
+    # Mangles remote repo URL to make a nicer local path, see specs for examples
+    def make_local_path(url)
+      uri = URI(url)
+      path = uri.path.to_s
+      path.gsub!(%r{^/repo}, '') if (uri.hostname == 'updates.suse.com')
+      path
+    end
+
+    def by_id(repository_id, custom: false)
+      return Repository.find_by(id: repository_id) if custom
+      Repository.find_by(scc_id: repository_id)
+    end
+
+    def by_url(url)
+      Repository.find_by(external_url: url)
+    end
+
   end
 
   def refresh_timestamp!
@@ -28,8 +49,14 @@ def change_mirroring!(mirroring_enabled)
     update_column(:mirroring_enabled, mirroring_enabled)
   end
 
-  def self.remove_suse_repos_without_tokens!
-    where(auth_token: nil).where('external_url LIKE ?', 'https://updates.suse.com%').delete_all
+  def custom?
+    scc_id.nil?
+  end
+
+  private
+
+  def ensure_destroy_possible
+    throw(:abort) unless custom?
   end
 
 end

app/services/create_repository_service.rb

@@ -0,0 +1,32 @@
+class CreateRepositoryService
+
+  class InvalidExternalUrl < RuntimeError
+  end
+
+  def call(product_service, url, attributes, custom: false)
+    repository = Repository.find_or_initialize_by(external_url: url)
+
+    # TODO: See if we can clean this up
+    repository.attributes = attributes.select do |k, _|
+      repository.attributes.keys.member?(k.to_s) && k.to_s != 'id'
+    end
+    repository.scc_id = attributes[:id] unless custom
+
+    repository.external_url = url
+    repository.local_path = Repository.make_local_path(url)
+
+    raise InvalidExternalUrl.new(url) if repository.local_path.to_s == ''
+
+    ActiveRecord::Base.transaction do
+      repository.save!
+
+      RepositoriesServicesAssociation.find_or_create_by(
+        service_id: product_service.id,
+        repository_id: repository.id
+      )
+    end
+
+    repository
+  end
+
+end

bin/rmt-cli

@@ -23,7 +23,7 @@ if Process.euid == 0
   Process::Sys.seteuid(Etc.getpwnam(user).uid)
 end
 
-relative_load_paths = %w[lib lib/rmt app/models].map { |dir| File.join(rmt_path, dir) }
+relative_load_paths = %w[lib lib/rmt app/models app/services].map { |dir| File.join(rmt_path, dir) }
 ActiveSupport::Dependencies.autoload_paths += relative_load_paths
 
 # FIXME: this check (or its message) is not correct. It already complains when the folder does just not exist, even-though it is perfectly writeable by my user.

db/migrate/20180115112449_add_scc_id_to_repositories.rb

@@ -0,0 +1,10 @@
+class AddSccIdToRepositories < ActiveRecord::Migration[5.1]
+
+  def change
+    add_column :repositories, :scc_id, 'BIGINT UNSIGNED', after: :id
+    Repository.all.each do |repo|
+      repo.update(scc_id: repo.id)
+    end
+  end
+
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171214093115) do
+ActiveRecord::Schema.define(version: 20180115112449) do
 
   create_table "activations", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.integer "service_id", null: false
@@ -61,6 +61,7 @@
   end
 
   create_table "repositories", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
+    t.bigint "scc_id", unsigned: true
     t.string "name", null: false
     t.string "description"
     t.boolean "enabled", default: false, null: false

lib/rmt/cli/array_printable.rb

@@ -0,0 +1,12 @@
+module RMT::CLI::ArrayPrintable
+
+  def array_to_table(array, options)
+    rows = []
+
+    array.all.each do |element|
+      rows << options.keys.map { |k| element[k] }
+    end
+    Terminal::Table.new headings: options.values, rows: rows
+  end
+
+end

lib/rmt/cli/base.rb

@@ -71,7 +71,7 @@ def needs_path(path)
     File.directory?(path) ? yield : warn("#{path} is not a directory.")
   end
 
-  def mirror(repo, to: RMT::DEFAULT_MIRROR_DIR)
+  def mirror!(repo, to: RMT::DEFAULT_MIRROR_DIR)
     puts "Mirroring repository #{repo.name} to #{to}"
     RMT::Mirror.from_repo_model(repo, to).mirror
     repo.refresh_timestamp!

lib/rmt/cli/custom_repos.rb

@@ -0,0 +1,78 @@
+class RMT::CLI::CustomRepos < RMT::CLI::Base
+
+  include ::RMT::CLI::ArrayPrintable
+
+  desc 'add URL NAME PRODUCT_ID', 'Add a custom repository to a product'
+  def add(url, name, product_id)
+    product = Product.find_by(id: product_id)
+
+    if product.nil?
+      warn "Cannot find product by id #{product_id}."
+      return
+    end
+
+    service = product.service
+    previous_repository = Repository.by_url(url)
+
+    if previous_repository && !previous_repository.custom?
+      warn "A non-custom repository by URL \"#{url}\" already exists."
+      return
+    end
+
+    begin
+      create_repository_service.call(service, url, {
+        name: name,
+        mirroring_enabled: true,
+        autorefresh: 1,
+        enabled: 0
+      }, custom: true)
+
+      puts 'Successfully added custom repository.'
+    rescue CreateRepositoryService::InvalidExternalUrl => e
+      warn "Invalid URL \"#{e.message}\" provided."
+    end
+  end
+
+  desc 'list', 'List all custom repositories'
+  def list
+    repositories = Repository.only_custom
+
+    if repositories.empty?
+      warn 'No custom repositories found.'
+    else
+      puts array_to_table(repositories, {
+        id: 'ID',
+        name: 'Name',
+        enabled: 'Mandatory?',
+        mirroring_enabled: 'Mirror?',
+        last_mirrored_at: 'Last Mirrored'
+      })
+    end
+  end
+  map ls: :list
+
+  desc 'remove ID', 'Remove a custom repository'
+  def remove(repository_id)
+    repository = Repository.by_id(repository_id, custom: true)
+
+    if repository.nil?
+      warn "Cannot find custom repository by id \"#{repository_id}\"."
+      return
+    end
+
+    unless repository.destroy
+      warn 'Cannot remove non-custom repositories.'
+      return
+    end
+
+    puts "Removed custom repository by id \"#{repository.id}\"."
+  end
+  map rm: :remove
+
+  private
+
+  def create_repository_service
+    @create_repository_service ||= CreateRepositoryService.new
+  end
+
+end

lib/rmt/cli/export.rb

@@ -33,7 +33,7 @@ def repos(path)
       repos_ids = JSON.parse(File.read(repos_file))
       repos_ids.each do |id|
         repo = Repository.find_by(id: id)
-        repo ? mirror(repo, to: path) : warn("No repo with id #{id} found in database.")
+        repo ? mirror!(repo, to: path) : warn("No repo with id #{id} found in database.")
       end
     end
   end

lib/rmt/cli/import.rb

@@ -18,7 +18,7 @@ def repos(path)
 
       repos.each do |repo|
         repo.external_url = 'file://' + path + Repository.make_local_path(repo.external_url)
-        mirror(repo)
+        mirror!(repo)
       end
     end
   end

lib/rmt/cli/main.rb

@@ -3,7 +3,6 @@
 
 class RMT::CLI::Main < RMT::CLI::Base
 
-  class_option :version, desc: 'Show RMT version', type: :boolean, aliases: '-v', required: false
   class_option :debug, desc: 'Enable debug output', type: :boolean, aliases: '-d', required: false
 
   desc 'sync', 'Sync database with SUSE Customer Center'
@@ -18,7 +17,14 @@ def sync
   subcommand 'repos', RMT::CLI::Repos
 
   desc 'mirror', 'Mirror repositories'
-  subcommand 'mirror', RMT::CLI::Mirror
+  def mirror
+    repos = Repository.where(mirroring_enabled: true)
+    if repos.empty?
+      warn 'There are no repositories marked for mirroring.'
+      return
+    end
+    repos.each { |repo| mirror!(repo) }
+  end
 
   desc 'import', 'Import commands for Offline Sync'
   subcommand 'import', RMT::CLI::Import