Fetch deps based on urls

lib/inspec/dependencies/dependency_set.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+require 'inspec/dependencies/vendor_index'
+require 'inspec/dependencies/resolver'
+
+module Inspec
+  #
+  # A DependencySet manages a list of dependencies for a profile.
+  #
+  # Currently this class is a thin wrapper interface to coordinate the
+  # VendorIndex and the Resolver.
+  #
+  class DependencySet
+    attr_reader :list, :vendor_path
+
+    # initialize
+    #
+    # @param cwd [String] current working directory for relative path includes
+    # @param vendor_path [String] path which contains vendored dependencies
+    # @return [dependencies] this
+    def initialize(cwd, vendor_path)
+      @cwd = cwd
+      @vendor_path = vendor_path || File.join(Dir.home, '.inspec', 'cache')
+      @list = nil
+    end
+
+    #
+    # 1. Get dependencies, pull things to a local cache if necessary
+    # 2. Resolve dependencies
+    #
+    # @param dependencies [Gem::Dependency] list of dependencies
+    # @return [nil]
+    def vendor(dependencies)
+      return nil if dependencies.nil? || dependencies.empty?
+      @vendor_index ||= VendorIndex.new(@vendor_path)
+      @list = Resolver.resolve(dependencies, @vendor_index, @cwd)
+    end
+  end
+end

lib/inspec/dependencies/requirement.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require 'inspec/fetcher'
+
+module Inspec
+  #
+  # Inspec::Requirement represents a given profile dependency, where
+  # appropriate we delegate to Inspec::Profile directly.
+  #
+  class Requirement
+    attr_reader :name, :dep, :cwd, :opts
+
+    def initialize(name, version_constraints, vendor_index, cwd, opts)
+      @name = name
+      @dep = Gem::Dependency.new(name,
+                                 Gem::Requirement.new(Array(version_constraints)),
+                                 :runtime)
+      @vendor_index = vendor_index
+      @opts = opts
+      @cwd = cwd
+    end
+
+    def matches_spec?(spec)
+      params = spec.profile.metadata.params
+      @dep.match?(params[:name], params[:version])
+    end
+
+    def source_url
+      case source_type
+      when :local_path
+        "file://#{File.expand_path(opts[:path])}"
+      when :url
+        @opts[:url]
+      end
+    end
+
+    def local_path
+      @local_path ||= case source_type
+                      when :local_path
+                        File.expand_path(opts[:path], @cwd)
+                      else
+                        @vendor_index.prefered_entry_for(@name, source_url)
+                      end
+    end
+
+    def source_type
+      @source_type ||= if @opts[:path]
+                         :local_path
+                       elsif opts[:url]
+                         :url
+                       else
+                         fail "Cannot determine source type from #{opts}"
+                       end
+    end
+
+    def fetcher_class
+      @fetcher_class ||= case source_type
+                         when :local_path
+                           Fetchers::Local
+                         when :url
+                           Fetchers::Url
+                         else
+                           fail "No known fetcher for dependency #{name} with source_type #{source_type}"
+                         end
+
+      fail "No fetcher for #{name} (options: #{opts})" if @fetcher_class.nil?
+      @fetcher_class
+    end
+
+    def pull
+      case source_type
+      when :local_path
+        local_path
+      else
+        if @vendor_index.exists?(@name, source_url)
+          local_path
+        else
+          archive = fetcher_class.download_archive(source_url)
+          @vendor_index.add(@name, source_url, archive.path)
+        end
+      end
+    end
+
+    def to_s
+      dep.to_s
+    end
+
+    def path
+      @path ||= pull
+    end
+
+    def profile
+      return nil if path.nil?
+      @profile ||= Inspec::Profile.for_target(path, {})
+    end
+
+    def self.from_metadata(dep, vendor_index, opts)
+      fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
+      name = dep[:name] || fail('You must provide a name for all dependencies')
+      version = dep[:version]
+      new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
+    end
+  end
+end

lib/inspec/dependencies.rb → lib/inspec/dependencies/resolver.rb

@@ -1,18 +1,20 @@
 # encoding: utf-8
 # author: Dominik Richter
 # author: Christoph Hartmann
-
 require 'logger'
-require 'fileutils'
 require 'molinillo'
 require 'inspec/errors'
-require 'inspec/requirement'
+require 'inspec/dependencies/requirement'
 
 module Inspec
+  #
+  # Inspec::Resolver is responsible for recursively resolving all the
+  # depenendencies for a given top-level dependency set.
+  #
   class Resolver
     def self.resolve(requirements, vendor_index, cwd, opts = {})
       reqs = requirements.map do |req|
-        req = Inspec::Requirement.from_metadata(req, cwd: cwd)
+        req = Inspec::Requirement.from_metadata(req, vendor_index, cwd: cwd)
         req || fail("Cannot initialize dependency: #{req}")
       end
 
@@ -21,7 +23,7 @@ def self.resolve(requirements, vendor_index, cwd, opts = {})
 
     def initialize(vendor_index, opts = {})
       @logger = opts[:logger] || Logger.new(nil)
-      @debug_mode = false # TODO: hardcoded for now, grab from options
+      @debug_mode = false
 
       @vendor_index = vendor_index
       @cwd = opts[:cwd] || './'
@@ -90,7 +92,7 @@ def uncached_search_for(dep)
     #   `specification`.
     def dependencies_for(specification)
       specification.profile.metadata.dependencies.map do |r|
-        Inspec::Requirement.from_metadata(r, cwd: @cwd)
+        Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
       end
     end
 
@@ -183,75 +185,4 @@ def print(what = '')
     end
     alias puts print
   end
-
-  class Package
-    def initialize(path, version)
-      @path = path
-      @version = version
-    end
-  end
-
-  class VendorIndex
-    attr_reader :list, :path
-    def initialize(path)
-      @path = path
-      FileUtils.mkdir_p(path) unless File.directory?(path)
-      @list = Dir[File.join(path, '*')].map { |x| load_path(x) }
-    end
-
-    def find(_dependency)
-      # TODO
-      fail NotImplementedError, '#find(dependency) on VendorIndex seeks implementation.'
-    end
-
-    private
-
-    def load_path(_path)
-      # TODO
-      fail NotImplementedError, '#load_path(path) on VendorIndex wants to be implemented.'
-    end
-  end
-
-  class SupermarketDependency
-    def initialize(url, requirement)
-      @url = url
-      @requirement = requirement
-    end
-
-    def self.load(dep)
-      return nil if dep.nil?
-      sname = dep[:supermarket]
-      return nil if sname.nil?
-      surl = dep[:supermarket_url] || 'default_url...'
-      requirement = dep[:version]
-      url = surl + '/' + sname
-      new(url, requirement)
-    end
-  end
-
-  class Dependencies
-    attr_reader :list, :vendor_path
-
-    # initialize
-    #
-    # @param cwd [String] current working directory for relative path includes
-    # @param vendor_path [String] path which contains vendored dependencies
-    # @return [dependencies] this
-    def initialize(cwd, vendor_path)
-      @cwd = cwd
-      @vendor_path = vendor_path || File.join(Dir.home, '.inspec', 'cache')
-      @list = nil
-    end
-
-    # 1. Get dependencies, pull things to a local cache if necessary
-    # 2. Resolve dependencies
-    #
-    # @param dependencies [Gem::Dependency] list of dependencies
-    # @return [nil]
-    def vendor(dependencies)
-      return if dependencies.nil? || dependencies.empty?
-      @vendor_index ||= VendorIndex.new(@vendor_path)
-      @list = Resolver.resolve(dependencies, @vendor_index, @cwd)
-    end
-  end
 end

lib/inspec/dependencies/vendor_index.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+require 'digest'
+require 'fileutils'
+
+module Inspec
+  #
+  # VendorIndex manages an on-disk cache of inspec profiles.  The
+  # cache can contain:
+  #
+  #  - .tar.gz profile archives
+  #  - .zip profile archives
+  #  - unpacked profiles
+  #
+  # Cache entries names include a hash of their source to prevent
+  # conflicts between depenedencies with the same name from different
+  # sources.
+  #
+  #
+  class VendorIndex
+    attr_reader :path
+    def initialize(path)
+      @path = path
+      FileUtils.mkdir_p(path) unless File.directory?(path)
+    end
+
+    def add(name, source, path_from)
+      path_to = base_path_for(name, source)
+      path_to = if File.directory?(path_to)
+                  path_to
+                elsif path_from.end_with?('.zip')
+                  "#{path_to}.tar.gz"
+                elsif path_from.end_with?('.tar.gz')
+                  "#{path_to}.tar.gz"
+                else
+                  fail "Cannot add unknown archive #{path} to vendor index"
+                end
+      FileUtils.cp_r(path_from, path_to)
+      path_to
+    end
+
+    def prefered_entry_for(name, source_url)
+      path = base_path_for(name, source_url)
+      if File.directory?(path)
+        path
+      elsif File.exist?("#{path}.tar.gz")
+        "#{path}.tar.gz"
+      elsif File.exist?("#{path}.zip")
+        "#{path}.zip"
+      end
+    end
+
+    #
+    # For a given name and source_url, return true if the
+    # profile exists in the VendorIndex.
+    #
+    # @param [String] name
+    # @param [String] source_url
+    # @return [Boolean]
+    #
+    def exists?(name, source_url)
+      path = base_path_for(name, source_url)
+      File.directory?(path) || File.exist?("#{path}.tar.gz") || File.exist?("#{path}.zip")
+    end
+
+    #
+    # Return the path to given profile in the vendor index.
+    #
+    # The `source_url` parameter should be a URI-like string that
+    # fully specifies the source of the exact version we want to pull
+    # down.
+    #
+    # @param [String] name
+    # @param [String] source_url
+    # @return [String]
+    #
+    def base_path_for(name, source_url)
+      File.join(@path, key_for(name, source_url))
+    end
+
+    private
+
+    #
+    # Return the key for a given profile in the vendor index.
+    #
+    # The `source_url` parameter should be a URI-like string that
+    # fully specifies the source of the exact version we want to pull
+    # down.
+    #
+    # @param [String] name
+    # @param [String] source_url
+    # @return [String]
+    #
+    def key_for(name, source_url)
+      source_hash = Digest::SHA256.hexdigest source_url
+      "#{name}-#{source_hash}"
+    end
+  end
+end

lib/inspec/profile.rb

@@ -8,7 +8,7 @@
 require 'inspec/fetcher'
 require 'inspec/source_reader'
 require 'inspec/metadata'
-require 'inspec/dependencies'
+require 'inspec/dependencies/dependency_set'
 
 module Inspec
   class Profile # rubocop:disable Metrics/ClassLength
@@ -298,7 +298,7 @@ def load_rule(rule, file, controls, groups)
 
     def load_dependencies
       cwd = File.directory?(@target) ? @target : nil
-      res = Inspec::Dependencies.new(cwd, nil)
+      res = Inspec::DependencySet.new(cwd, nil)
       res.vendor(metadata.dependencies)
       res
     end