WIP: Fetch deps based on urls

This extends the dependency feature to include support for url-based
dependencies.  It takes some deviations from the current support for
URLs that we'll likely want to make more consistent.

To better support vendoring consistently across all future source
types, we unpack the tarball into the cache. If we think we are *only*
going to support tarballs, we could avoid the unpacking and use the
existing streaming support only. Another advnatage of unpacking the
tarball into the cache/vendor directory is that eventually most
classes would *only* need to care about how to read data from a path
on disk and the fetchers would only need to care about how to download
and unpack something to a location on disk.

Signed-off-by: Steven Danna <steve@chef.io>

examples/inheritance/controls/example.rb

@@ -12,3 +12,14 @@
 
 include_controls 'profile_a'
 include_controls 'profile_b'
+
+include_controls 'os-hardening' do
+  skip_control 'package-01'
+  skip_control 'package-02'
+  skip_control 'package-03'
+  skip_control 'package-04'
+  skip_control 'package-05'
+  1.upto(33) do |i|
+    skip_control "sysctl-%02d" % i
+  end
+end

examples/inheritance/inspec.yml

@@ -15,3 +15,5 @@ depends:
     path: ../profile_a
   - name: profile_b
     path: ../profile_b
+  - name: os-hardening
+    url: https://github.com/dev-sec/tests-os-hardening/archive/master.tar.gz

lib/fetchers/tar.rb

@@ -32,6 +32,54 @@ def read(file)
       @contents[file] ||= read_from_tar(file)
     end
 
+    def find_root_level(entry_iterator)
+      entry_iterator.each do |e|
+        if File.basename(e.full_name) == "inspec.yml"
+          parts = e.full_name.split(File::SEPARATOR)
+          ret = if parts[0] == "."
+                  0
+                else
+                  parts.length - 1
+                end
+          return ret
+        end
+      end
+      0
+    end
+
+    def download_to(path)
+      Gem::Package::TarReader.new(Zlib::GzipReader.open(@target)) do |tar|
+        #
+        # Try to determine how many parts we need to strip off
+        #
+        strip_components = find_root_level(tar)
+        tar.rewind
+        tar.each do |entry|
+          # Split the path and trim off the leading components
+          file_parts = entry.full_name.split(File::SEPARATOR)
+          trimmed_parts = file_parts.drop(strip_components)
+          trimmed_filename = File.join(trimmed_parts)
+
+          dest = File.join(path, trimmed_filename)
+
+          if entry.directory?
+            File.delete(dest) if File.file?(dest)
+            FileUtils.mkdir_p(dest, { mode: entry.header.mode,
+                                      verbose: false })
+          elsif entry.file?
+            FileUtils.rm_rf(dest) if File.directory?(dest)
+            File.open(dest, "wb") do |f|
+              f.print entry.read
+            end
+            FileUtils.chmod(entry.header.mode, dest, verbose: false)
+          elsif entry.header.typeflag == '2'
+            File.symlink(entry.header.linkname, dest)
+          end
+        end
+      end
+      path
+    end
+
     def read_from_tar(file)
       return nil unless @files.include?(file)
       res = nil

lib/inspec/requirement.rb

@@ -5,12 +5,12 @@ module Inspec
   class Requirement
     attr_reader :name, :dep, :cwd, :opts
 
-    def initialize(name, version_constraints, cwd, opts)
+    def initialize(name, version_constraints, vendor_index, cwd, opts)
       @name = name
       @dep = Gem::Dependency.new(name,
                                  Gem::Requirement.new(Array(version_constraints)),
                                  :runtime)
-      @fetcher = fetcher_for_options(opts)
+      @vendor_index = vendor_index
       @opts = opts
       @cwd = cwd
     end
@@ -20,28 +20,65 @@ def matches_spec?(spec)
       @dep.match?(params[:name], params[:version])
     end
 
-    def fetcher_for_options(opts)
-      f = if opts[:path]
-            Fetchers::Local.resolve(File.expand_path(opts[:path], @cwd))
-          elsif opts[:url]
-            Fetchers::Url.resolve(opts[:url])
-          else
-            fail "No known fetcher for dependency #{name} (options: #{opts})"
-          end
+    def source_url
+      case source_type
+      when :local_path
+        "file://#{File.expand_path(opts[:path])}"
+      when :url
+        @opts[:url]
+      end
+    end
 
-      fail "Unable to resolve source for #{name} (options: #{opts})" if f.nil?
-      f
+    def local_path
+      @local_path ||= case source_type
+                      when :local_path
+                        File.expand_path(opts[:path])
+                      else
+                        @vendor_index.path_for(@name, source_url)
+                      end
+    end
+
+    def source_type
+      @source_type ||= if @opts[:path]
+                         :local_path
+                       elsif opts[:url]
+                         :url
+                       else
+                         fail "Cannot determine source type from #{opts}"
+                       end
+    end
+
+    def fetcher
+      @fetcher ||= case source_type
+                   when :local_path
+                     Fetchers::Local.resolve(File.expand_path(@opts[:path], @cwd))
+                   when :url
+                     Fetchers::Url.resolve(@opts[:url])
+                   else
+                     fail "No known fetcher for dependency #{name} with source_type #{source_type}"
+                   end
+
+      fail "Unable to resolve source for #{name} (options: #{opts})" if @fetcher.nil?
+      @fetcher
     end
 
     def pull
-      case
-      when @opts[:path]
-        File.expand_path(@opts[:path], @cwd)
+      case source_type
+      when :local_path
+        local_path
       else
-        fail 'You must specify the source of the dependency (for now...)'
+        if @vendor_index.exists?(@name, source_url)
+          local_path
+        else
+          fetcher.download_to(local_path)
+        end
       end
     end
 
+    def to_s
+      dep.to_s
+    end
+
     def path
       @path ||= pull
     end
@@ -51,11 +88,11 @@ def profile
       @profile ||= Inspec::Profile.for_target(path, {})
     end
 
-    def self.from_metadata(dep, opts)
+    def self.from_metadata(dep, vendor_index, opts)
       fail 'Cannot load empty dependency.' if dep.nil? || dep.empty?
       name = dep[:name] || fail('You must provide a name for all dependencies')
       version = dep[:version]
-      new(name, version, opts[:cwd], opts.merge(dep))
+      new(name, version, vendor_index, opts[:cwd], opts.merge(dep))
     end
   end
 end

lib/inspec/resolver.rb

@@ -10,7 +10,7 @@ module Inspec
   class Resolver
     def self.resolve(requirements, vendor_index, cwd, opts = {})
       reqs = requirements.map do |req|
-        req = Inspec::Requirement.from_metadata(req, cwd: cwd)
+        req = Inspec::Requirement.from_metadata(req, vendor_index, cwd: cwd)
         req || fail("Cannot initialize dependency: #{req}")
       end
 
@@ -19,7 +19,7 @@ def self.resolve(requirements, vendor_index, cwd, opts = {})
 
     def initialize(vendor_index, opts = {})
       @logger = opts[:logger] || Logger.new(nil)
-      @debug_mode = false # TODO: hardcoded for now, grab from options
+      @debug_mode = false
 
       @vendor_index = vendor_index
       @cwd = opts[:cwd] || './'
@@ -88,7 +88,7 @@ def uncached_search_for(dep)
     #   `specification`.
     def dependencies_for(specification)
       specification.profile.metadata.dependencies.map do |r|
-        Inspec::Requirement.from_metadata(r, cwd: @cwd)
+        Inspec::Requirement.from_metadata(r, @vendor_index, cwd: @cwd)
       end
     end
 

lib/inspec/vendor_index.rb

@@ -1,23 +1,57 @@
 # encoding: utf-8
+require 'digest'
+
 module Inspec
   class VendorIndex
-    attr_reader :list, :path
+    attr_reader :path
     def initialize(path)
       @path = path
       FileUtils.mkdir_p(path) unless File.directory?(path)
-      @list = Dir[File.join(path, '*')].map { |x| load_path(x) }
     end
 
-    def find(_dependency)
-      # TODO
-      fail NotImplementedError, '#find(dependency) on VendorIndex seeks implementation.'
+    #
+    # For a given name and source_url, return true if the
+    # profile exists in the VendorIndex.
+    #
+    # @param [String] name
+    # @param [String] source_url
+    # @return [Boolean]
+    #
+    def exists?(name, source_url)
+      File.exist?("#{path_for(name, source_url)}/inspec.yml")
+    end
+
+    #
+    # Return the path to given profile in the vendor index.
+    #
+    # The `source_url` parameter should be a URI-like string that
+    # fully specifies the source of the exact version we want to pull
+    # down.
+    #
+    # @param [String] name
+    # @param [String] source_url
+    # @return [String]
+    #
+    def path_for(name, source_url)
+      File.join(@path, key_for(name, source_url))
     end
 
     private
 
-    def load_path(_path)
-      # TODO
-      fail NotImplementedError, '#load_path(path) on VendorIndex wants to be implemented.'
+    #
+    # Return the key for a given profile in the vendor index.
+    #
+    # The `source_url` parameter should be a URI-like string that
+    # fully specifies the source of the exact version we want to pull
+    # down.
+    #
+    # @param [String] name
+    # @param [String] source_url
+    # @return [String]
+    #
+    def key_for(name, source_url)
+      source_hash = Digest::SHA256.hexdigest source_url
+      "#{name}-#{source_hash}"
     end
   end
 end