Plugins: Add support for 'bundles' migration

Rakefile

@@ -54,7 +54,10 @@ task default: [:lint, :test]
 
 Rake::TestTask.new do |t|
   t.libs << 'test'
-  t.pattern = 'test/unit/**/*_test.rb'
+  t.test_files = Dir.glob([
+    'test/unit/**/*_test.rb',
+    'lib/plugins/inspec-*/test/unit/**/*_test.rb',
+  ])
   t.warning = true
   t.verbose = true
   t.ruby_opts = ['--dev'] if defined?(JRUBY_VERSION)
@@ -69,7 +72,10 @@ namespace :test do
 
   Rake::TestTask.new(:functional) do |t|
     t.libs << 'test'
-    t.pattern = 'test/functional/**/*_test.rb'
+    t.test_files = Dir.glob([
+      'test/functional/**/*_test.rb',
+      'lib/plugins/inspec-*/test/functional/**/*_test.rb',
+    ])
     t.warning = true
     t.verbose = true
     t.ruby_opts = ['--dev'] if defined?(JRUBY_VERSION)

lib/inspec/plugin/v2/loader.rb

@@ -17,7 +17,14 @@ def initialize(options = {})
       determine_plugin_conf_file
       read_conf_file
       unpack_conf_file
+
+      # Old-style (v0, v1) co-distributed plugins were called 'bundles'
+      # and were located in lib/bundles
       detect_bundled_plugins unless options[:omit_bundles]
+
+      # New-style (v2) co-distributed plugins are in lib/plugins,
+      # and may be safely loaded
+      detect_core_plugins unless options[:omit_core_plugins]
     end
 
     def load_all
@@ -26,17 +33,20 @@ def load_all
         # rubocop: disable Lint/RescueException
         begin
           # We could use require, but under testing, we need to repeatedly reload the same
-          # plugin.
-          if plugin_details.entry_point.include?('test/unit/mock/plugins')
-            load plugin_details.entry_point + '.rb'
-          else
+          # plugin.  However, gems only work with require (rubygems dooes not overload `load`)
+          if plugin_details.installation_type == :gem
+            activate_managed_gems_for_plugin(plugin_name)
             require plugin_details.entry_point
+          else
+            load_path = plugin_details.entry_point
+            load_path += '.rb' unless plugin_details.entry_point.end_with?('.rb')
+            load load_path
           end
           plugin_details.loaded = true
           annotate_status_after_loading(plugin_name)
         rescue ::Exception => ex
           plugin_details.load_exception = ex
-          Inspec::Log.error "Could not load plugin #{plugin_name}"
+          Inspec::Log.error "Could not load plugin #{plugin_name}: #{ex.message}"
         end
         # rubocop: enable Lint/RescueException
       end
@@ -60,7 +70,7 @@ def exit_on_load_error
     end
 
     def activate(plugin_type, hook_name)
-      activator = registry.find_activators(plugin_type: plugin_type, activation_name: hook_name).first
+      activator = registry.find_activators(plugin_type: plugin_type, activator_name: hook_name).first
       # We want to capture literally any possible exception here, since we are storing them.
       # rubocop: disable Lint/RescueException
       begin
@@ -80,7 +90,7 @@ def activate_mentioned_cli_plugins(cli_args = ARGV)
         next if act.activated
         # If there is anything in the CLI args with the same name, activate it
         # If the word 'help' appears in the first position, load all CLI plugins
-        if cli_args.include?(act.activator_name.to_s) || cli_args[0] == 'help'
+        if cli_args.include?(act.activator_name.to_s) || cli_args[0] == 'help' || cli_args.size.empty?
           activate(:cli_command, act.activator_name)
           act.implementation_class.register_with_thor
         end
@@ -138,6 +148,22 @@ def determine_plugin_conf_file
       @plugin_conf_file_path = File.join(@plugin_conf_file_path, 'plugins.json')
     end
 
+    def detect_core_plugins
+      core_plugins_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'plugins'))
+      # These are expected to be organized as proper separate projects,
+      # with lib/ dirs, etc.
+      Dir.glob(File.join(core_plugins_dir, 'inspec-*')).each do |plugin_dir|
+        status = Inspec::Plugin::V2::Status.new
+        status.name = File.basename(plugin_dir)
+        status.entry_point = File.join(plugin_dir, 'lib', status.name + '.rb')
+        status.installation_type = :path
+        status.loaded = false
+        registry[status.name.to_sym] = status
+      end
+    end
+
+    # TODO: DRY up re: Installer read_or_init_config_file
+    # TODO: refactor the plugin.json file to have its own class, which Loader consumes
     def read_conf_file
       if File.exist?(@plugin_conf_file_path)
         @plugin_file_contents = JSON.parse(File.read(@plugin_conf_file_path))

lib/plugins/README.md

@@ -0,0 +1,16 @@
+# Core Plugins of InSpec
+
+This area contains the plugins that ship with InSpec.  They are automatically detected by the plugin loader.
+
+## inspec-* directories
+
+Each subdirectory here that begins with `inspec-` is intended to be a self-contained plugin project,
+with code, docs, and tests.
+
+## shared directory
+
+This directory contains material that is reusable for core plugins, such as a test helper tuned to assisting
+core plugin testing.
+
+
+

lib/plugins/shared/core_plugin_test_helper.rb

@@ -0,0 +1,45 @@
+
+# Load test harness - MiniTest
+require 'minitest/autorun'
+require 'minitest/unit'
+require 'minitest/pride'
+require 'minitest/spec'
+
+# Data formats commonly used in testing
+require 'json'
+require 'ostruct'
+
+# Utilities often needed
+require 'fileutils'
+
+# Configure MiniTest to expose things like `let`
+class Module
+  include Minitest::Spec::DSL
+end
+
+module CorePluginBaseHelper
+  let(:repo_path) { File.expand_path(File.join(__FILE__, '..', '..', '..', '..')) }
+  let(:inspec_bin_path) { File.join(repo_path, 'bin', 'inspec') }
+  let(:core_mock_path) { File.join(repo_path, 'test', 'unit', 'mock') }
+  let(:core_fixture_plugins_path) { File.join(core_mock_path, 'plugins') }
+  let(:core_config_dir_path) { File.join(core_mock_path, 'config_dirs') }
+
+  let(:registry) { Inspec::Plugin::V2::Registry.instance }
+end
+
+module CorePluginFunctionalHelper
+  include CorePluginBaseHelper
+
+  require 'train'
+  TRAIN_CONNECTION = Train.create('local', command_runner: :generic).connection
+
+  def run_inspec_process(command_line, env = {})
+    env_prefix = env.to_a.map { |assignment| "#{assignment[0]}=#{assignment[1]}" }.join(' ')
+    TRAIN_CONNECTION.run_command("#{env_prefix} #{inspec_bin_path} #{command_line}")
+  end
+end
+
+module CorePluginUnitHelper
+  include CorePluginBaseHelper
+  require 'inspec'
+end

test/functional/plugins_test.rb

@@ -79,4 +79,4 @@
     outcome.exit_status.must_equal 0
     outcome.stdout.must_include 'inspec meaningoflife'
   end
-end
+end

test/helper.rb

@@ -26,6 +26,7 @@
 require 'inspec/fetcher'
 require 'inspec/source_reader'
 require 'inspec/resource'
+require 'resource_support/aws'
 require 'inspec/reporters'
 require 'inspec/backend'
 require 'inspec/profile'

test/unit/plugin/v2/loader_test.rb

@@ -29,6 +29,8 @@ def setup
       :init,
       :supermarket,
      ]
+     @core_plugins = [
+     ]
   end
 
   def teardown
@@ -53,6 +55,14 @@ def test_constructor_should_detect_bundled_plugins
     end
   end
 
+  def test_constructor_should_detect_core_plugins
+    reg = Inspec::Plugin::V2::Registry.instance
+    loader = Inspec::Plugin::V2::Loader.new
+    @core_plugins.each do |core_plugin_name|
+      assert reg.known_plugin?(core_plugin_name), "\n#{core_plugin_name} should be detected as a core plugin"
+    end
+  end
+
   def test_constructor_should_skip_bundles_when_option_is_set
     reg = Inspec::Plugin::V2::Registry.instance
     loader = Inspec::Plugin::V2::Loader.new(omit_bundles: true)
@@ -61,6 +71,14 @@ def test_constructor_should_skip_bundles_when_option_is_set
     end
   end
 
+  def test_constructor_should_skip_core_when_option_is_set
+    reg = Inspec::Plugin::V2::Registry.instance
+    loader = Inspec::Plugin::V2::Loader.new(omit_core_plugins: true)
+    @core_plugins.each do |core_plugin_name|
+      refute reg.known_plugin?(core_plugin_name), "\n#{core_plugin_name} should not be detected when omit_core_plugins is set"
+    end
+  end
+
   def test_constructor_when_using_home_dir_detects_declared_plugins
     ENV['HOME'] = File.join(@config_dir_path, 'fakehome')
     reg = Inspec::Plugin::V2::Registry.instance
@@ -97,7 +115,7 @@ def test_constuctor_when_the_plugin_config_is_a_bad_version_it_throws_an_excepti
 
   def test_load_no_plugins_should_load_no_plugins
     reg = Inspec::Plugin::V2::Registry.instance
-    loader = Inspec::Plugin::V2::Loader.new(omit_bundles: true)
+    loader = Inspec::Plugin::V2::Loader.new(omit_bundles: true, omit_core_plugins: true)
     loader.load_all
     assert_equal 0, reg.loaded_count, "\nRegistry load count"
   end
@@ -166,7 +184,7 @@ def test_activation
     assert_nil activator.implementation_class, 'Test activator should not know implementation class prior to activation'
     refute InspecPlugins::MeaningOfLife.const_defined?(:MockPlugin), 'impl_class should not be defined prior to activation'
 
-    loader.activate(:mock_plugin_type, 'meaning-of-life-the-universe-and-everything')
+    loader.activate(:mock_plugin_type, :'meaning-of-life-the-universe-and-everything')
 
     # Activation postconditions
     assert activator.activated, 'Test activator should be activated after activate'

test/unit/reporters/automate_test.rb

@@ -40,7 +40,7 @@
       }
       stub = Net::HTTP::Post.new("/data-collector/v0/", headers)
       Net::HTTP::Post.expects(:new).with("/data-collector/v0/", headers).returns(stub)
-      Net::HTTP.any_instance.stubs(:request).returns(true)
+      Net::HTTP.any_instance.stubs(:request).returns(Net::HTTPSuccess.new(nil, nil, nil))
       report.send_report.must_equal true
     end
   end

test/unit/resources/aws_route_tables_test.rb

@@ -2,7 +2,7 @@
 
 class EmptyAwsRouteTablesTest < Minitest::Test
   def setup
-    AwsRouteTables::BackendFactory.select(AwsMRtbB::Empty)
+    AwsRouteTables::BackendFactory.select(AwsMRtbsB::Empty)
   end
 
   def test_constructor_no_args_ok
@@ -20,7 +20,7 @@ def test_constructor_reject_unknown_resource_params
 
 class BasicAwsRouteTablesTest2 < Minitest::Test
   def setup
-    AwsRouteTables::BackendFactory.select(AwsMRtbB::Basic)
+    AwsRouteTables::BackendFactory.select(AwsMRtbsB::Basic)
   end
 
   def test_search_hit
@@ -45,7 +45,7 @@ def test_property_route_table_ids
 end
 
 # MRtbB = Mock Routetable Backend
-module AwsMRtbB
+module AwsMRtbsB
   class Empty < AwsBackendBase
     def describe_route_tables(query)
       OpenStruct.new(route_tables: [])