Telemetry - determine run context from stack introspection

Berksfile

@@ -2,4 +2,5 @@ source "https://supermarket.chef.io"
 
 cookbook "audit"
 cookbook "build-essential"
+cookbook "install_inspec", path: "./test/kitchen/cookbooks/install_inspec"
 cookbook "os_prepare", path: "./test/kitchen/cookbooks/os_prepare"

kitchen.chef.yml

@@ -11,7 +11,8 @@ verifier:
 
 lifecycle:
   pre_converge:
-    - local: gem build inspec-core.gemspec --output test/kitchen/cookbooks/os_prepare/files/inspec-core-local.gem
+    - local: cd inspec-bin && gem build inspec-core-bin.gemspec --output ../test/kitchen/cookbooks/install_inspec/files/inspec-core-bin.gem
+    - local: gem build inspec-core.gemspec --output test/kitchen/cookbooks/install_inspec/files/inspec-core.gem
 
 platforms:
 # The following (private) boxes are shared via VagrantCloud and are only

kitchen.run_context.yml

@@ -0,0 +1,49 @@
+---
+driver:
+  name: vagrant
+
+provisioner:
+  product_name: chef
+  product_version: 15
+
+verifier:
+  name: inspec
+  sudo: true
+
+lifecycle:
+  # This pre_create hook, along with the install_inspec cookbook, ensures that
+  # the VM has a version of InSpec that matches that built from source.
+  # Note that audit cookbook will ignore this.
+  pre_create:
+    - local: cd inspec-bin && gem build inspec-core-bin.gemspec --output ../test/kitchen/cookbooks/install_inspec/files/inspec-core-bin.gem
+    - local: gem build inspec-core.gemspec --output test/kitchen/cookbooks/install_inspec/files/inspec-core.gem
+
+platforms:
+  - name: ubuntu-18.04
+
+suites:
+  # This test suite uses test/integration/run-context-tk to check the
+  # Telemetry Run Context detection system. This test should detect
+  # running under Test Kitchen.
+  - name: run-context-tk
+    run_list:
+      - recipe[install_inspec]
+
+  # This test suite uses a pair of profiles to check the Telemetry Run Context
+  # detection system under audit cookbook.
+  - name: run-context-after-audit
+    run_list:
+      - recipe[install_inspec]
+      - recipe[audit]
+    attributes:
+      audit:
+        profiles:
+          # This actually runs during converge time, and performs the stack
+          # probe that we care about. It writes a JSON copy of the stack that
+          # it saw to /tmp/audit_stack.json . run-context-after-audit examines
+          # that JSON file and runs it through the context probe to see what it gets.
+          run-context-during-audit:
+            # This should work but doesn't :-(
+            # path: <%= Dir.pwd %>/test/integration/run-context-during-audit/
+            url: https://github.com/inspec/inspec-test-profile-run-context-audit/archive/v0.3.2.zip
+

kitchen.yml

@@ -9,7 +9,8 @@ transport:
 
 lifecycle:
   pre_converge:
-    - local: gem build inspec-core.gemspec --output test/kitchen/cookbooks/os_prepare/files/inspec-core-local.gem
+    - local: cd inspec-bin && gem build inspec-core-bin.gemspec --output ../test/kitchen/cookbooks/install_inspec/files/inspec-core-bin.gem
+    - local: gem build inspec-core.gemspec --output test/kitchen/cookbooks/install_inspec/files/inspec-core.gem
 
 provisioner:
   name: dokken

lib/inspec/cli.rb

@@ -375,6 +375,12 @@ def schema(name)
     puts "Valid schemas are #{Inspec::Schema::OutputSchema.names.join(", ")}"
   end
 
+  desc "run_context", "used to test run-context detection", hide: true
+  def run_context
+    require "inspec/utils/telemetry/run_context_probe"
+    puts Inspec::Telemetry::RunContextProbe.guess_run_context
+  end
+
   desc "version", "prints the version of this tool"
   option :format, type: :string
   def version
@@ -387,11 +393,6 @@ def version
   end
   map %w{-v --version} => :version
 
-  desc "nothing", "does nothing"
-  def nothing
-    puts "you did nothing"
-  end
-
   private
 
   def run_command(opts)

lib/inspec/utils/telemetry/run_context_probe.rb

@@ -0,0 +1,48 @@
+module Inspec
+  module Telemetry
+    # Guesses the run context of InSpec - how were we invoked?
+    # All stack values here are determined experimentally
+
+    class RunContextProbe
+      def self.guess_run_context(stack = nil)
+        stack ||= caller_locations
+        return "test-kitchen" if kitchen?(stack)
+        return "cli" if run_by_thor?(stack)
+        return "audit-cookbook" if audit_cookbook?(stack)
+
+        "unknown"
+      end
+
+      def self.run_by_thor?(stack)
+        stack_match(stack: stack, path: "thor/command", label: "run") &&
+          stack_match(stack: stack, path: "thor/invocation", label: "invoke_command")
+      end
+
+      def self.kitchen?(stack)
+        stack_match(stack: stack, path: "kitchen/instance", label: "verify_action") &&
+          stack_match(stack: stack, path: "kitchen/instance", label: "verify")
+      end
+
+      def self.audit_cookbook?(stack)
+        stack_match(stack: stack, path: "chef/handler", label: "run_report_handlers") &&
+          stack_match(stack: stack, path: "handler/audit_report", label: "report")
+      end
+
+      def self.stack_match(stack: [], label: nil, path: nil)
+        return false if stack.nil?
+
+        stack.any? do |frame|
+          if label && path
+            frame.label == label && frame.absolute_path.include?(path)
+          elsif label
+            frame.label == label
+          elsif path
+            frame.absolute_path.include?(path)
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end

test/functional/telemetry_test.rb

@@ -0,0 +1,14 @@
+require "functional/helper"
+
+describe "telemetry" do
+  include FunctionalHelper
+  parallelize_me!
+
+  describe "detecting CLI runtime context" do
+    let(:run_result) { run_inspec_process("run_context") }
+    it "should detect cli context" do
+      _(run_result.stderr).must_equal ""
+      _(run_result.stdout).must_include "cli"
+    end
+  end
+end

test/integration/run-context-after-audit/controls/after-audit.rb

@@ -0,0 +1,16 @@
+
+# run-context-during-audit should have left a file with a JSON
+# representation of the stack as experienced by the audit cookbook.
+
+raw_data = JSON.parse(file("/tmp/audit_stack.json").content)
+# These aren't really stack frames, so we do some duck typing
+Frame = Struct.new(:absolute_path, :label)
+reconstructed_stack = raw_data.map{ |f| Frame.new(f["absolute_path"], f["label"]) }
+
+require "inspec/utils/telemetry/run_context_probe"
+
+control "run-context" do
+  describe Inspec::Telemetry::RunContextProbe.guess_run_context(reconstructed_stack) do
+    it { should eq "audit-cookbook" }
+  end
+end

test/integration/run-context-after-audit/inspec.yml

@@ -0,0 +1,10 @@
+name: run-context-after-audit
+title: InSpec Profile
+maintainer: InSpec Engineering
+copyright: Chef Software, Inc.
+copyright_email: inspec@chef.io
+license: Apache-2.0
+summary: An InSpec Compliance Profile to test inspec run context detection under audit-cookbook
+version: 0.1.0
+supports:
+  platform: os

test/integration/run-context-tk/controls/during-verify.rb

@@ -0,0 +1,16 @@
+# This seemingly pointless test is here so that a dump of
+# the stack will appear in the test output for debugging
+stack = caller_locations(4)
+control "json-stack" do
+  describe stack.to_json do
+    it { should be_kind_of String }
+  end
+end
+
+require "inspec/utils/telemetry/run_context_probe"
+
+control "run-context" do
+  describe Inspec::Telemetry::RunContextProbe.guess_run_context do
+    it { should cmp "test-kitchen" }
+  end
+end

test/integration/run-context-tk/inspec.yml

@@ -0,0 +1,10 @@
+name: run-context-tk
+title: InSpec Profile
+maintainer: InSpec Engineering
+copyright: Chef Software, Inc.
+copyright_email: inspec@chef.io
+license: Apache-2.0
+summary: An InSpec Compliance Profile to test inspec run context detection during test-kitchen verify
+version: 0.1.0
+supports:
+  platform: os

test/kitchen/cookbooks/install_inspec/metadata.rb

@@ -0,0 +1,6 @@
+name "install_inspec"
+maintainer "Chef Software, Inc."
+maintainer_email "support@chef.io"
+description "This cookbook installs the InSpec gem."
+license "Apache-2.0"
+version "1.0.0"

test/kitchen/cookbooks/install_inspec/recipes/default.rb

@@ -0,0 +1,20 @@
+cookbook_file "/root/inspec-core.gem" do
+  source "inspec-core.gem"
+  action :create
+end
+
+cookbook_file "/root/inspec-core-bin.gem" do
+  source "inspec-core-bin.gem"
+  action :create
+end
+
+
+chef_gem "inspec-core" do
+  source "/root/inspec-core.gem"
+  action :upgrade
+end
+
+chef_gem "inspec-core-bin" do
+  source "/root/inspec-core-bin.gem"
+  action :upgrade
+end

test/kitchen/cookbooks/os_prepare/metadata.rb

@@ -7,6 +7,7 @@
 # this cookbook has changed rapidly in the past so we pin it here to prevent build failures
 depends "postgresql", "= 7.1.3"
 
+depends "install_inspec"
 depends "runit"
 depends "ssh-hardening"
 depends "docker"

test/kitchen/cookbooks/os_prepare/recipes/default.rb

@@ -5,14 +5,7 @@
 
 # inject the current inspec gem for use with audit cookbook
 # this is generated via Rake test:integration
-cookbook_file "/root/inspec-core-local.gem" do
-  source "inspec-core-local.gem"
-  action :create
-end
-
-chef_gem "inspec" do
-  source "/root/inspec-core-local.gem"
-end
+include_recipe("install_inspec")
 
 def uuid_from_string(string)
   require "digest/sha1"