Merge pull request #100 from DataDog/anmarchenko/parametrized_cucumbe…

…r_tests

[CIVIS-7722] Parametrized tests support for cucumber

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -14,6 +14,8 @@ class Formatter
           private :config
 
           def initialize(config)
+            @ast_lookup = ::Cucumber::Formatter::AstLookup.new(config) if defined?(::Cucumber::Formatter::AstLookup)
+
             @config = config
             @failed_tests_count = 0
 
@@ -54,20 +56,26 @@ def on_test_run_finished(event)
           def on_test_case_started(event)
             test_suite_name = event.test_case.location.file
 
+            tags = {
+              CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+              CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
+              CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
+              CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file),
+              CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s
+            }
+
             start_test_suite(test_suite_name) unless same_test_suite_as_current?(test_suite_name)
 
-            CI.start_test(
+            test_span = CI.start_test(
               event.test_case.name,
               test_suite_name,
-              tags: {
-                CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
-                CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file),
-                CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s
-              },
+              tags: tags,
               service: configuration[:service_name]
             )
+
+            if (parameters = extract_parameters_hash(event.test_case))
+              test_span.set_parameters(parameters)
+            end
           end
 
           def on_test_case_finished(event)
@@ -155,6 +163,27 @@ def same_test_suite_as_current?(test_suite_name)
             test_suite.name == test_suite_name
           end
 
+          def extract_parameters_hash(test_case)
+            # not supported in cucumber < 4.0
+            return nil unless @ast_lookup
+
+            scenario_source = @ast_lookup.scenario_source(test_case)
+
+            # cucumber examples are only supported for scenario outlines
+            return nil unless scenario_source.type == :ScenarioOutline
+
+            scenario_source.examples.table_header.cells.map(&:value).zip(
+              scenario_source.row.cells.map(&:value)
+            ).to_h
+          rescue => e
+            Datadog.logger.warn do
+              "Unable to extract parameters from test case #{test_case.name}: " \
+                "#{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+            end
+
+            nil
+          end
+
           def configuration
             Datadog.configuration.ci[:cucumber]
           end

lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -17,8 +17,9 @@ module InstanceMethods
             attr_reader :datadog_formatter
 
             def formatters
+              existing_formatters = super
               @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
-              [@datadog_formatter] + super
+              [@datadog_formatter] + existing_formatters
             end
           end
         end

lib/datadog/ci/ext/test.rb

@@ -12,16 +12,16 @@ module Test
         TAG_FRAMEWORK = "test.framework"
         TAG_FRAMEWORK_VERSION = "test.framework_version"
         TAG_NAME = "test.name"
-        TAG_SKIP_REASON = "test.skip_reason" # DEV: Not populated yet
+        TAG_SKIP_REASON = "test.skip_reason"
         TAG_STATUS = "test.status"
         TAG_SUITE = "test.suite"
         TAG_MODULE = "test.module"
-        TAG_TRAITS = "test.traits"
         TAG_TYPE = "test.type"
         TAG_COMMAND = "test.command"
         TAG_SOURCE_FILE = "test.source.file"
         TAG_SOURCE_START = "test.source.start"
         TAG_CODEOWNERS = "test.codeowners"
+        TAG_PARAMETERS = "test.parameters"
 
         TEST_TYPE = "test"
 

lib/datadog/ci/null_span.rb

@@ -55,6 +55,9 @@ def set_environment_runtime_tags
       def set_default_tags
       end
 
+      def set_parameters(arguments, metadata = {})
+      end
+
       def to_s
         self.class.to_s
       end

lib/datadog/ci/span.rb

@@ -133,6 +133,25 @@ def set_default_tags
         tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST)
       end
 
+      # Sets the parameters for this span for parametrized tests (e.g. Cucumber examples or RSpec shared specs).
+      # Parameters are needed to compute test fingerprint to distinguish between different tests having same names.
+      # @param [Hash] arguments the arguments that test accepts as key-value hash
+      # @param [Hash] metadata optional metadata
+      # @return [void]
+      def set_parameters(arguments, metadata = {})
+        return if arguments.nil? || arguments.empty?
+
+        set_tag(
+          Ext::Test::TAG_PARAMETERS,
+          JSON.generate(
+            {
+              arguments: arguments,
+              metadata: metadata
+            }
+          )
+        )
+      end
+
       def to_s
         "#{self.class}(name:#{name},tracer_span:#{@tracer_span})"
       end

lib/datadog/ci/test.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 require_relative "span"
 
 module Datadog

sig/datadog/ci/contrib/cucumber/formatter.rbs

@@ -6,6 +6,7 @@ module Datadog
           private
           @failed_tests_count: Integer
           @current_test_suite: Datadog::CI::Span?
+          @ast_lookup: ::Cucumber::Formatter::AstLookup
 
           attr_reader config: untyped
 
@@ -39,6 +40,8 @@ module Datadog
 
           def finish_test: (Datadog::CI::Span test, Cucumber::Core::Test::Result result) -> void
 
+          def extract_parameters_hash: (untyped test_case) -> Hash[String, String]?
+
           def configuration: () -> untyped
         end
       end

sig/datadog/ci/ext/test.rbs

@@ -20,8 +20,6 @@ module Datadog
 
         TAG_MODULE: String
 
-        TAG_TRAITS: String
-
         TAG_TYPE: String
 
         TAG_COMMAND: String
@@ -32,6 +30,8 @@ module Datadog
 
         TAG_CODEOWNERS: String
 
+        TAG_PARAMETERS: String
+
         TAG_TEST_SESSION_ID: String
 
         TAG_TEST_MODULE_ID: String

sig/datadog/ci/span.rbs

@@ -42,6 +42,8 @@ module Datadog
       def set_environment_runtime_tags: () -> void
 
       def set_default_tags: () -> void
+
+      def set_parameters: (Hash[String, Object] arguments, ?Hash[String, Object] metadata) -> void
     end
   end
 end

spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb

@@ -5,3 +5,11 @@
 Then "failure" do
   expect(1 + 1).to eq(3)
 end
+
+Then(/I add (-?\d+) and (-?\d+)/) do |n1, n2|
+  @res = n1.to_i + n2.to_i
+end
+
+Then(/the result should be (-?\d+)/) do |res|
+  expect(@res).to eq(res.to_i)
+end

spec/datadog/ci/contrib/cucumber/features/with_parameters.feature

@@ -0,0 +1,12 @@
+Feature: Datadog integration for parametrized tests
+
+  Scenario Outline: scenario with examples
+    Given datadog
+    When I add <num1> and <num2>
+    Then the result should be <total>
+
+  Examples:
+    | num1 | num2 | total |
+    | 0    | 1    | 1     |
+    | 1    | 2    | 3     |
+    | 2    | 3    | 5     |

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -10,6 +10,9 @@
     let(:integration_options) { {service_name: "jalapenos"} }
   end
 
+  let(:cucumber_8_or_above) { Gem::Version.new("8.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version }
+  let(:cucumber_4_or_above) { Gem::Version.new("4.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version }
+
   let(:run_id) { rand(1..2**64 - 1) }
   let(:steps_file_definition_path) { "spec/datadog/ci/contrib/cucumber/features/step_definitions/steps.rb" }
   let(:steps_file_for_run_path) do
@@ -35,12 +38,10 @@
   let(:kernel) { double(:kernel) }
 
   let(:cli) do
-    cucumber_8 = Gem::Version.new("8.0.0")
-
-    if Datadog::CI::Contrib::Cucumber::Integration.version < cucumber_8
-      Cucumber::Cli::Main.new(args, stdin, stdout, stderr, kernel)
-    else
+    if cucumber_8_or_above
       Cucumber::Cli::Main.new(args, stdout, stderr, kernel)
+    else
+      Cucumber::Cli::Main.new(args, stdin, stdout, stderr, kernel)
     end
   end
 
@@ -202,14 +203,45 @@
     end
   end
 
+  context "executing a scenario with examples" do
+    let(:feature_file_to_run) { "with_parameters.feature" }
+
+    it "a single test suite, and a test span for each example with parameters JSON" do
+      expect(test_spans.count).to eq(3)
+      expect(test_suite_spans.count).to eq(1)
+
+      test_spans.each_with_index do |span, index|
+        # test parameters are available since cucumber 4
+        if cucumber_4_or_above
+          expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("scenario with examples")
+
+          expect(span.get_tag(Datadog::CI::Ext::Test::TAG_PARAMETERS)).to eq(
+            "{\"arguments\":{\"num1\":\"#{index}\",\"num2\":\"#{index + 1}\",\"total\":\"#{index + index + 1}\"},\"metadata\":{}}"
+          )
+        else
+          expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq(
+            "scenario with examples, Examples (##{index + 1})"
+          )
+        end
+        expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
+          "spec/datadog/ci/contrib/cucumber/features/with_parameters.feature"
+        )
+        expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s)
+        expect(span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
+          Datadog::CI::Ext::Test::Status::PASS
+        )
+      end
+    end
+  end
+
   context "executing several features at once" do
     let(:expected_test_run_code) { 2 }
 
     let(:passing_test_suite) { test_suite_spans.find { |span| span.name =~ /passing/ } }
     let(:failing_test_suite) { test_suite_spans.find { |span| span.name =~ /failing/ } }
 
     it "creates a test suite span for each feature" do
-      expect(test_suite_spans.count).to eq(2)
+      expect(test_suite_spans.count).to eq(3)
       expect(passing_test_suite.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
         Datadog::CI::Ext::Test::Status::PASS
       )

spec/datadog/ci/span_spec.rb

@@ -228,4 +228,16 @@
       expect(span.span_type).to eq("test")
     end
   end
+
+  describe "#set_parameters" do
+    let(:parameters) { {"foo" => "bar", "baz" => "qux"} }
+
+    it "sets the parameters" do
+      expect(tracer_span).to receive(:set_tag).with(
+        "test.parameters", JSON.generate({arguments: parameters, metadata: {}})
+      )
+
+      span.set_parameters(parameters)
+    end
+  end
 end

vendor/rbs/cucumber/0/cucumber.rbs

@@ -11,9 +11,19 @@ class Cucumber::Runtime
   def formatters: () -> untyped
 end
 
+module Cucumber::Formatter
+end
+
+
 class Cucumber::Core::Test::Result
   def failed?: () -> bool
   def ok?: () -> bool
   def skipped?: () -> bool
   def exception: () -> untyped
+end
+
+class Cucumber::Formatter::AstLookup
+  def initialize: (untyped config) -> void
+
+  def scenario_source: (untyped test_case) -> untyped
 end