Merge pull request #105 from DataDog/anmarchenko/readable_test_suite_…

…names

[CIVIS-8548] make test suite names more readable

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -54,7 +54,7 @@ def on_test_run_finished(event)
           end
 
           def on_test_case_started(event)
-            test_suite_name = event.test_case.location.file
+            test_suite_name = test_suite_name(event.test_case)
 
             tags = {
               CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
@@ -108,6 +108,21 @@ def on_test_step_finished(event)
 
           private
 
+          def test_suite_name(test_case)
+            feature = if test_case.respond_to?(:feature)
+              test_case.feature
+            elsif @ast_lookup
+              gherkin_doc = @ast_lookup.gherkin_document(test_case.location.file)
+              gherkin_doc.feature if gherkin_doc
+            end
+
+            if feature
+              "#{feature.name} at #{test_case.location.file}"
+            else
+              test_case.location.file
+            end
+          end
+
           def finish_test(span, result)
             if result.skipped?
               span.skipped!

lib/datadog/ci/contrib/minitest/hooks.rb

@@ -14,8 +14,6 @@ def before_setup
             super
             return unless datadog_configuration[:enabled]
 
-            test_name = "#{class_name}##{name}"
-
             test_suite_name = Helpers.test_suite_name(self.class, name)
             if Helpers.parallel?(self.class)
               test_suite_name = "#{test_suite_name} (#{name} concurrently)"
@@ -27,7 +25,7 @@ def before_setup
             source_file, line_number = method(name).source_location
 
             CI.start_test(
-              test_name,
+              name,
               test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,

lib/datadog/ci/contrib/rspec/example.rb

@@ -14,28 +14,33 @@ def self.included(base)
             base.prepend(InstanceMethods)
           end
 
-          # Instance methods for configuration
           module InstanceMethods
-            def run(example_group_instance, reporter)
-              return super unless configuration[:enabled]
+            def run(*)
+              return super unless datadog_configuration[:enabled]
 
               test_name = full_description.strip
               if metadata[:description].empty?
                 # for unnamed it blocks this appends something like "example at ./spec/some_spec.rb:10"
                 test_name += " #{description}"
               end
 
+              test_suite_description = fetch_top_level_example_group[:description]
+              suite_name = "#{test_suite_description} at #{metadata[:example_group][:rerun_file_path]}"
+
+              # remove suite name from test name to avoid duplication
+              test_name = test_name.sub(test_suite_description, "").strip
+
               CI.trace_test(
                 test_name,
-                metadata[:example_group][:rerun_file_path],
+                suite_name,
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
                   CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
                   CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]),
                   CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               ) do |test_span|
                 test_span.set_parameters({}, {"scoped_id" => metadata[:scoped_id]})
 
@@ -56,7 +61,17 @@ def run(example_group_instance, reporter)
 
             private
 
-            def configuration
+            def fetch_top_level_example_group
+              return metadata[:example_group] unless metadata[:example_group][:parent_example_group]
+
+              res = metadata[:example_group][:parent_example_group]
+              while (parent = res[:parent_example_group])
+                res = parent
+              end
+              res
+            end
+
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

lib/datadog/ci/contrib/rspec/example_group.rb

@@ -7,19 +7,20 @@ module Datadog
   module CI
     module Contrib
       module RSpec
-        # Instrument RSpec::Core::Example
+        # Instrument RSpec::Core::ExampleGroup
         module ExampleGroup
           def self.included(base)
             base.singleton_class.prepend(ClassMethods)
           end
 
           # Instance methods for configuration
           module ClassMethods
-            def run(reporter = ::RSpec::Core::NullReporter)
-              return super unless configuration[:enabled]
+            def run(*)
+              return super unless datadog_configuration[:enabled]
               return super unless top_level?
 
-              test_suite = Datadog::CI.start_test_suite(file_path)
+              suite_name = "#{description} at #{file_path}"
+              test_suite = Datadog::CI.start_test_suite(suite_name)
 
               result = super
 
@@ -35,7 +36,7 @@ def run(reporter = ::RSpec::Core::NullReporter)
 
             private
 
-            def configuration
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

lib/datadog/ci/contrib/rspec/runner.rb

@@ -14,16 +14,16 @@ def self.included(base)
           end
 
           module InstanceMethods
-            def run_specs(example_groups)
-              return super unless configuration[:enabled]
+            def run_specs(*)
+              return super unless datadog_configuration[:enabled]
 
               test_session = CI.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
                   CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
                 },
-                service: configuration[:service_name]
+                service: datadog_configuration[:service_name]
               )
 
               test_module = CI.start_test_module(test_session.name)
@@ -46,7 +46,7 @@ def run_specs(example_groups)
 
             private
 
-            def configuration
+            def datadog_configuration
               Datadog.configuration.ci[:rspec]
             end
           end

sig/datadog/ci/contrib/cucumber/formatter.rbs

@@ -30,6 +30,8 @@ module Datadog
 
           private
 
+          def test_suite_name: (untyped test_case) -> String
+
           def start_test_suite: (String test_suite_name) -> void
 
           def finish_current_test_suite: () -> void

sig/datadog/ci/contrib/rspec/example.rbs

@@ -11,7 +11,8 @@ module Datadog
 
             private
 
-            def configuration: () -> untyped
+            def fetch_top_level_example_group: () -> Hash[Symbol, untyped]
+            def datadog_configuration: () -> untyped
           end
         end
       end

sig/datadog/ci/contrib/rspec/example_group.rbs

@@ -12,7 +12,7 @@ module Datadog
 
             private
 
-            def configuration: () -> untyped
+            def datadog_configuration: () -> untyped
           end
         end
       end

sig/datadog/ci/contrib/rspec/runner.rbs

@@ -12,7 +12,7 @@ module Datadog
 
             private
 
-            def configuration: () -> untyped
+            def datadog_configuration: () -> untyped
           end
         end
       end

spec/datadog/ci/contrib/cucumber/features/failing.feature

@@ -1,4 +1,4 @@
-Feature: Datadog integration
+Feature: Datadog integration - test failing features
   Scenario: cucumber failing scenario
     Given datadog
     And datadog

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -80,7 +80,7 @@
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("cucumber scenario")
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
-        "spec/datadog/ci/contrib/cucumber/features/passing.feature"
+        "Datadog integration at spec/datadog/ci/contrib/cucumber/features/passing.feature"
       )
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
@@ -147,7 +147,7 @@
 
     it "creates test suite span" do
       expect(test_suite_span).not_to be_nil
-      expect(test_suite_span.name).to eq(features_path)
+      expect(test_suite_span.name).to eq("Datadog integration at spec/datadog/ci/contrib/cucumber/features/passing.feature")
       expect(test_suite_span.service).to eq("jalapenos")
       expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
         Datadog::CI::Ext::AppTypes::TYPE_TEST
@@ -189,7 +189,9 @@
         Datadog::CI::Ext::Test::Status::FAIL
       )
 
-      expect(test_suite_span.name).to eq(features_path)
+      expect(test_suite_span.name).to eq(
+        "Datadog integration - test failing features at spec/datadog/ci/contrib/cucumber/features/failing.feature"
+      )
       expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
         Datadog::CI::Ext::Test::Status::FAIL
       )
@@ -224,7 +226,7 @@
           )
         end
         expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
-          "spec/datadog/ci/contrib/cucumber/features/with_parameters.feature"
+          "Datadog integration for parametrized tests at spec/datadog/ci/contrib/cucumber/features/with_parameters.feature"
         )
         expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID)).to eq(test_suite_span.id.to_s)
         expect(span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -51,10 +51,10 @@ def test_foo
       klass.new(:test_foo).run
 
       expect(span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-      expect(span.name).to eq("SomeTest#test_foo")
-      expect(span.resource).to eq("SomeTest#test_foo")
+      expect(span.name).to eq("test_foo")
+      expect(span.resource).to eq("test_foo")
       expect(span.service).to eq("ltest")
-      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeTest#test_foo")
+      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("test_foo")
       expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
         "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
       )
@@ -110,9 +110,9 @@ def self.name
       klass.new(method_name).run
 
       expect(span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-      expect(span.resource).to eq("SomeSpec##{method_name}")
+      expect(span.resource).to eq(method_name)
       expect(span.service).to eq("ltest")
-      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeSpec##{method_name}")
+      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq(method_name)
       expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
         "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
       )
@@ -514,7 +514,7 @@ def test_fail
         end
 
         it "traces test, test session, test module with failed status" do
-          expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeFailedTest#test_fail")
+          expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("test_fail")
           expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(
             Datadog::CI::Ext::Test::Status::FAIL
           )
@@ -561,10 +561,10 @@ class SomeSpec < Minitest::Spec
 
           expect(test_names).to eq(
             [
-              "SomeSpec#test_0001_does not fail",
-              "in context#test_0001_does not fail",
-              "in context::deeper context#test_0001_does not fail",
-              "in other context#test_0001_does not fail"
+              "test_0001_does not fail",
+              "test_0001_does not fail",
+              "test_0001_does not fail",
+              "test_0001_does not fail"
             ]
           )
         end
@@ -635,10 +635,10 @@ def test_b_2
           test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort
           expect(test_names).to eq(
             [
-              "TestA#test_a_1",
-              "TestA#test_a_2",
-              "TestB#test_b_1",
-              "TestB#test_b_2"
+              "test_a_1",
+              "test_a_2",
+              "test_b_1",
+              "test_b_2"
             ]
           )