From 01f9256b40c2c2b2f79c6f962279b4a9eb5e11eb Mon Sep 17 00:00:00 2001 From: Onsi Fakhouri Date: Wed, 6 Oct 2021 11:27:33 -0600 Subject: [PATCH] Ginkgo correctly handles tests suites that filter out all tests via tags Fixes #427 Fixes #495 --- ginkgo/command/abort.go | 8 +++ ginkgo/internal/compile.go | 6 ++ ginkgo/internal/profiles_and_reports.go | 5 +- ginkgo/internal/run.go | 3 +- ginkgo/internal/test_suite.go | 2 + ginkgo/run/run_command.go | 10 +-- .../_fixtures/no_tagged_tests_fixture/doc.go | 1 + .../no_tagged_tests_fixture_suite_test.go | 15 ++++ integration/run_test.go | 72 ++++++++++++------- 9 files changed, 91 insertions(+), 31 deletions(-) create mode 100644 integration/_fixtures/no_tagged_tests_fixture/doc.go create mode 100644 integration/_fixtures/no_tagged_tests_fixture/no_tagged_tests_fixture_suite_test.go diff --git a/ginkgo/command/abort.go b/ginkgo/command/abort.go index 6f3542b8e..2efd28608 100644 --- a/ginkgo/command/abort.go +++ b/ginkgo/command/abort.go @@ -12,6 +12,14 @@ func Abort(details AbortDetails) { panic(details) } +func AbortGracefullyWith(format string, args ...interface{}) { + Abort(AbortDetails{ + ExitCode: 0, + Error: fmt.Errorf(format, args...), + EmitUsage: false, + }) +} + func AbortWith(format string, args ...interface{}) { Abort(AbortDetails{ ExitCode: 1, diff --git a/ginkgo/internal/compile.go b/ginkgo/internal/compile.go index 8f37c4486..48671730c 100644 --- a/ginkgo/internal/compile.go +++ b/ginkgo/internal/compile.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "sync" "github.com/onsi/ginkgo/types" @@ -45,6 +46,11 @@ func CompileSuite(suite TestSuite, goFlagsConfig types.GoFlagsConfig) TestSuite return suite } + if strings.Contains(string(output), "[no test files]") { + suite.State = TestSuiteStateSkippedDueToEmptyCompilation + return suite + } + if len(output) > 0 { fmt.Println(string(output)) } diff --git a/ginkgo/internal/profiles_and_reports.go b/ginkgo/internal/profiles_and_reports.go index c667d3501..e4a5b5e33 100644 --- a/ginkgo/internal/profiles_and_reports.go +++ b/ginkgo/internal/profiles_and_reports.go @@ -104,7 +104,7 @@ func FinalizeProfilesAndReportsForSuites(suites TestSuites, cliConfig types.CLIC } reportableSuites := suites.ThatAreGinkgoSuites() - for _, suite := range reportableSuites.WithState(TestSuiteStateFailedToCompile, TestSuiteStateFailedDueToTimeout, TestSuiteStateSkippedDueToPriorFailures) { + for _, suite := range reportableSuites.WithState(TestSuiteStateFailedToCompile, TestSuiteStateFailedDueToTimeout, TestSuiteStateSkippedDueToPriorFailures, TestSuiteStateSkippedDueToEmptyCompilation) { report := types.Report{ SuitePath: suite.AbsPath(), SuiteConfig: suiteConfig, @@ -117,6 +117,9 @@ func FinalizeProfilesAndReportsForSuites(suites TestSuites, cliConfig types.CLIC report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, TIMEOUT_ELAPSED_FAILURE_REASON) case TestSuiteStateSkippedDueToPriorFailures: report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, PRIOR_FAILURES_FAILURE_REASON) + case TestSuiteStateSkippedDueToEmptyCompilation: + report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, EMPTY_SKIP_FAILURE_REASON) + report.SuiteSucceeded = true } for _, format := range reportFormats { diff --git a/ginkgo/internal/run.go b/ginkgo/internal/run.go index bb10f1e96..2ed4738b9 100644 --- a/ginkgo/internal/run.go +++ b/ginkgo/internal/run.go @@ -192,9 +192,10 @@ func runParallel(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig fmt.Fprintln(os.Stderr, "** Ginkgo timed out waiting for all parallel nodes to report back. **") fmt.Fprintf(os.Stderr, "%s (%s)\n", suite.PackageName, suite.Path) for node := 1; node <= cliConfig.ComputedNodes(); node++ { - fmt.Fprintf(os.Stderr, "Output from node %d\n:", node) + fmt.Fprintf(os.Stderr, "Output from node %d:\n", node) fmt.Fprintln(os.Stderr, formatter.Fi(1, "%s", nodeOutput[node-1].String())) } + fmt.Fprintf(os.Stderr, "** End **") } for node := 1; node <= cliConfig.ComputedNodes(); node++ { diff --git a/ginkgo/internal/test_suite.go b/ginkgo/internal/test_suite.go index e42b3a5a0..c1c72ffd2 100644 --- a/ginkgo/internal/test_suite.go +++ b/ginkgo/internal/test_suite.go @@ -14,6 +14,7 @@ import ( const TIMEOUT_ELAPSED_FAILURE_REASON = "Suite did not run because the timeout elapsed" const PRIOR_FAILURES_FAILURE_REASON = "Suite did not run because prior suites failed and --keep-going is not set" +const EMPTY_SKIP_FAILURE_REASON = "Suite did not run go test reported that no test files were found" type TestSuiteState uint @@ -25,6 +26,7 @@ const ( TestSuiteStatePassed + TestSuiteStateSkippedDueToEmptyCompilation TestSuiteStateSkippedByFilter TestSuiteStateSkippedDueToPriorFailures diff --git a/ginkgo/run/run_command.go b/ginkgo/run/run_command.go index 85b81521b..d8247b0ae 100644 --- a/ginkgo/run/run_command.go +++ b/ginkgo/run/run_command.go @@ -77,10 +77,7 @@ func (r *SpecRunner) RunSpecs(args []string, additionalArgs []string) { } if len(skippedSuites) > 0 && len(suites) == 0 { - command.Abort(command.AbortDetails{ - ExitCode: 0, - Error: fmt.Errorf("All tests skipped! Exiting..."), - }) + command.AbortGracefullyWith("All tests skipped! Exiting...") } if len(suites) == 0 { @@ -123,6 +120,11 @@ OUTER_LOOP: break OUTER_LOOP } + if suites[suiteIdx].State.Is(internal.TestSuiteStateSkippedDueToEmptyCompilation) { + fmt.Printf("Skipping %s (no test files)\n", suite.Path) + continue SUITE_LOOP + } + if suites[suiteIdx].State.Is(internal.TestSuiteStateFailedToCompile) { fmt.Println(suites[suiteIdx].CompilationError.Error()) if !r.cliConfig.KeepGoing { diff --git a/integration/_fixtures/no_tagged_tests_fixture/doc.go b/integration/_fixtures/no_tagged_tests_fixture/doc.go new file mode 100644 index 000000000..3bac806b3 --- /dev/null +++ b/integration/_fixtures/no_tagged_tests_fixture/doc.go @@ -0,0 +1 @@ +package notaggedtestsfixture diff --git a/integration/_fixtures/no_tagged_tests_fixture/no_tagged_tests_fixture_suite_test.go b/integration/_fixtures/no_tagged_tests_fixture/no_tagged_tests_fixture_suite_test.go new file mode 100644 index 000000000..baad00e09 --- /dev/null +++ b/integration/_fixtures/no_tagged_tests_fixture/no_tagged_tests_fixture_suite_test.go @@ -0,0 +1,15 @@ +//+build integration + +package notaggedtestsfixture + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestNoTaggedTestsFixture(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "No Tagged Tests Fixture Suite") +} diff --git a/integration/run_test.go b/integration/run_test.go index b3c6ded46..b90e0c630 100644 --- a/integration/run_test.go +++ b/integration/run_test.go @@ -226,6 +226,22 @@ var _ = Describe("Running Specs", func() { }) }) + Context("when pointed at a package with tests, but the tests have been excluded via go tags", func() { + BeforeEach(func() { + fm.MountFixture("no_tagged_tests") + }) + + It("should exit 0, not run anything, and generate a json report if asked", func() { + session := startGinkgo(fm.PathTo("no_tagged_tests"), "--no-color", "--json-report=report.json") + Eventually(session).Should(gexec.Exit(0)) + Ω(session).Should(gbytes.Say("no test files")) + + report := fm.LoadJSONReports("no_tagged_tests", "report.json")[0] + Ω(report.SuiteSucceeded).Should(BeTrue()) + Ω(report.SpecialSuiteFailureReasons).Should(ConsistOf("Suite did not run go test reported that no test files were found")) + }) + }) + Context("when pointed at a package that fails to compile", func() { BeforeEach(func() { fm.MountFixture("does_not_compile") @@ -311,6 +327,7 @@ var _ = Describe("Running Specs", func() { BeforeEach(func() { fm.MountFixture("passing_ginkgo_tests") fm.MountFixture("more_ginkgo_tests") + fm.MountFixture("no_tagged_tests") }) Context("when all the tests pass", func() { @@ -322,7 +339,8 @@ var _ = Describe("Running Specs", func() { outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) - Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) + Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) + Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) @@ -334,7 +352,8 @@ var _ = Describe("Running Specs", func() { outputLines := strings.Split(output, "\n") Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) - Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) + Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) + Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Passed")) }) }) @@ -346,13 +365,14 @@ var _ = Describe("Running Specs", func() { }) It("should fail and stop running tests", func() { - session := startGinkgo(fm.TmpDir, "--no-color", "passing_ginkgo_tests", "failing_ginkgo_tests", "more_ginkgo_tests") + session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "failing_ginkgo_tests", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") - Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) - Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) + Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) + Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) + Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter))) Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Failed")) @@ -365,13 +385,14 @@ var _ = Describe("Running Specs", func() { }) It("should fail and stop running tests", func() { - session := startGinkgo(fm.TmpDir, "--no-color", "passing_ginkgo_tests", "does_not_compile", "more_ginkgo_tests") + session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") - Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) - Ω(outputLines[1]).Should(ContainSubstring("Failed to compile does_not_compile:")) + Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) + Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) + Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:")) Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) Ω(output).Should(ContainSubstring("Test Suite Failed")) }) @@ -384,37 +405,38 @@ var _ = Describe("Running Specs", func() { }) It("should soldier on", func() { - session := startGinkgo(fm.TmpDir, "--no-color", "-keep-going", "passing_ginkgo_tests", "does_not_compile", "failing_ginkgo_tests", "more_ginkgo_tests") + session := startGinkgo(fm.TmpDir, "--no-color", "-keep-going", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "failing_ginkgo_tests", "more_ginkgo_tests") Eventually(session).Should(gexec.Exit(1)) output := string(session.Out.Contents()) outputLines := strings.Split(output, "\n") - Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) - Ω(outputLines[1]).Should(ContainSubstring("Failed to compile does_not_compile:")) + Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) + Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) + Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:")) Ω(output).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter))) Ω(output).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) Ω(output).Should(ContainSubstring("Test Suite Failed")) }) }) + }) - Context("when running large suites in parallel", Label("slow"), func() { - BeforeEach(func() { - fm.MountFixture("large") - }) + Context("when running large suites in parallel", Label("slow"), func() { + BeforeEach(func() { + fm.MountFixture("large") + }) - It("doesn't miss any tests (a sanity test)", func() { - session := startGinkgo(fm.PathTo("large"), "--no-color", "--nodes=3", "--json-report=report.json") - Eventually(session).Should(gexec.Exit(0)) - report := Reports(fm.LoadJSONReports("large", "report.json")[0].SpecReports) + It("doesn't miss any tests (a sanity test)", func() { + session := startGinkgo(fm.PathTo("large"), "--no-color", "--nodes=3", "--json-report=report.json") + Eventually(session).Should(gexec.Exit(0)) + report := Reports(fm.LoadJSONReports("large", "report.json")[0].SpecReports) - expectedNames := []string{} - for i := 0; i < 2048; i++ { - expectedNames = append(expectedNames, fmt.Sprintf("%d", i)) - } - Ω(report.Names()).Should(ConsistOf(expectedNames)) + expectedNames := []string{} + for i := 0; i < 2048; i++ { + expectedNames = append(expectedNames, fmt.Sprintf("%d", i)) + } + Ω(report.Names()).Should(ConsistOf(expectedNames)) - }) }) }) })