Cod 1210 support testifyquicktest in go benchmarks

compat/testing/slogtest/slogtest.go

@@ -0,0 +1,18 @@
+//go:build codspeed
+// +build codspeed
+
+package slogtest
+
+import (
+	"log/slog"
+
+	testing "github.com/CodSpeedHQ/codspeed-go/testing/testing"
+	slogtest "github.com/CodSpeedHQ/codspeed-go/testing/testing/slogtest"
+)
+
+func Run(t *testing.T, newHandler func(*testing.T) slog.Handler, result func(*testing.T) map[string]any) {
+	slogtest.Run(t, newHandler, result)
+}
+func TestHandler(h slog.Handler, results func() []map[string]any) error {
+	return slogtest.TestHandler(h, results)
+}

compat/testing/slogtest/slogtest_compat.go

@@ -0,0 +1,18 @@
+//go:build !codspeed
+// +build !codspeed
+
+// https://pkg.go.dev/testing/slogtest
+package slogtest
+
+import (
+	"log/slog"
+	"testing"
+	"testing/slogtest"
+)
+
+func Run(t *testing.T, newHandler func(*testing.T) slog.Handler, result func(*testing.T) map[string]any) {
+	slogtest.Run(t, newHandler, result)
+}
+func TestHandler(h slog.Handler, results func() []map[string]any) error {
+	return slogtest.TestHandler(h, results)
+}

compat/testing/testing.go

@@ -68,3 +68,7 @@ func Testing() bool {
 func Verbose() bool {
 	return codspeed_testing.Verbose()
 }
+
+func Benchmark(f func(b *B)) BenchmarkResult {
+	return codspeed_testing.Benchmark(f)
+}

compat/testing/testing_compat.go

@@ -68,3 +68,7 @@ func Testing() bool {
 func Verbose() bool {
 	return stdtesting.Verbose()
 }
+
+func Benchmark(f func(b *B)) BenchmarkResult {
+	return stdtesting.Benchmark(f)
+}

example-codspeed/cli/runner.go

@@ -13,6 +13,7 @@ import (
 
 	// Import parent package containing the benchmarks
 	example "example"
+	compat "example/compat"
 )
 
 // TestDeps is an implementation of the testing.testDeps interface,
@@ -83,6 +84,13 @@ func main() {
 			Name: "BenchmarkFibonacci20",
 			F:    example.BenchmarkFibonacci20,
 		},
+		{
+			Name: "BenchmarkWithSlogAssert",
+			F:    compat.BenchmarkWithSlogAssert,
+		}, {
+			Name: "BenchmarkQuicktest",
+			F:    compat.BenchmarkQuicktest,
+		},
 	}
 
 	m := codspeed_testing.MainStart(TestDeps{}, tests, benchmarks, fuzzTargets, examples)

example-codspeed/compat/quicktest_codspeed.go

@@ -0,0 +1,25 @@
+package compat
+
+import (
+	testing "github.com/CodSpeedHQ/codspeed-go/compat/testing"
+	qt "github.com/CodSpeedHQ/codspeed-go/pkg/quicktest"
+)
+
+func TestQuicktest(t *testing.T) {
+	t.Run("numbers", func(t *testing.T) {
+		c := qt.New(t)
+
+		c.Assert("hello world", qt.Contains, "world")
+		c.Assert([]int{3, 5, 7, 99}, qt.Contains, 7)
+		c.Assert([]int{3, 5, 8}, qt.All(qt.Not(qt.Equals)), 0)
+	})
+}
+
+func BenchmarkQuicktest(b *testing.B) {
+	for b.Loop() {
+		c := qt.New(b)
+		c.Assert("hello world", qt.Contains, "world")
+		c.Assert([]int{3, 5, 7, 99}, qt.Contains, 7)
+		c.Assert([]int{3, 5, 8}, qt.All(qt.Not(qt.Equals)), 0)
+	}
+}

example-codspeed/compat/slogassert_codspeed.go

@@ -0,0 +1,24 @@
+package compat
+
+import (
+	"log/slog"
+
+	testing "github.com/CodSpeedHQ/codspeed-go/compat/testing"
+	slogassert "github.com/CodSpeedHQ/codspeed-go/pkg/slogassert"
+)
+
+func TestWithSlogAssert(t *testing.T) {
+	handler := slogassert.NewDefault(t)
+	slog.Info("This is a test log message")
+	handler.AssertMessage("This is a test log message")
+	handler.AssertEmpty()
+}
+
+func BenchmarkWithSlogAssert(b *testing.B) {
+	for b.Loop() {
+		handler := slogassert.NewDefault(b)
+		slog.Info("This is a test log message")
+		handler.AssertMessage("This is a test log message")
+		handler.AssertEmpty()
+	}
+}

example-codspeed/go.mod

@@ -2,6 +2,13 @@ module example
 
 go 1.24.3
 
-require github.com/CodSpeedHQ/codspeed-go v0.0.0
+require github.com/CodSpeedHQ/codspeed-go v0.1.2
+
+require (
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+)
 
 replace github.com/CodSpeedHQ/codspeed-go => ..

example-codspeed/go.sum

@@ -0,0 +1,10 @@
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=

example/compat/quicktest_test.go

@@ -0,0 +1,26 @@
+package example
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestQuicktest(t *testing.T) {
+	t.Run("numbers", func(t *testing.T) {
+		c := qt.New(t)
+
+		c.Assert("hello world", qt.Contains, "world")
+		c.Assert([]int{3, 5, 7, 99}, qt.Contains, 7)
+		c.Assert([]int{3, 5, 8}, qt.All(qt.Not(qt.Equals)), 0)
+	})
+}
+
+func BenchmarkQuicktest(b *testing.B) {
+	for b.Loop() {
+		c := qt.New(b)
+		c.Assert("hello world", qt.Contains, "world")
+		c.Assert([]int{3, 5, 7, 99}, qt.Contains, 7)
+		c.Assert([]int{3, 5, 8}, qt.All(qt.Not(qt.Equals)), 0)
+	}
+}

example/compat/slogassert_test.go

@@ -0,0 +1,24 @@
+package example
+
+import (
+	"log/slog"
+	"testing"
+
+	"github.com/thejerf/slogassert"
+)
+
+func TestWithSlogAssert(t *testing.T) {
+	handler := slogassert.NewDefault(t)
+	slog.Info("This is a test log message")
+	handler.AssertMessage("This is a test log message")
+	handler.AssertEmpty()
+}
+
+func BenchmarkWithSlogAssert(b *testing.B) {
+	for b.Loop() {
+		handler := slogassert.NewDefault(b)
+		slog.Info("This is a test log message")
+		handler.AssertMessage("This is a test log message")
+		handler.AssertEmpty()
+	}
+}

example/compat/testify_test.go

@@ -0,0 +1,60 @@
+package example
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTestifyWithT(t *testing.T) {
+	useAssertWithT(t)
+	useRequireWithT(t)
+}
+
+func TestTestifyWithNew(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	useAssert(assert)
+	useRequire(require)
+}
+
+func BenchmarkTestifyWithT(b *testing.B) {
+	for b.Loop() {
+		useAssertWithT(b)
+		useRequireWithT(b)
+	}
+}
+
+func BenchmarkTestifyWithNew(b *testing.B) {
+	assert := assert.New(b)
+	require := require.New(b)
+
+	for b.Loop() {
+		useAssert(assert)
+		useRequire(require)
+	}
+}
+
+func useAssertWithT(t assert.TestingT) {
+	assert.Equal(t, 42, 42, "they should be equal")
+	assert.NotEqual(t, 42, 24, "they should not be equal")
+	assert.True(t, true, "True is true!")
+}
+
+func useAssert(assert *assert.Assertions) {
+	assert.Equal(42, 42, "they should be equal")
+	assert.NotEqual(42, 24, "they should not be equal")
+	assert.True(true, "True is true!")
+}
+
+func useRequireWithT(t require.TestingT) {
+	require.Equal(t, "hello", "hello", "they should be equal")
+	require.NotEqual(t, "hello", "world", "they should not be equal")
+}
+
+func useRequire(require *require.Assertions) {
+	require.Equal("hello", "hello", "they should be equal")
+	require.NotEqual("hello", "world", "they should not be equal")
+}

example/go.mod

@@ -1,3 +1,19 @@
 module example
 
 go 1.24.3
+
+require (
+	github.com/frankban/quicktest v1.14.6
+	github.com/stretchr/testify v1.11.1
+	github.com/thejerf/slogassert v0.3.4
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

example/go.sum

@@ -0,0 +1,24 @@
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/thejerf/slogassert v0.3.4 h1:VoTsXixRbXMrRSSxDjYTiEDCM4VWbsYPW5rB/hX24kM=
+github.com/thejerf/slogassert v0.3.4/go.mod h1:0zn9ISLVKo1aPMTqcGfG1o6dWwt+Rk574GlUxHD4rs8=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go-runner/src/builder/discovery.rs

@@ -7,7 +7,6 @@ use std::{
     process::Command,
 };
 
-use crate::builder::verifier;
 use crate::prelude::*;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
@@ -79,7 +78,7 @@ impl GoPackage {
         };
 
         let mut benchmarks = Vec::new();
-        'file_loop: for file in test_go_files.iter().sorted() {
+        for file in test_go_files.iter().sorted() {
             assert!(file.ends_with("_test.go"));
 
             let file_path = self.dir.join(file);
@@ -94,22 +93,6 @@ impl GoPackage {
                 }
             };
 
-            // Check for unsupported imports
-            const UNSUPPORTED_IMPORTS: &[(&str, &str)] = &[
-                ("github.com/frankban/quicktest", "quicktest"),
-                ("github.com/stretchr/testify", "testify"),
-            ];
-            for (import_path, import_name) in UNSUPPORTED_IMPORTS {
-                if file
-                    .imports
-                    .iter()
-                    .any(|import| import.path.value.contains(import_path))
-                {
-                    warn!("Skipping file with {import_name} import: {file_path:?}");
-                    continue 'file_loop;
-                }
-            }
-
             // We can't import packages that are declared as `main`
             if file.pkg_name.name == "main" {
                 warn!("Skipping file with main package: {file_path:?}");
@@ -147,27 +130,7 @@ impl GoPackage {
                 format!("Couldn't strip the module dir from file path: {file_path:?}"),
             )?;
 
-            let valid_benchmarks =
-                verifier::FuncVisitor::verify_source_code(&content, &found_benchmarks)?;
-            if valid_benchmarks.len() != found_benchmarks.len() {
-                warn!(
-                    "Only {} out of {} are valid, skipping file",
-                    valid_benchmarks.len(),
-                    found_benchmarks.len()
-                );
-                warn!("Valid benchmarks: {valid_benchmarks:?}");
-                warn!(
-                    "Invalid benchmarks: {:?}",
-                    found_benchmarks
-                        .iter()
-                        .filter(|f| !valid_benchmarks.contains(f))
-                        .collect::<Vec<_>>()
-                );
-
-                continue;
-            }
-
-            for func in valid_benchmarks.into_iter().sorted() {
+            for func in found_benchmarks.into_iter().sorted() {
                 benchmarks.push(GoBenchmark::new(
                     package_import_path.clone(),
                     func,
@@ -237,7 +200,11 @@ impl BenchmarkPackage {
         go_project_path: &Path,
         packages: &[String],
     ) -> anyhow::Result<Vec<BenchmarkPackage>> {
-        let raw_packages = Self::run_go_list(go_project_path, packages)?;
+        let mut raw_packages = Self::run_go_list(go_project_path, packages)?;
+
+        // Sort packages by import path to ensure deterministic order
+        raw_packages.sort_by(|a, b| a.import_path.cmp(&b.import_path));
+
         let has_test_files =
             |files: &Vec<String>| files.iter().any(|name| name.ends_with("_test.go"));
         let has_test_imports = |imports: &Vec<String>| {