Run tests using appropriate binary

Currently we support two binaries:
1. openshift-tests one, which is the current way how to run tests
2. k8s-tests, which allows running k8s native tests

All the variables required to run tests are injected through environment
variables.

pkg/test/ginkgo/cmd_runsuite.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -15,6 +16,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/onsi/ginkgo/v2"
+
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/openshift/origin/pkg/monitor"
@@ -160,6 +163,19 @@ func (opt *Options) Run(suite *TestSuite, junitSuiteName string) error {
 		return err
 	}
 
+	externalTests, err := externalTestsForSuite(ctx)
+	if err != nil {
+		return err
+	}
+	// TODO: origin's rules.go need to by applied against external tests suites
+	// TODO: should we stop generating hardcoded annotations in favor of always
+	// dynamically applying the lables?
+
+	tests = append(tests, externalTests...)
+	suiteConfig, _ := ginkgo.GinkgoConfiguration()
+	r := rand.New(rand.NewSource(suiteConfig.RandomSeed))
+	r.Shuffle(len(tests), func(i, j int) { tests[i], tests[j] = tests[j], tests[i] })
+
 	discoveryClient, err := getDiscoveryClient()
 	if err != nil {
 		if opt.DryRun {

pkg/test/ginkgo/external.go

@@ -0,0 +1,57 @@
+package ginkgo
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/openshift/origin/test/extended/util/annotate/generated"
+)
+
+type serializedTest struct {
+	Name string
+}
+
+func externalTestsForSuite(ctx context.Context) ([]*testCase, error) {
+	var tests []*testCase
+
+	// TODO: make the binary name configurable
+	testBinary := filepath.Dir(os.Args[0]) + "/k8s-tests"
+	command := exec.Command(testBinary, "list")
+
+	testList, err := runWithTimeout(ctx, command, 1*time.Minute)
+	if err != nil {
+		return nil, err
+	}
+	buf := bytes.NewBuffer(testList)
+	for {
+		line, err := buf.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if !strings.HasPrefix(line, "[{") {
+			continue
+		}
+		serializedTests := []serializedTest{}
+		err = json.Unmarshal([]byte(line), &serializedTests)
+		if err != nil {
+			return nil, err
+		}
+		for _, test := range serializedTests {
+			if append, ok := generated.Annotations[test.Name]; ok {
+				test.Name += append
+			}
+			tests = append(tests, &testCase{
+				name:       test.Name,
+				binaryName: testBinary,
+			})
+		}
+	}
+	return tests, nil
+}

pkg/test/ginkgo/ginkgo.go

@@ -1,8 +1,6 @@
 package ginkgo
 
 import (
-	"math/rand"
-
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/ginkgo/v2/types"
 
@@ -33,8 +31,5 @@ func testsForSuite() ([]*testCase, error) {
 	if len(errs) > 0 {
 		return nil, errors.NewAggregate(errs)
 	}
-	suiteConfig, _ := ginkgo.GinkgoConfiguration()
-	r := rand.New(rand.NewSource(suiteConfig.RandomSeed))
-	r.Shuffle(len(tests), func(i, j int) { tests[i], tests[j] = tests[j], tests[i] })
 	return tests, nil
 }

pkg/test/ginkgo/test.go

@@ -12,10 +12,11 @@ import (
 )
 
 type testCase struct {
-	name      string
-	spec      types.TestSpec
-	locations []types.CodeLocation
-	apigroups []string
+	name       string
+	binaryName string
+	spec       types.TestSpec
+	locations  []types.CodeLocation
+	apigroups  []string
 
 	// identifies which tests can be run in parallel (ginkgo runs suites linearly)
 	testExclusion string

pkg/test/ginkgo/test_runner.go

@@ -3,6 +3,7 @@ package ginkgo
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -12,8 +13,11 @@ import (
 	"syscall"
 	"time"
 
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+
 	"github.com/openshift/origin/pkg/monitor"
 	"github.com/openshift/origin/pkg/monitor/monitorapi"
+	exutilcluster "github.com/openshift/origin/test/extended/util/cluster"
 )
 
 type testSuiteRunner interface {
@@ -191,11 +195,16 @@ func newCommandContext(env []string, timeout time.Duration) *commandContext {
 
 func (c *commandContext) commandString(test *testCase) string {
 	buf := &bytes.Buffer{}
-	for _, env := range c.env {
+	envs := updateEnvVars(c.env)
+	for _, env := range envs {
 		parts := strings.SplitN(env, "=", 2)
 		fmt.Fprintf(buf, "%s=%q ", parts[0], parts[1])
 	}
-	fmt.Fprintf(buf, "%s %s %q", os.Args[0], "run-test", test.name)
+	testBinary := test.binaryName
+	if len(testBinary) == 0 {
+		testBinary = os.Args[0]
+	}
+	fmt.Fprintf(buf, "%s %s %q", testBinary, "run-test", test.name)
 	return buf.String()
 }
 
@@ -287,9 +296,15 @@ func (c *commandContext) RunTestInNewProcess(ctx context.Context, test *testCase
 		return ret
 	}
 
+	fmt.Printf(">>> running %s\n\n", c.commandString(test))
+
 	ret.start = time.Now()
-	command := exec.Command(os.Args[0], "run-test", test.name)
-	command.Env = append(os.Environ(), c.env...)
+	testBinary := test.binaryName
+	if len(testBinary) == 0 {
+		testBinary = os.Args[0]
+	}
+	command := exec.Command(testBinary, "run-test", test.name)
+	command.Env = append(os.Environ(), updateEnvVars(c.env)...)
 
 	timeout := c.timeout
 	if test.testTimeout != 0 {
@@ -299,6 +314,8 @@ func (c *commandContext) RunTestInNewProcess(ctx context.Context, test *testCase
 	testOutputBytes, err := runWithTimeout(ctx, command, timeout)
 	ret.end = time.Now()
 
+	fmt.Printf(">>> output:\n%s\n\n", testOutputBytes)
+
 	ret.testOutputBytes = testOutputBytes
 	if err == nil {
 		ret.testState = TestSucceeded
@@ -334,6 +351,28 @@ func (c *commandContext) RunTestInNewProcess(ctx context.Context, test *testCase
 	return ret
 }
 
+func updateEnvVars(envs []string) []string {
+	result := []string{}
+	for _, env := range envs {
+		if !strings.HasPrefix(env, "TEST_PROVIDER") {
+			result = append(result, env)
+		}
+	}
+	// copied from provider.go
+	// TODO: add error handling, and maybe turn this into sharable helper?
+	clientConfig, _ := e2e.LoadConfig(true)
+	clusterState, _ := exutilcluster.DiscoverClusterState(clientConfig)
+	config, _ := exutilcluster.LoadConfig(clusterState)
+	if len(config.ProviderName) == 0 {
+		config.ProviderName = "skeleton"
+	}
+	provider, _ := json.Marshal(config)
+	result = append(result, fmt.Sprintf("TEST_PROVIDER=%s", provider))
+	// TODO: do we need to inject KUBECONFIG?
+	// result = append(result, "KUBECONFIG=%s", )
+	return result
+}
+
 func runWithTimeout(ctx context.Context, c *exec.Cmd, timeout time.Duration) ([]byte, error) {
 	if timeout > 0 {
 		go func() {