UPSTREAM: <carry>: Create minimal wrapper needed to run k8s e2e tests

UPSTREAM: <carry>: Change annotation mechanics to allow injecting testMaps and filter out tests

UPSTREAM: <carry>: Move k8s-specific rules to our fork

UPSTREAM: <carry>: Create minimal wrapper needed to run k8s e2e tests

hack/lib/golang.sh

@@ -81,6 +81,7 @@ kube::golang::server_targets() {
     vendor/k8s.io/apiextensions-apiserver
     cluster/gce/gci/mounter
     cmd/watch-termination
+    openshift-hack/cmd/k8s-tests
   )
   echo "${targets[@]}"
 }
@@ -440,7 +441,7 @@ kube::golang::set_platform_envs() {
 
   # if CC is defined for platform then always enable it
   ccenv=$(echo "$platform" | awk -F/ '{print "KUBE_" toupper($1) "_" toupper($2) "_CC"}')
-  if [ -n "${!ccenv-}" ]; then 
+  if [ -n "${!ccenv-}" ]; then
     export CGO_ENABLED=1
     export CC="${!ccenv}"
   fi

openshift-hack/cmd/k8s-tests/k8s-tests.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	utilflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	// initialize framework extensions
+	_ "k8s.io/kubernetes/test/e2e/framework/debug/init"
+	_ "k8s.io/kubernetes/test/e2e/framework/metrics/init"
+)
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+
+	root := &cobra.Command{
+		Long: "OpenShift Tests compatible wrapper",
+	}
+
+	root.AddCommand(
+		newRunTestCommand(),
+		newListTestsCommand(),
+	)
+
+	f := flag.CommandLine.Lookup("v")
+	root.PersistentFlags().AddGoFlag(f)
+	pflag.CommandLine = pflag.NewFlagSet("empty", pflag.ExitOnError)
+	flag.CommandLine = flag.NewFlagSet("empty", flag.ExitOnError)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+
+	if err := func() error {
+		return root.Execute()
+	}(); err != nil {
+		if ex, ok := err.(ExitError); ok {
+			fmt.Fprintf(os.Stderr, "Ginkgo exit error %d: %v\n", ex.Code, err)
+			os.Exit(ex.Code)
+		}
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func newRunTestCommand() *cobra.Command {
+	testOpt := NewTestOptions(os.Stdout, os.Stderr)
+
+	cmd := &cobra.Command{
+		Use:          "run-test NAME",
+		Short:        "Run a single test by name",
+		Long:         "Execute a single test.",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := initializeTestFramework(os.Getenv("TEST_PROVIDER")); err != nil {
+				return err
+			}
+
+			return testOpt.Run(args)
+		},
+	}
+	return cmd
+}
+
+func newListTestsCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:          "list",
+		Short:        "List available tests",
+		Long:         "List the available tests in this binary.",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			tests := testsForSuite()
+			sort.Slice(tests, func(i, j int) bool { return tests[i].Name < tests[j].Name })
+			data, err := json.Marshal(tests)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(os.Stdout, "%s\n", data)
+			return nil
+		},
+	}
+
+	return cmd
+}

openshift-hack/cmd/k8s-tests/provider.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	kclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/openshift-hack/e2e"
+	conformancetestdata "k8s.io/kubernetes/test/conformance/testdata"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/storage/external"
+	e2etestingmanifests "k8s.io/kubernetes/test/e2e/testing-manifests"
+	testfixtures "k8s.io/kubernetes/test/fixtures"
+
+	// this appears to inexplicably auto-register global flags.
+	_ "k8s.io/kubernetes/test/e2e/storage/drivers"
+
+	// these are loading important global flags that we need to get and set
+	_ "k8s.io/kubernetes/test/e2e"
+	_ "k8s.io/kubernetes/test/e2e/lifecycle"
+)
+
+// copied directly from github.com/openshift/origin/cmd/openshift-tests/provider.go
+// and github.com/openshift/origin/test/extended/util/test.go
+func initializeTestFramework(provider string) error {
+	providerInfo := &ClusterConfiguration{}
+	if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
+		return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
+	}
+	if len(providerInfo.ProviderName) == 0 {
+		return fmt.Errorf("provider must be a JSON object with the 'type' key")
+	}
+	config := &ClusterConfiguration{}
+	if err := json.Unmarshal([]byte(provider), config); err != nil {
+		return fmt.Errorf("provider must decode into the ClusterConfig object: %v", err)
+	}
+
+	// update testContext with loaded config
+	testContext := &framework.TestContext
+	testContext.Provider = config.ProviderName
+	testContext.CloudConfig = framework.CloudConfig{
+		ProjectID:   config.ProjectID,
+		Region:      config.Region,
+		Zone:        config.Zone,
+		Zones:       config.Zones,
+		NumNodes:    config.NumNodes,
+		MultiMaster: config.MultiMaster,
+		MultiZone:   config.MultiZone,
+		ConfigFile:  config.ConfigFile,
+	}
+	testContext.AllowedNotReadyNodes = -1
+	testContext.MinStartupPods = -1
+	testContext.MaxNodesToGather = 0
+	testContext.KubeConfig = os.Getenv("KUBECONFIG")
+
+	// allow the CSI tests to access test data, but only briefly
+	// TODO: ideally CSI would not use any of these test methods
+	// var err error
+	// exutil.WithCleanup(func() { err = initCSITests(dryRun) })
+	// TODO: for now I'm only initializing CSI directly, but we probably need that
+	// WithCleanup here as well
+	if err := initCSITests(); err != nil {
+		return err
+	}
+
+	if ad := os.Getenv("ARTIFACT_DIR"); len(strings.TrimSpace(ad)) == 0 {
+		os.Setenv("ARTIFACT_DIR", filepath.Join(os.TempDir(), "artifacts"))
+	}
+
+	testContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false"
+	testContext.VerifyServiceAccount = true
+	testfiles.AddFileSource(e2etestingmanifests.GetE2ETestingManifestsFS())
+	testfiles.AddFileSource(testfixtures.GetTestFixturesFS())
+	testfiles.AddFileSource(conformancetestdata.GetConformanceTestdataFS())
+	testContext.KubectlPath = "kubectl"
+	// context.KubeConfig = KubeConfigPath()
+	testContext.KubeConfig = os.Getenv("KUBECONFIG")
+
+	// "debian" is used when not set. At least GlusterFS tests need "custom".
+	// (There is no option for "rhel" or "centos".)
+	testContext.NodeOSDistro = "custom"
+	testContext.MasterOSDistro = "custom"
+
+	// load and set the host variable for kubectl
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: testContext.KubeConfig}, &clientcmd.ConfigOverrides{})
+	cfg, err := clientConfig.ClientConfig()
+	if err != nil {
+		return err
+	}
+	testContext.Host = cfg.Host
+
+	// Ensure that Kube tests run privileged (like they do upstream)
+	testContext.CreateTestingNS = func(ctx context.Context, baseName string, c kclientset.Interface, labels map[string]string) (*corev1.Namespace, error) {
+		return e2e.CreateTestingNS(baseName, c, labels, true)
+	}
+
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	framework.AfterReadingAllFlags(testContext)
+	testContext.DumpLogsOnFailure = true
+
+	// these constants are taken from kube e2e and used by tests
+	testContext.IPFamily = "ipv4"
+	if config.HasIPv6 && !config.HasIPv4 {
+		testContext.IPFamily = "ipv6"
+	}
+
+	testContext.ReportDir = os.Getenv("TEST_JUNIT_DIR")
+
+	return nil
+}
+
+const (
+	manifestEnvVar = "TEST_CSI_DRIVER_FILES"
+)
+
+// copied directly from github.com/openshift/origin/cmd/openshift-tests/csi.go
+// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES.
+func initCSITests() error {
+	manifestList := os.Getenv(manifestEnvVar)
+	if manifestList != "" {
+		manifests := strings.Split(manifestList, ",")
+		for _, manifest := range manifests {
+			if err := external.AddDriverDefinition(manifest); err != nil {
+				return fmt.Errorf("failed to load manifest from %q: %s", manifest, err)
+			}
+			// Register the base dir of the manifest file as a file source.
+			// With this we can reference the CSI driver's storageClass
+			// in the manifest file (FromFile field).
+			testfiles.AddFileSource(testfiles.RootFileSource{
+				Root: filepath.Dir(manifest),
+			})
+		}
+	}
+
+	return nil
+}