test: Generate test annotations rather than doing them dynamically

It is expensive to perform 100 checks against 1k names on every
startup. Instead, move these to generated files that can be reviewed
out of bounds on change. Add them to generated-bindata for verification
and regen.

cmd/openshift-tests/e2e.go

@@ -9,6 +9,7 @@ import (
 	"github.com/openshift/origin/pkg/test/ginkgo"
 
 	_ "github.com/openshift/origin/test/extended"
+	_ "github.com/openshift/origin/test/extended/util/annotate/generated"
 )
 
 // staticSuites are all known test suites this binary should run

cmd/openshift-tests/openshift-tests.go

@@ -288,7 +288,6 @@ func initProvider(provider string, dryRun bool) error {
 		return err
 	}
 
-	exutil.AnnotateTestSuite()
 	err := exutil.InitTest(dryRun)
 	gomega.RegisterFailHandler(ginkgo.Fail)
 

hack/update-generated-bindata.sh

@@ -2,6 +2,8 @@
 STARTTIME=$(date +%s)
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 
+go generate ./test/extended
+
 os::build::setup_env
 
 OUTPUT_PARENT=${OUTPUT_ROOT:-$OS_ROOT}

hack/verify-generated-bindata.sh

@@ -13,6 +13,7 @@ os::test::junit::declare_suite_start "verify/bindata"
 os::cmd::expect_success "${OS_ROOT}/hack/update-generated-bindata.sh"
 os::cmd::expect_success "git diff --quiet vendor/k8s.io/kubernetes/staging/src/k8s.io/kubectl/pkg/generated/bindata.go"
 os::cmd::expect_success "git diff --quiet vendor/k8s.io/kubernetes/test/e2e/generated/bindata.go"
+os::cmd::expect_success "git diff --quiet ${OS_ROOT}/test/extended/util/annotate/generated/"
 os::cmd::expect_success "git diff --quiet ${OS_ROOT}/test/extended/testdata/bindata.go"
 
 os::test::junit::declare_suite_end

test/extended/include.go

@@ -1,5 +1,7 @@
 package extended
 
+//go:generate go run ./util/annotate -- ./util/annotate/generated/zz_generated.annotations.go
+
 import (
 	_ "k8s.io/kubernetes/test/e2e"
 

test/extended/util/annotate/OWNERS

@@ -0,0 +1,5 @@
+reviewers:
+  - smarterclayton
+approvers:
+  - smarterclayton
+  - decarr

test/extended/util/annotate/annotate.go

@@ -0,0 +1,204 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/types"
+)
+
+func main() {
+	if len(os.Args) != 2 && len(os.Args) != 3 {
+		fmt.Fprintf(os.Stderr, "error: requires exactly one argument\n")
+		os.Exit(1)
+	}
+	filename := os.Args[len(os.Args)-1]
+
+	generator := newGenerator()
+	ginkgo.WalkTests(generator.generateRename)
+
+	renamer := newRenamerFromGenerated(generator.output)
+	ginkgo.WalkTests(renamer.updateNodeText)
+	if len(renamer.missing) > 0 {
+		var names []string
+		for name := range renamer.missing {
+			names = append(names, name)
+		}
+		sort.Strings(names)
+		fmt.Fprintf(os.Stderr, "failed:\n%s\n", strings.Join(names, "\n"))
+		os.Exit(1)
+	}
+
+	var pairs []string
+	for from, to := range generator.output {
+		pairs = append(pairs, fmt.Sprintf("%q:\n%q,", from, to))
+	}
+	sort.Strings(pairs)
+	contents := fmt.Sprintf(`
+package generated
+
+import (
+	"fmt"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/types"
+)
+
+var annotations = map[string]string{
+%s
+}
+
+func init() {
+	ginkgo.WalkTests(func(name string, node types.TestNode) {
+		if updated, ok := annotations[name]; ok {
+			node.SetText(updated)
+		} else {
+			panic(fmt.Sprintf("unable to find test %%s", name))
+		}	
+	})
+}
+`, strings.Join(pairs, "\n"))
+	if err := ioutil.WriteFile(filename, []byte(contents), 0644); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v", err)
+		os.Exit(1)
+	}
+	if _, err := exec.Command("gofmt", "-s", "-w", filename).Output(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v", err)
+		os.Exit(1)
+	}
+}
+
+func newGenerator() *ginkgoTestRenamer {
+	var allLabels []string
+	matches := make(map[string]*regexp.Regexp)
+	stringMatches := make(map[string][]string)
+	excludes := make(map[string]*regexp.Regexp)
+
+	for label, items := range testMaps {
+		sort.Strings(items)
+		allLabels = append(allLabels, label)
+		var remain []string
+		for _, item := range items {
+			re := regexp.MustCompile(item)
+			if p, ok := re.LiteralPrefix(); ok {
+				stringMatches[label] = append(stringMatches[label], p)
+			} else {
+				remain = append(remain, item)
+			}
+		}
+		if len(remain) > 0 {
+			matches[label] = regexp.MustCompile(strings.Join(remain, `|`))
+		}
+	}
+	for label, items := range labelExcludes {
+		sort.Strings(items)
+		excludes[label] = regexp.MustCompile(strings.Join(items, `|`))
+	}
+	sort.Strings(allLabels)
+
+	excludedTestsFilter := regexp.MustCompile(strings.Join(excludedTests, `|`))
+
+	return &ginkgoTestRenamer{
+		allLabels:           allLabels,
+		stringMatches:       stringMatches,
+		matches:             matches,
+		excludes:            excludes,
+		excludedTestsFilter: excludedTestsFilter,
+
+		output: make(map[string]string),
+	}
+}
+
+func newRenamerFromGenerated(names map[string]string) *ginkgoTestRenamer {
+	return &ginkgoTestRenamer{
+		output:  names,
+		missing: make(map[string]struct{}),
+	}
+}
+
+type ginkgoTestRenamer struct {
+	allLabels           []string
+	stringMatches       map[string][]string
+	matches             map[string]*regexp.Regexp
+	excludes            map[string]*regexp.Regexp
+	excludedTestsFilter *regexp.Regexp
+
+	output  map[string]string
+	missing map[string]struct{}
+}
+
+func (r *ginkgoTestRenamer) updateNodeText(name string, node types.TestNode) {
+	if updated, ok := r.output[name]; ok {
+		node.SetText(updated)
+	} else {
+		r.missing[name] = struct{}{}
+	}
+}
+
+func (r *ginkgoTestRenamer) generateRename(name string, node types.TestNode) {
+	originalName := name
+
+	labels := ""
+	for {
+		count := 0
+		for _, label := range r.allLabels {
+			if strings.Contains(name, label) {
+				continue
+			}
+
+			var hasLabel bool
+			for _, segment := range r.stringMatches[label] {
+				hasLabel = strings.Contains(name, segment)
+				if hasLabel {
+					break
+				}
+			}
+			if !hasLabel {
+				if re := r.matches[label]; re != nil {
+					hasLabel = r.matches[label].MatchString(name)
+				}
+			}
+
+			if hasLabel {
+				// TODO: remove when we no longer need it
+				if re, ok := r.excludes[label]; ok && re.MatchString(name) {
+					continue
+				}
+				count++
+				labels += " " + label
+				name += " " + label
+			}
+		}
+		if count == 0 {
+			break
+		}
+	}
+
+	if !r.excludedTestsFilter.MatchString(name) {
+		isSerial := strings.Contains(name, "[Serial]")
+		isConformance := strings.Contains(name, "[Conformance]")
+		switch {
+		case isSerial && isConformance:
+			name += " [Suite:openshift/conformance/serial/minimal]"
+		case isSerial:
+			name += " [Suite:openshift/conformance/serial]"
+		case isConformance:
+			name += " [Suite:openshift/conformance/parallel/minimal]"
+		default:
+			name += " [Suite:openshift/conformance/parallel]"
+		}
+	}
+	if strings.Contains(node.CodeLocation().FileName, "/origin/test/") && !strings.Contains(name, "[Suite:openshift") {
+		name += " [Suite:openshift]"
+	}
+	if strings.Contains(node.CodeLocation().FileName, "/kubernetes/test/e2e/") {
+		name += " [Suite:k8s]"
+	}
+
+	r.output[originalName] = name
+}