From 429b5077e24bfecd61ec763e50bee67ff32ce3f2 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Thu, 26 Mar 2026 09:52:14 -0300 Subject: [PATCH] Allow JobTier=candidate in featuregate-test-analyzer with warning Candidate tier jobs are not covered by the Component Readiness main view and lack standard regression protection mechanisms. This change allows candidate as a valid JobTier value, includes candidate tier results in the pass/fail calculation alongside other tiers, and emits a non-blocking warning when candidate tier queries return data. Co-Authored-By: Claude Opus 4.6 --- .../codegen/cmd/featuregate-test-analyzer.go | 30 ++- .../cmd/featuregate-test-analyzer_test.go | 189 ++++++++++++++++++ tools/codegen/pkg/sippy/json_types.go | 2 + 3 files changed, 216 insertions(+), 5 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index da7bf20ca4b..040b86b3e6e 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -217,7 +217,7 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { md.Textf("* Tests must be be run on every TechPreview platform (ask for an exception if your feature doesn't support a variant)") md.Textf("* All tests must run at least 14 times on every platform") md.Textf("* All tests must pass at least 95%% of the time") - md.Textf("* JobTier must be one of: standard, informing, blocking\n") + md.Textf("* JobTier must be one of: standard, informing, blocking, candidate (candidate is allowed but produces a warning as it is not covered by Component Readiness)\n") md.Text("") if len(warnings) > 0 { @@ -375,6 +375,18 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian // Optional variants (like RHEL 10 in 4.22) have non-blocking warnings isOptional := jobVariant.Optional + // If candidate-tier queries returned results for this variant, emit a warning. + // Candidate tier jobs are not covered by the Component Readiness main view and + // do not have our standard regression protection mechanisms. The results are still + // included in the pass/fail calculation alongside other tiers. + if testedVariant.HasCandidateTierResults { + results = append(results, ValidationResult{ + Error: fmt.Errorf("warning: variant %v includes test data from candidate-tier jobs which are not covered by Component Readiness and lack standard regression protection", + jobVariant), + IsWarning: true, + }) + } + if len(testedVariant.TestResults) < requiredNumberOfTests { results = append(results, ValidationResult{ Error: fmt.Errorf("error: only %d tests found, need at least %d for %q on %v", @@ -639,7 +651,8 @@ func (a OrderedJobVariants) Less(i, j int) bool { type TestingResults struct { JobVariant JobVariant - TestResults []TestResults + TestResults []TestResults + HasCandidateTierResults bool // true if candidate-tier queries returned any test data } type TestResults struct { @@ -674,6 +687,7 @@ func validateJobTiers(jobVariant JobVariant) error { "standard": true, "informing": true, "blocking": true, + "candidate": true, } hasValidTier := false @@ -682,7 +696,7 @@ func validateJobTiers(jobVariant JobVariant) error { if tier != "" { hasValidTier = true if !validTiers[tier] { - return fmt.Errorf("invalid JobTier %q in variant %+v - must be one of: standard, informing, blocking", tier, jobVariant) + return fmt.Errorf("invalid JobTier %q in variant %+v - must be one of: standard, informing, blocking, candidate", tier, jobVariant) } } } @@ -838,6 +852,7 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi } testNameToResults := map[string]*TestResults{} + hasCandidateTierResults := false queries := sippy.QueriesFor(jobVariant.Cloud, jobVariant.Architecture, jobVariant.Topology, jobVariant.NetworkStack, jobVariant.OS, jobVariant.JobTiers, testPattern) release, err := getRelease() if err != nil { @@ -884,6 +899,10 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi return nil, err } + if currQuery.TierName == "candidate" && len(testInfos) > 0 { + hasCandidateTierResults = true + } + for _, currTest := range testInfos { testResults, ok := testNameToResults[currTest.Name] if !ok { @@ -912,8 +931,9 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi } jobVariantResults := &TestingResults{ - JobVariant: jobVariant, - TestResults: nil, + JobVariant: jobVariant, + TestResults: nil, + HasCandidateTierResults: hasCandidateTierResults, } testNames := sets.StringKeySet(testNameToResults) for _, testName := range testNames.List() { diff --git a/tools/codegen/cmd/featuregate-test-analyzer_test.go b/tools/codegen/cmd/featuregate-test-analyzer_test.go index 5de8d36c5aa..df66edcac67 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer_test.go +++ b/tools/codegen/cmd/featuregate-test-analyzer_test.go @@ -79,6 +79,195 @@ func Test_listTestResultFor(t *testing.T) { } } +func Test_validateJobTiers_candidate(t *testing.T) { + tests := []struct { + name string + variant JobVariant + wantErr bool + }{ + { + name: "candidate is valid", + variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "candidate"}, + wantErr: false, + }, + { + name: "candidate with standard is valid", + variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "standard,candidate"}, + wantErr: false, + }, + { + name: "invalid tier still rejected", + variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "bogus"}, + wantErr: true, + }, + { + name: "candidate with invalid tier rejected", + variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "candidate,bogus"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateJobTiers(tt.variant) + if (err != nil) != tt.wantErr { + t.Errorf("validateJobTiers() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_checkIfTestingIsSufficient_CandidateVariants(t *testing.T) { + sufficientTests := []TestResults{ + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test5", TotalRuns: 15, SuccessfulRuns: 15}, + } + insufficientTests := []TestResults{ + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + // Only 2 tests, need 5 + } + + tests := []struct { + name string + featureGate string + testingResults map[JobVariant]*TestingResults + wantBlockingErrors int + wantWarnings int + }{ + { + name: "candidate tier returned results with sufficient tests - warning about component readiness", + featureGate: "TestFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: sufficientTests, + HasCandidateTierResults: true, + }, + }, + wantBlockingErrors: 0, + wantWarnings: 1, // component readiness warning + }, + { + name: "candidate tier returned results with insufficient tests - blocking error plus warning", + featureGate: "TestFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: insufficientTests, + HasCandidateTierResults: true, + }, + }, + wantBlockingErrors: 1, // insufficient tests is still blocking + wantWarnings: 1, // component readiness warning + }, + { + name: "no candidate tier results - no warning", + featureGate: "TestFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: sufficientTests, + HasCandidateTierResults: false, + }, + }, + wantBlockingErrors: 0, + wantWarnings: 0, + }, + { + name: "candidate tier returned results with low pass rate - blocking error plus warning", + featureGate: "TestFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "test1", TotalRuns: 20, SuccessfulRuns: 18}, // 90% + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test5", TotalRuns: 15, SuccessfulRuns: 15}, + }, + HasCandidateTierResults: true, + }, + }, + wantBlockingErrors: 1, // low pass rate is still blocking + wantWarnings: 1, // component readiness warning + }, + { + name: "mix of variants - one with candidate results one without", + featureGate: "TestFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: sufficientTests, + HasCandidateTierResults: false, + }, + { + Cloud: "gcp", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: sufficientTests, + HasCandidateTierResults: true, + }, + }, + wantBlockingErrors: 0, + wantWarnings: 1, // only gcp variant has candidate warning + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + results := checkIfTestingIsSufficient(tt.featureGate, tt.testingResults) + + blockingErrors := 0 + warnings := 0 + for _, result := range results { + if result.IsWarning { + warnings++ + } else { + blockingErrors++ + } + } + + if blockingErrors != tt.wantBlockingErrors { + t.Errorf("got %d blocking errors, want %d", blockingErrors, tt.wantBlockingErrors) + for _, result := range results { + if !result.IsWarning { + t.Logf(" Blocking error: %v", result.Error) + } + } + } + if warnings != tt.wantWarnings { + t.Errorf("got %d warnings, want %d", warnings, tt.wantWarnings) + for _, result := range results { + if result.IsWarning { + t.Logf(" Warning: %v", result.Error) + } + } + } + }) + } +} + func Test_checkIfTestingIsSufficient_OptionalVariants(t *testing.T) { tests := []struct { name string diff --git a/tools/codegen/pkg/sippy/json_types.go b/tools/codegen/pkg/sippy/json_types.go index 204cb59e0b0..2785d496f4a 100644 --- a/tools/codegen/pkg/sippy/json_types.go +++ b/tools/codegen/pkg/sippy/json_types.go @@ -12,6 +12,7 @@ import ( type SippyQueryStruct struct { Items []SippyQueryItem `json:"items"` LinkOperator string `json:"linkOperator"` + TierName string `json:"-"` // not serialized, used to track which tier this query is for } type SippyQueryItem struct { @@ -138,6 +139,7 @@ func QueriesFor(cloud, architecture, topology, networkStack, os, jobTiers, testP queries = append(queries, &SippyQueryStruct{ Items: items, LinkOperator: "and", + TierName: jobTier, }) }