diff --git a/pkg/api/component_report.go b/pkg/api/component_report.go index eaae9b1a3..55f0febf1 100644 --- a/pkg/api/component_report.go +++ b/pkg/api/component_report.go @@ -115,6 +115,15 @@ func GetComponentTestVariantsFromBigQuery(client *bqcachedclient.Client, gcsBuck return getDataFromCacheOrGenerate[apitype.ComponentReportTestVariants](client.Cache, cache.RequestOptions{}, GetPrefixedCacheKey("TestVariants~", generator), generator.GenerateVariants, apitype.ComponentReportTestVariants{}) } +func GetJobVariantsFromBigQuery(client *bqcachedclient.Client, gcsBucket string) (apitype.JobVariants, []error) { + generator := componentReportGenerator{ + client: client, + gcsBucket: gcsBucket, + } + + return getDataFromCacheOrGenerate[apitype.JobVariants](client.Cache, cache.RequestOptions{}, GetPrefixedCacheKey("TestAllVariants~", generator), generator.GenerateJobVariants, apitype.JobVariants{}) +} + func GetComponentReportFromBigQuery(client *bqcachedclient.Client, prowURL, gcsBucket string, baseRelease, sampleRelease apitype.ComponentReportRequestReleaseOptions, testIDOption apitype.ComponentReportRequestTestIdentificationOptions, @@ -215,6 +224,59 @@ func (c *componentReportGenerator) GenerateVariants() (apitype.ComponentReportTe }, errs } +func (c *componentReportGenerator) GenerateJobVariants() (apitype.JobVariants, []error) { + errs := []error{} + variants := apitype.JobVariants{Variants: map[string][]string{}} + queryString := fmt.Sprintf(`SELECT variant_name, ARRAY_AGG(DISTINCT variant_value ORDER BY variant_value) AS variant_values + FROM + %s.job_variants + WHERE + variant_value!="" + GROUP BY + variant_name`, c.client.Dataset) + query := c.client.BQ.Query(queryString) + it, err := query.Read(context.TODO()) + if err != nil { + log.WithError(err).Errorf("error querying variants from bigquery for %s", queryString) + return variants, []error{err} + } + + floatVariants := sets.NewString("FromRelease", "FromReleaseMajor", "FromReleaseMinor", "Release", "ReleaseMajor", "ReleaseMinor") + for { + row := apitype.JobVariant{} + err := it.Next(&row) + if err == iterator.Done { + break + } + if err != nil { + wrappedErr := errors.Wrapf(err, "error fetching variant row") + log.WithError(err).Error("error fetching variants from bigquery") + errs = append(errs, wrappedErr) + return variants, errs + } + + // Sort all releases in proper orders + if floatVariants.Has(row.VariantName) { + sort.Slice(row.VariantValues, func(i, j int) bool { + iStrings := strings.Split(row.VariantValues[i], ".") + jStrings := strings.Split(row.VariantValues[j], ".") + for idx, iString := range iStrings { + if iValue, err := strconv.ParseInt(iString, 10, 32); err == nil { + if jValue, err := strconv.ParseInt(jStrings[idx], 10, 32); err == nil { + if iValue != jValue { + return iValue < jValue + } + } + } + } + return false + }) + } + variants.Variants[row.VariantName] = row.VariantValues + } + return variants, nil +} + func (c *componentReportGenerator) GenerateReport() (apitype.ComponentReport, []error) { before := time.Now() componentReportTestStatus, errs := c.GenerateComponentReportTestStatus() diff --git a/pkg/apis/api/types.go b/pkg/apis/api/types.go index a890cad7a..50ec274f1 100644 --- a/pkg/apis/api/types.go +++ b/pkg/apis/api/types.go @@ -1026,6 +1026,17 @@ type ComponentReportTestVariants struct { Variant []string `json:"variant,omitempty"` } +// JobVariant defines a variant and the possible values +type JobVariant struct { + VariantName string `bigquery:"variant_name"` + VariantValues []string `bigquery:"variant_values"` +} + +// JobVariants contains all variants supported in the system. +type JobVariants struct { + Variants map[string][]string `json:"variants,omitempty"` +} + var FailureRiskLevelNone = RiskLevel{Name: "None", Level: 0} var FailureRiskLevelLow = RiskLevel{Name: "Low", Level: 1} var FailureRiskLevelUnknown = RiskLevel{Name: "Unknown", Level: 25} diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 6221cf65c..5ceb547e1 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -627,6 +627,29 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re api.RespondWithJSON(http.StatusOK, w, outputs) } +func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Request) { + if s.bigQueryClient == nil { + api.RespondWithJSON(http.StatusBadRequest, w, map[string]interface{}{ + "code": http.StatusBadRequest, + "message": "job variants API is only available when google-service-account-credential-file is configured", + }) + return + } + outputs, errs := api.GetJobVariantsFromBigQuery(s.bigQueryClient, s.gcsBucket) + if len(errs) > 0 { + log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs)) + for _, err := range errs { + log.Error(err.Error()) + } + api.RespondWithJSON(http.StatusInternalServerError, w, map[string]interface{}{ + "code": http.StatusInternalServerError, + "message": fmt.Sprintf("error querying job variants from big query: %v", errs), + }) + return + } + api.RespondWithJSON(http.StatusOK, w, outputs) +} + func (s *Server) jsonComponentReportFromBigQuery(w http.ResponseWriter, req *http.Request) { baseRelease, sampleRelease, testIDOption, variantOption, excludeOption, advancedOption, cacheOption, err := s.parseComponentReportRequest(req) if err != nil { @@ -1451,6 +1474,12 @@ func (s *Server) Serve() { Capabilities: []string{LocalDBCapability}, HandlerFunc: s.jsonJobBugsFromDB, }, + { + EndpointPath: "/api/job_variants", + Description: "Reports all job variants defined in BigQuery", + Capabilities: []string{ComponentReadinessCapability}, + HandlerFunc: s.jsonJobVariantsFromBigQuery, + }, { EndpointPath: "/api/pull_requests", Description: "Reports on pull requests",