From 1efbecd07bddb8448015f03e54d691f7cadeb805 Mon Sep 17 00:00:00 2001 From: Eris Hoxha Date: Wed, 19 Feb 2025 15:10:58 +0100 Subject: [PATCH 1/5] Revert "disable the feature analysis endpoint and remove the link in the UI" This reverts commit fb59d83863102bfa95af92998c56365a2ca7abae. --- cmd/release-controller-api/http.go | 927 +++++++++++++++-------------- 1 file changed, 472 insertions(+), 455 deletions(-) diff --git a/cmd/release-controller-api/http.go b/cmd/release-controller-api/http.go index 954efaad1..1edc3f841 100644 --- a/cmd/release-controller-api/http.go +++ b/cmd/release-controller-api/http.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/openshift/release-controller/pkg/apis/release/v1alpha1" "io/fs" "math" "net/http" @@ -18,8 +19,6 @@ import ( "text/template" "time" - "github.com/openshift/release-controller/pkg/apis/release/v1alpha1" - releasecontroller "github.com/openshift/release-controller/pkg/release-controller" "github.com/openshift/release-controller/pkg/rhcos" @@ -33,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" + "sigs.k8s.io/prow/pkg/jira" ) //go:embed static @@ -42,16 +42,16 @@ var resources, _ = fs.Sub(static, "static") var htmlPageStart = loadStaticHTML("htmlPageStart.html") var htmlPageEnd = loadStaticHTML("htmlPageEnd.html") -//const ( -// sectionTypeNoEpicWithFeature = "noEpicWithFeature" -// sectionTypeNoFeatureWithEpic = "noFeatureWithEpic" -// sectionTypeNoEpicNoFeature = "noEpicNoFeature" -// sectionTypeUnknowns = "unknowns" -// sectionTypeUnsortedUnknowns = "unsorted_unknowns" -//) +const ( + sectionTypeNoEpicWithFeature = "noEpicWithFeature" + sectionTypeNoFeatureWithEpic = "noFeatureWithEpic" + sectionTypeNoEpicNoFeature = "noEpicNoFeature" + sectionTypeUnknowns = "unknowns" + sectionTypeUnsortedUnknowns = "unsorted_unknowns" +) -// var unlinkedIssuesSections = sets.NewString(sectionTypeNoEpicWithFeature, sectionTypeNoFeatureWithEpic, sectionTypeNoEpicNoFeature, sectionTypeUnknowns, sectionTypeUnsortedUnknowns) -//var statusComplete = sets.NewString(strings.ToLower(jira.StatusOnQA), strings.ToLower(jira.StatusVerified), strings.ToLower(jira.StatusModified), strings.ToLower(jira.StatusClosed)) +var unlinkedIssuesSections = sets.NewString(sectionTypeNoEpicWithFeature, sectionTypeNoFeatureWithEpic, sectionTypeNoEpicNoFeature, sectionTypeUnknowns, sectionTypeUnsortedUnknowns) +var statusComplete = sets.NewString(strings.ToLower(jira.StatusOnQA), strings.ToLower(jira.StatusVerified), strings.ToLower(jira.StatusModified), strings.ToLower(jira.StatusClosed)) // Find the stream from releaseTag. // Eg if we have release.openshift.io/releaseTag, we find the corresponding stream metadata @@ -188,9 +188,8 @@ func (c *Controller) userInterfaceHandler() http.Handler { mux.HandleFunc("/api/v1/releasestreams/all", c.apiAllStreams) mux.HandleFunc("/api/v1/releasestreams/approvals", c.apiReleaseApprovals) - // Disable the Release Analysis Feature - //mux.HandleFunc("/api/v1/features/{tag}", c.apiFeatureInfo) - //mux.HandleFunc("/features/{tag}", c.httpFeatureInfo) + mux.HandleFunc("/api/v1/features/{tag}", c.apiFeatureInfo) + mux.HandleFunc("/features/{tag}", c.httpFeatureInfo) // static files mux.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.FS(resources)))) @@ -198,264 +197,264 @@ func (c *Controller) userInterfaceHandler() http.Handler { return mux } -//func (c *Controller) releaseFeatureInfo(tagInfo *releaseTagInfo) ([]*FeatureTree, error) { -// // Get change log -// changeLogJSON := renderResult{} -// c.changeLogWorker(&changeLogJSON, tagInfo, "json") -// if changeLogJSON.err != nil { -// return nil, changeLogJSON.err -// } -// -// var changeLog releasecontroller.ChangeLog -// if err := json.Unmarshal([]byte(changeLogJSON.out), &changeLog); err != nil { -// return nil, err -// } -// -// // Get issue details -// info, err := c.releaseInfo.IssuesInfo(changeLogJSON.out) -// if err != nil { -// return nil, err -// } -// -// var mapIssueDetails map[string]releasecontroller.IssueDetails -// if err := json.Unmarshal([]byte(info), &mapIssueDetails); err != nil { -// return nil, err -// } -// -// // Create feature trees -// var featureTrees []*FeatureTree -// for key, details := range mapIssueDetails { -// if details.IssueType != releasecontroller.JiraTypeFeature { -// continue -// } -// featureTree := addChild(key, details, &changeLog.To.Created) -// featureTrees = append(featureTrees, featureTree) -// } -// -// linkedIssues := sets.Set[string]{} -// visited := make(map[string]bool) -// if !GetFeatureInfo(featureTrees, mapIssueDetails, &changeLog.To.Created, &linkedIssues, 10000, visited) { -// return nil, errors.New("failed getting the features information, cycle limit reached! ") -// } -// -// var noFeatureWithEpic []*FeatureTree -// var unknowns []*FeatureTree -// -// for issue, details := range mapIssueDetails { -// if linkedIssues.Has(issue) || details.IssueType == releasecontroller.JiraTypeEpic || details.IssueType == releasecontroller.JiraTypeFeature || details.IssueType == releasecontroller.JiraTypeMarketProblem { -// continue -// } -// feature := addChild(issue, details, &changeLog.To.Created) -// if details.Feature == "" && details.Epic == "" && details.Parent == "" { -// feature.NotLinkedType = sectionTypeNoEpicNoFeature -// featureTrees = append(featureTrees, feature) -// } else if details.Epic != "" { -// noFeatureWithEpic = append(noFeatureWithEpic, feature) -// } else { -// feature.NotLinkedType = sectionTypeUnknowns -// unknowns = append(unknowns, feature) -// } -// } -// -// epicWithoutFeatureMap := make(map[string][]*FeatureTree, 0) -// for _, child := range noFeatureWithEpic { -// epicWithoutFeatureMap[child.Epic] = append(epicWithoutFeatureMap[child.Epic], child) -// } -// -// for epic, children := range epicWithoutFeatureMap { -// f := &FeatureTree{ -// IssueKey: epic, -// Summary: mapIssueDetails[epic].Summary, -// Description: mapIssueDetails[epic].Description, -// ReleaseNotes: mapIssueDetails[epic].ReleaseNotes, -// Type: mapIssueDetails[epic].IssueType, -// Epic: mapIssueDetails[epic].Epic, -// Feature: mapIssueDetails[epic].Feature, -// Parent: mapIssueDetails[epic].Parent, -// NotLinkedType: sectionTypeNoFeatureWithEpic, -// PRs: mapIssueDetails[epic].PRs, -// IncludedInBuild: statusOnBuild(&changeLog.To.Created, mapIssueDetails[epic].ResolutionDate, mapIssueDetails[epic].Transitions), -// Children: children, -// Demos: mapIssueDetails[epic].Demos, -// } -// featureTrees = append(featureTrees, f) -// } -// -// // TODO - find a better way to do this, this it is to expensive -// redistributedUnknowns := sets.Set[string]{} -// for _, unknown := range unknowns { -// if unknown.Parent != "" { -// redistributeUnknowns(featureTrees, unknown.Parent, unknown, &redistributedUnknowns, 10000) -// } -// if unknown.Epic != "" { -// redistributeUnknowns(featureTrees, unknown.Epic, unknown, &redistributedUnknowns, 10000) -// } -// if unknown.Feature != "" { -// redistributeUnknowns(featureTrees, unknown.Feature, unknown, &redistributedUnknowns, 10000) -// } -// } -// for _, ticket := range unknowns { -// if !redistributedUnknowns.Has(ticket.IssueKey) { -// ticket.NotLinkedType = sectionTypeUnsortedUnknowns -// featureTrees = append(featureTrees, ticket) -// } -// } -// -// // Remove every tree from sectionTypeNoEpicNoFeature that has no PRs, since it implies that it is not part of the -// // change log. Specifically, cards within the parent/epics/features group are gathered for the featureTree but are -// // not linked properly (e.g., an Epic that links directly to a "Market Problem" instead of a Feature, and Feature -// // is the root). -// toRemove := sets.Set[string]{} -// for _, ticket := range featureTrees { -// if ticket.NotLinkedType == sectionTypeNoEpicNoFeature { -// if isPRsEmpty(ticket) { -// toRemove.Insert(ticket.IssueKey) -// } -// } -// } -// return removeUnnecessaryTrees(featureTrees, toRemove), nil -//} - -//func removeUnnecessaryTrees(slice []*FeatureTree, toRemove sets.Set[string]) []*FeatureTree { -// newFeatureTree := make([]*FeatureTree, 0) -// for _, feature := range slice { -// if !toRemove.Has(feature.IssueKey) { -// newFeatureTree = append(newFeatureTree, feature) -// } -// } -// return newFeatureTree -//} - -//func isPRsEmpty(ft *FeatureTree) bool { -// if len(ft.PRs) > 0 { -// return false -// } -// for _, child := range ft.Children { -// if !isPRsEmpty(child) { -// return false -// } -// } -// return true -//} - -//func redistributeUnknowns(slice []*FeatureTree, key string, feature *FeatureTree, s *sets.Set[string], limit int) bool { -// if limit <= 0 { -// klog.Errorf("breaking the recursion: limit reached for the redistributeUnknowns func! This might indicate a cyclic tree!") -// return false -// } -// for _, node := range slice { -// if node.IssueKey == key { -// node.Children = append(node.Children, feature) -// s.Insert(feature.IssueKey) -// return true -// } -// redistributeUnknowns(node.Children, key, feature, s, limit-1) -// } -// return false -//} - -//func GetFeatureInfo(ft []*FeatureTree, issues map[string]releasecontroller.IssueDetails, buildTimeStamp *time.Time, linkedIssues *sets.Set[string], limit int, visited map[string]bool) bool { -// -// // add a fail-safe to protect against stack-overflows caused by a cyclic link. If the limit has been reached, the -// //function will return immediately without making any further recursive calls. -// if limit <= 0 { -// klog.Errorf("breaking the recursion: limit reached for the GetFeatureInfo func! This might indicate a cyclic tree!") -// return false -// } -// -// for _, child := range ft { -// -// // Check if the child has already been visited. This will protect against cyclic links and redundant work -// if visited[child.IssueKey] { -// klog.Infof("Skipping child %v as it has already been visited", child.IssueKey) -// continue -// } -// visited[child.IssueKey] = true // mark the child as visited -// -// var children []*FeatureTree -// for issueKey, issueDetails := range issues { -// var featureTree *FeatureTree -// if child.Type == releasecontroller.JiraTypeFeature && issueDetails.Feature == child.IssueKey { -// featureTree = addChild(issueKey, issueDetails, buildTimeStamp) -// } else if child.Type == releasecontroller.JiraTypeEpic && issueDetails.Epic == child.IssueKey { -// featureTree = addChild(issueKey, issueDetails, buildTimeStamp) -// linkedIssues.Insert(issueKey) -// } else { -// if issueDetails.Parent == child.IssueKey { -// featureTree = addChild(issueKey, issueDetails, buildTimeStamp) -// linkedIssues.Insert(issueKey) -// } -// } -// if featureTree != nil { -// children = append(children, featureTree) -// } -// } -// child.Children = children -// -// GetFeatureInfo(child.Children, issues, buildTimeStamp, linkedIssues, limit-1, visited) -// -// } -// return true -//} - -//func addChild(issueKey string, issueDetails releasecontroller.IssueDetails, buildTimeStamp *time.Time) *FeatureTree { -// return &FeatureTree{ -// IssueKey: issueKey, -// Summary: issueDetails.Summary, -// Description: issueDetails.Description, -// ReleaseNotes: issueDetails.ReleaseNotes, -// Type: issueDetails.IssueType, -// Epic: issueDetails.Epic, -// Feature: issueDetails.Feature, -// Parent: issueDetails.Parent, -// IncludedInBuild: statusOnBuild(buildTimeStamp, issueDetails.ResolutionDate, issueDetails.Transitions), -// PRs: issueDetails.PRs, -// Children: nil, -// Demos: issueDetails.Demos, -// } -//} - -//func (c *Controller) apiFeatureInfo(w http.ResponseWriter, req *http.Request) { -// tagInfo, err := c.getReleaseTagInfo(req) -// if err != nil { -// http.Error(w, err.Error(), http.StatusNotFound) -// return -// } -// featureTrees, err := c.releaseFeatureInfo(tagInfo) -// if err != nil { -// http.Error(w, err.Error(), http.StatusInternalServerError) -// return -// } -// data, err := json.MarshalIndent(&featureTrees, "", " ") -// if err != nil { -// http.Error(w, err.Error(), http.StatusInternalServerError) -// return -// } -// w.Header().Set("Content-Type", "application/json") -// w.WriteHeader(http.StatusOK) -// if _, err := w.Write(data); err != nil { -// http.Error(w, err.Error(), http.StatusInternalServerError) -// } -//} - -//func statusOnBuild(buildTimeStamp *time.Time, issueTimestamp time.Time, transitions []releasecontroller.Transition) bool { -// if !issueTimestamp.IsZero() && issueTimestamp.Before(*buildTimeStamp) { -// return true -// } -// status := getPastStatus(transitions, buildTimeStamp) -// return statusComplete.Has(strings.ToLower(status)) -//} - -//func getPastStatus(transitions []releasecontroller.Transition, buildTime *time.Time) string { -// status := "New" -// for _, t := range transitions { -// if t.Time.After(*buildTime) { -// break -// } -// status = t.ToStatus -// } -// return status -//} +func (c *Controller) releaseFeatureInfo(tagInfo *releaseTagInfo) ([]*FeatureTree, error) { + // Get change log + changeLogJSON := renderResult{} + c.changeLogWorker(&changeLogJSON, tagInfo, "json") + if changeLogJSON.err != nil { + return nil, changeLogJSON.err + } + + var changeLog releasecontroller.ChangeLog + if err := json.Unmarshal([]byte(changeLogJSON.out), &changeLog); err != nil { + return nil, err + } + + // Get issue details + info, err := c.releaseInfo.IssuesInfo(changeLogJSON.out) + if err != nil { + return nil, err + } + + var mapIssueDetails map[string]releasecontroller.IssueDetails + if err := json.Unmarshal([]byte(info), &mapIssueDetails); err != nil { + return nil, err + } + + // Create feature trees + var featureTrees []*FeatureTree + for key, details := range mapIssueDetails { + if details.IssueType != releasecontroller.JiraTypeFeature { + continue + } + featureTree := addChild(key, details, &changeLog.To.Created) + featureTrees = append(featureTrees, featureTree) + } + + linkedIssues := sets.Set[string]{} + visited := make(map[string]bool) + if !GetFeatureInfo(featureTrees, mapIssueDetails, &changeLog.To.Created, &linkedIssues, 10000, visited) { + return nil, errors.New("failed getting the features information, cycle limit reached! ") + } + + var noFeatureWithEpic []*FeatureTree + var unknowns []*FeatureTree + + for issue, details := range mapIssueDetails { + if linkedIssues.Has(issue) || details.IssueType == releasecontroller.JiraTypeEpic || details.IssueType == releasecontroller.JiraTypeFeature || details.IssueType == releasecontroller.JiraTypeMarketProblem { + continue + } + feature := addChild(issue, details, &changeLog.To.Created) + if details.Feature == "" && details.Epic == "" && details.Parent == "" { + feature.NotLinkedType = sectionTypeNoEpicNoFeature + featureTrees = append(featureTrees, feature) + } else if details.Epic != "" { + noFeatureWithEpic = append(noFeatureWithEpic, feature) + } else { + feature.NotLinkedType = sectionTypeUnknowns + unknowns = append(unknowns, feature) + } + } + + epicWithoutFeatureMap := make(map[string][]*FeatureTree, 0) + for _, child := range noFeatureWithEpic { + epicWithoutFeatureMap[child.Epic] = append(epicWithoutFeatureMap[child.Epic], child) + } + + for epic, children := range epicWithoutFeatureMap { + f := &FeatureTree{ + IssueKey: epic, + Summary: mapIssueDetails[epic].Summary, + Description: mapIssueDetails[epic].Description, + ReleaseNotes: mapIssueDetails[epic].ReleaseNotes, + Type: mapIssueDetails[epic].IssueType, + Epic: mapIssueDetails[epic].Epic, + Feature: mapIssueDetails[epic].Feature, + Parent: mapIssueDetails[epic].Parent, + NotLinkedType: sectionTypeNoFeatureWithEpic, + PRs: mapIssueDetails[epic].PRs, + IncludedInBuild: statusOnBuild(&changeLog.To.Created, mapIssueDetails[epic].ResolutionDate, mapIssueDetails[epic].Transitions), + Children: children, + Demos: mapIssueDetails[epic].Demos, + } + featureTrees = append(featureTrees, f) + } + + // TODO - find a better way to do this, this it is to expensive + redistributedUnknowns := sets.Set[string]{} + for _, unknown := range unknowns { + if unknown.Parent != "" { + redistributeUnknowns(featureTrees, unknown.Parent, unknown, &redistributedUnknowns, 10000) + } + if unknown.Epic != "" { + redistributeUnknowns(featureTrees, unknown.Epic, unknown, &redistributedUnknowns, 10000) + } + if unknown.Feature != "" { + redistributeUnknowns(featureTrees, unknown.Feature, unknown, &redistributedUnknowns, 10000) + } + } + for _, ticket := range unknowns { + if !redistributedUnknowns.Has(ticket.IssueKey) { + ticket.NotLinkedType = sectionTypeUnsortedUnknowns + featureTrees = append(featureTrees, ticket) + } + } + + // Remove every tree from sectionTypeNoEpicNoFeature that has no PRs, since it implies that it is not part of the + // change log. Specifically, cards within the parent/epics/features group are gathered for the featureTree but are + // not linked properly (e.g., an Epic that links directly to a "Market Problem" instead of a Feature, and Feature + // is the root). + toRemove := sets.Set[string]{} + for _, ticket := range featureTrees { + if ticket.NotLinkedType == sectionTypeNoEpicNoFeature { + if isPRsEmpty(ticket) { + toRemove.Insert(ticket.IssueKey) + } + } + } + return removeUnnecessaryTrees(featureTrees, toRemove), nil +} + +func removeUnnecessaryTrees(slice []*FeatureTree, toRemove sets.Set[string]) []*FeatureTree { + newFeatureTree := make([]*FeatureTree, 0) + for _, feature := range slice { + if !toRemove.Has(feature.IssueKey) { + newFeatureTree = append(newFeatureTree, feature) + } + } + return newFeatureTree +} + +func isPRsEmpty(ft *FeatureTree) bool { + if len(ft.PRs) > 0 { + return false + } + for _, child := range ft.Children { + if !isPRsEmpty(child) { + return false + } + } + return true +} + +func redistributeUnknowns(slice []*FeatureTree, key string, feature *FeatureTree, s *sets.Set[string], limit int) bool { + if limit <= 0 { + klog.Errorf("breaking the recursion: limit reached for the redistributeUnknowns func! This might indicate a cyclic tree!") + return false + } + for _, node := range slice { + if node.IssueKey == key { + node.Children = append(node.Children, feature) + s.Insert(feature.IssueKey) + return true + } + redistributeUnknowns(node.Children, key, feature, s, limit-1) + } + return false +} + +func GetFeatureInfo(ft []*FeatureTree, issues map[string]releasecontroller.IssueDetails, buildTimeStamp *time.Time, linkedIssues *sets.Set[string], limit int, visited map[string]bool) bool { + + // add a fail-safe to protect against stack-overflows caused by a cyclic link. If the limit has been reached, the + //function will return immediately without making any further recursive calls. + if limit <= 0 { + klog.Errorf("breaking the recursion: limit reached for the GetFeatureInfo func! This might indicate a cyclic tree!") + return false + } + + for _, child := range ft { + + // Check if the child has already been visited. This will protect against cyclic links and redundant work + if visited[child.IssueKey] { + klog.Infof("Skipping child %v as it has already been visited", child.IssueKey) + continue + } + visited[child.IssueKey] = true // mark the child as visited + + var children []*FeatureTree + for issueKey, issueDetails := range issues { + var featureTree *FeatureTree + if child.Type == releasecontroller.JiraTypeFeature && issueDetails.Feature == child.IssueKey { + featureTree = addChild(issueKey, issueDetails, buildTimeStamp) + } else if child.Type == releasecontroller.JiraTypeEpic && issueDetails.Epic == child.IssueKey { + featureTree = addChild(issueKey, issueDetails, buildTimeStamp) + linkedIssues.Insert(issueKey) + } else { + if issueDetails.Parent == child.IssueKey { + featureTree = addChild(issueKey, issueDetails, buildTimeStamp) + linkedIssues.Insert(issueKey) + } + } + if featureTree != nil { + children = append(children, featureTree) + } + } + child.Children = children + + GetFeatureInfo(child.Children, issues, buildTimeStamp, linkedIssues, limit-1, visited) + + } + return true +} + +func addChild(issueKey string, issueDetails releasecontroller.IssueDetails, buildTimeStamp *time.Time) *FeatureTree { + return &FeatureTree{ + IssueKey: issueKey, + Summary: issueDetails.Summary, + Description: issueDetails.Description, + ReleaseNotes: issueDetails.ReleaseNotes, + Type: issueDetails.IssueType, + Epic: issueDetails.Epic, + Feature: issueDetails.Feature, + Parent: issueDetails.Parent, + IncludedInBuild: statusOnBuild(buildTimeStamp, issueDetails.ResolutionDate, issueDetails.Transitions), + PRs: issueDetails.PRs, + Children: nil, + Demos: issueDetails.Demos, + } +} + +func (c *Controller) apiFeatureInfo(w http.ResponseWriter, req *http.Request) { + tagInfo, err := c.getReleaseTagInfo(req) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + featureTrees, err := c.releaseFeatureInfo(tagInfo) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + data, err := json.MarshalIndent(&featureTrees, "", " ") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func statusOnBuild(buildTimeStamp *time.Time, issueTimestamp time.Time, transitions []releasecontroller.Transition) bool { + if !issueTimestamp.IsZero() && issueTimestamp.Before(*buildTimeStamp) { + return true + } + status := getPastStatus(transitions, buildTimeStamp) + return statusComplete.Has(strings.ToLower(status)) +} + +func getPastStatus(transitions []releasecontroller.Transition, buildTime *time.Time) string { + status := "New" + for _, t := range transitions { + if t.Time.After(*buildTime) { + break + } + status = t.ToStatus + } + return status +} type FeatureTree struct { IssueKey string `json:"key"` @@ -976,182 +975,182 @@ func (c *Controller) getReleaseTagInfo(req *http.Request) (*releaseTagInfo, erro }, nil } -//type Sections struct { -// Tickets []*FeatureTree -// Title string -// Header string -// Note string -//} - -//type httpFeatureData struct { -// DisplaySections []SectionInfo -// From string -// To string -//} - -//type SectionInfo struct { -// Name string -// Section Sections -//} - -//func sortByTitle(features []*FeatureTree) { -// sort.Slice(features, func(i, j int) bool { -// return features[i].IssueKey < features[j].IssueKey -// }) -//} - -//func (c *Controller) httpFeatureInfo(w http.ResponseWriter, req *http.Request) { -// tagInfo, err := c.getReleaseTagInfo(req) -// if err != nil { -// http.Error(w, err.Error(), http.StatusNotFound) -// return -// } -// -// from := req.URL.Query().Get("from") -// if from == "" { -// from = "the last version" -// } -// -// featureTrees, err := c.releaseFeatureInfo(tagInfo) -// if err != nil { -// http.Error(w, err.Error(), http.StatusInternalServerError) -// return -// } -// -// var ( -// buf bytes.Buffer -// completedFeatures []*FeatureTree -// unCompletedFeatures []*FeatureTree -// completedEpicWithoutFeature []*FeatureTree -// unCompletedEpicWithoutFeature []*FeatureTree -// completedNoEpicNoFeature []*FeatureTree -// unCompletedNoEpicNoFeature []*FeatureTree -// ) -// -// for _, feature := range featureTrees { -// if !unlinkedIssuesSections.Has(feature.NotLinkedType) { -// if feature.IncludedInBuild { -// completedFeatures = append(completedFeatures, feature) -// } else { -// unCompletedFeatures = append(unCompletedFeatures, feature) -// } -// } -// if feature.NotLinkedType == sectionTypeNoFeatureWithEpic { -// if feature.IncludedInBuild { -// completedEpicWithoutFeature = append(completedEpicWithoutFeature, feature) -// } else { -// unCompletedEpicWithoutFeature = append(unCompletedEpicWithoutFeature, feature) -// } -// -// } -// if feature.NotLinkedType == sectionTypeNoEpicNoFeature { -// if feature.IncludedInBuild { -// completedNoEpicNoFeature = append(completedNoEpicNoFeature, feature) -// } else { -// unCompletedNoEpicNoFeature = append(unCompletedNoEpicNoFeature, feature) -// } -// } -// } -// for _, s := range [][]*FeatureTree{completedFeatures, unCompletedFeatures, completedEpicWithoutFeature, unCompletedEpicWithoutFeature} { -// sortByTitle(s) -// // TODO - check this, should be moot, since every leaf has a PR linked to it -// //sortByPRs(s, 1000) -// } -// -// var sections []SectionInfo -// -// // define the UI sections -// completed := Sections{ -// Tickets: completedFeatures, -// Title: "Lists of features that were completed when this image was built", -// Header: "Complete Features", -// Note: "These features were completed when this image was assembled", -// } -// unCompleted := Sections{ -// Tickets: unCompletedFeatures, -// Title: "Lists of features that were not completed when this image was built", -// Header: "Incomplete Features", -// Note: "When this image was assembled, these features were not yet completed. Therefore, only the Jira Cards included here are part of this release", -// } -// completedEpicWithoutFeatureSection := Sections{ -// Tickets: completedEpicWithoutFeature, -// Title: "", -// Header: "Complete Epics", -// Note: "This section includes Jira cards that are linked to an Epic, but the Epic itself is not linked to any Feature. These epics were completed when this image was assembled", -// } -// unCompletedEpicWithoutFeatureSection := Sections{ -// Tickets: unCompletedEpicWithoutFeature, -// Title: "", -// Header: "Incomplete Epics", -// Note: "This section includes Jira cards that are linked to an Epic, but the Epic itself is not linked to any Feature. These epics were not completed when this image was assembled", -// } -// completedNoEpicNoFeatureSection := Sections{ -// Tickets: completedNoEpicNoFeature, -// Title: "", -// Header: "Other Complete", -// Note: "This section includes Jira cards that are not linked to either an Epic or a Feature. These tickets were completed when this image was assembled", -// } -// unCompletedNoEpicNoFeatureSection := Sections{ -// Tickets: unCompletedNoEpicNoFeature, -// Title: "", -// Header: "Other Incomplete", -// Note: "This section includes Jira cards that are not linked to either an Epic or a Feature. These tickets were not completed when this image was assembled", -// } -// -// // the key needs to be a unique value per section -// for _, section := range []SectionInfo{ -// {"completed_features", completed}, -// {"uncompleted_features", unCompleted}, -// {"completed_epic_without_feature", completedEpicWithoutFeatureSection}, -// {"uncompleted_epic_without_feature", unCompletedEpicWithoutFeatureSection}, -// {"completed_no_epic_no_feature", completedNoEpicNoFeatureSection}, -// {"uncompleted_no_epic_no_feature", unCompletedNoEpicNoFeatureSection}, -// } { -// if len(section.Section.Tickets) > 0 { -// sections = append(sections, section) -// } -// } -// -// data := template.Must(template.New("featureRelease.html").Funcs( -// template.FuncMap{ -// "jumpLinks": jumpLinks, -// "includeKey": includeKey, -// }, -// ).ParseFS(resources, "featureRelease.html")) -// -// err = data.Execute(&buf, httpFeatureData{ -// DisplaySections: sections, -// To: tagInfo.Tag, -// From: from, -// }) -// -// if err != nil { -// klog.Errorf("Unable to render page: %v", err) -// http.Error(w, "Unable to render page", http.StatusInternalServerError) -// return -// } -// -// w.Header().Set("Content-Type", "text/html;charset=UTF-8") -// if _, err := w.Write(buf.Bytes()); err != nil { -// http.Error(w, err.Error(), http.StatusInternalServerError) -// } -//} - -//func includeKey(key string) bool { -// return !unlinkedIssuesSections.Has(key) -//} - -//func jumpLinks(data httpFeatureData) string { -// var sb strings.Builder -// for _, s := range data.DisplaySections { -// if len(s.Section.Tickets) > 0 { -// link := fmt.Sprintf("%s", template.HTMLEscapeString(s.Name), template.HTMLEscapeString(s.Section.Header)) -// sb.WriteString(link) -// sb.WriteString(" | ") -// } -// } -// return sb.String() -//} +type Sections struct { + Tickets []*FeatureTree + Title string + Header string + Note string +} + +type httpFeatureData struct { + DisplaySections []SectionInfo + From string + To string +} + +type SectionInfo struct { + Name string + Section Sections +} + +func sortByTitle(features []*FeatureTree) { + sort.Slice(features, func(i, j int) bool { + return features[i].IssueKey < features[j].IssueKey + }) +} + +func (c *Controller) httpFeatureInfo(w http.ResponseWriter, req *http.Request) { + tagInfo, err := c.getReleaseTagInfo(req) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + from := req.URL.Query().Get("from") + if from == "" { + from = "the last version" + } + + featureTrees, err := c.releaseFeatureInfo(tagInfo) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var ( + buf bytes.Buffer + completedFeatures []*FeatureTree + unCompletedFeatures []*FeatureTree + completedEpicWithoutFeature []*FeatureTree + unCompletedEpicWithoutFeature []*FeatureTree + completedNoEpicNoFeature []*FeatureTree + unCompletedNoEpicNoFeature []*FeatureTree + ) + + for _, feature := range featureTrees { + if !unlinkedIssuesSections.Has(feature.NotLinkedType) { + if feature.IncludedInBuild { + completedFeatures = append(completedFeatures, feature) + } else { + unCompletedFeatures = append(unCompletedFeatures, feature) + } + } + if feature.NotLinkedType == sectionTypeNoFeatureWithEpic { + if feature.IncludedInBuild { + completedEpicWithoutFeature = append(completedEpicWithoutFeature, feature) + } else { + unCompletedEpicWithoutFeature = append(unCompletedEpicWithoutFeature, feature) + } + + } + if feature.NotLinkedType == sectionTypeNoEpicNoFeature { + if feature.IncludedInBuild { + completedNoEpicNoFeature = append(completedNoEpicNoFeature, feature) + } else { + unCompletedNoEpicNoFeature = append(unCompletedNoEpicNoFeature, feature) + } + } + } + for _, s := range [][]*FeatureTree{completedFeatures, unCompletedFeatures, completedEpicWithoutFeature, unCompletedEpicWithoutFeature} { + sortByTitle(s) + // TODO - check this, should be moot, since every leaf has a PR linked to it + //sortByPRs(s, 1000) + } + + var sections []SectionInfo + + // define the UI sections + completed := Sections{ + Tickets: completedFeatures, + Title: "Lists of features that were completed when this image was built", + Header: "Complete Features", + Note: "These features were completed when this image was assembled", + } + unCompleted := Sections{ + Tickets: unCompletedFeatures, + Title: "Lists of features that were not completed when this image was built", + Header: "Incomplete Features", + Note: "When this image was assembled, these features were not yet completed. Therefore, only the Jira Cards included here are part of this release", + } + completedEpicWithoutFeatureSection := Sections{ + Tickets: completedEpicWithoutFeature, + Title: "", + Header: "Complete Epics", + Note: "This section includes Jira cards that are linked to an Epic, but the Epic itself is not linked to any Feature. These epics were completed when this image was assembled", + } + unCompletedEpicWithoutFeatureSection := Sections{ + Tickets: unCompletedEpicWithoutFeature, + Title: "", + Header: "Incomplete Epics", + Note: "This section includes Jira cards that are linked to an Epic, but the Epic itself is not linked to any Feature. These epics were not completed when this image was assembled", + } + completedNoEpicNoFeatureSection := Sections{ + Tickets: completedNoEpicNoFeature, + Title: "", + Header: "Other Complete", + Note: "This section includes Jira cards that are not linked to either an Epic or a Feature. These tickets were completed when this image was assembled", + } + unCompletedNoEpicNoFeatureSection := Sections{ + Tickets: unCompletedNoEpicNoFeature, + Title: "", + Header: "Other Incomplete", + Note: "This section includes Jira cards that are not linked to either an Epic or a Feature. These tickets were not completed when this image was assembled", + } + + // the key needs to be a unique value per section + for _, section := range []SectionInfo{ + {"completed_features", completed}, + {"uncompleted_features", unCompleted}, + {"completed_epic_without_feature", completedEpicWithoutFeatureSection}, + {"uncompleted_epic_without_feature", unCompletedEpicWithoutFeatureSection}, + {"completed_no_epic_no_feature", completedNoEpicNoFeatureSection}, + {"uncompleted_no_epic_no_feature", unCompletedNoEpicNoFeatureSection}, + } { + if len(section.Section.Tickets) > 0 { + sections = append(sections, section) + } + } + + data := template.Must(template.New("featureRelease.html").Funcs( + template.FuncMap{ + "jumpLinks": jumpLinks, + "includeKey": includeKey, + }, + ).ParseFS(resources, "featureRelease.html")) + + err = data.Execute(&buf, httpFeatureData{ + DisplaySections: sections, + To: tagInfo.Tag, + From: from, + }) + + if err != nil { + klog.Errorf("Unable to render page: %v", err) + http.Error(w, "Unable to render page", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/html;charset=UTF-8") + if _, err := w.Write(buf.Bytes()); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func includeKey(key string) bool { + return !unlinkedIssuesSections.Has(key) +} + +func jumpLinks(data httpFeatureData) string { + var sb strings.Builder + for _, s := range data.DisplaySections { + if len(s.Section.Tickets) > 0 { + link := fmt.Sprintf("%s", template.HTMLEscapeString(s.Name), template.HTMLEscapeString(s.Section.Header)) + sb.WriteString(link) + sb.WriteString(" | ") + } + } + return sb.String() +} func previousMinor(tagInfo *releaseTagInfo) string { var v semver.Versions @@ -1218,13 +1217,31 @@ func (c *Controller) httpReleaseInfo(w http.ResponseWriter, req *http.Request) { fmt.Fprintf(w, "

Back to index

\n") - fmt.Fprintf(w, "
"+ - "
"+ - "
"+ - "

%s

"+ - "
"+ - "
"+ - "
", template.HTMLEscapeString(tagInfo.Tag)) + if previousMinor(tagInfo) == "" { + fmt.Fprintf(w, "
"+ + "
"+ + "
"+ + "

%s

"+ + "
"+ + "
"+ + "
", template.HTMLEscapeString(tagInfo.Tag)) + } else { + fmt.Fprintf(w, "
"+ + "
"+ + "
"+ + "

%s

"+ + "
"+ + "
"+ + "
"+ + "
"+ + ""+ + "
"+ + "
"+ + "

New features since version %s

"+ + "
"+ + "
"+ + "
", template.HTMLEscapeString(tagInfo.Tag), template.HTMLEscapeString(tagInfo.Tag), previousMinor(tagInfo), previousMinor(tagInfo)) + } switch tagInfo.Info.Tag.Annotations[releasecontroller.ReleaseAnnotationPhase] { case releasecontroller.ReleasePhaseFailed: From 47a29e5ff512fd834c642b1676ad92ebaf063fa6 Mon Sep 17 00:00:00 2001 From: Eris Hoxha Date: Mon, 24 Feb 2025 21:19:05 +0100 Subject: [PATCH 2/5] enable feature analysis, limit jira API calls --- go.mod | 1 + go.sum | 2 + pkg/release-controller/release_info.go | 151 ++++++++++++++++--------- vendor/modules.txt | 3 + 4 files changed, 102 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 75df85b9d..caf300460 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/openshift/ci-tools v0.0.0-20240710031808-de122ac79fa9 github.com/openshift/client-go v3.9.0+incompatible github.com/openshift/library-go v0.0.0-20231017173800-126f85ed0cc7 + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.19.1 github.com/russross/blackfriday v2.0.0+incompatible github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index dfabfaabd..9764fce63 100644 --- a/go.sum +++ b/go.sum @@ -520,6 +520,8 @@ github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3/go.mod h1:cwhy github.com/openshift/golang-glog v0.0.0-20190322123450-3c92600d7533/go.mod h1:3sa6LKKRDnR1xy4Kn8htvPwqIOVwXh8fIU3LRY22q3U= github.com/openshift/library-go v0.0.0-20231017173800-126f85ed0cc7 h1:pJLcCSJzdiWCaJ4bAepgnvwMdP33LumbVJyWSW7+3ng= github.com/openshift/library-go v0.0.0-20231017173800-126f85ed0cc7/go.mod h1:jgxNp8aApJnZtECid9SUSr5Bu6DLo8Hfdv1DgFZaYA8= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/pkg/release-controller/release_info.go b/pkg/release-controller/release_info.go index 878f2f920..5ce0efcde 100644 --- a/pkg/release-controller/release_info.go +++ b/pkg/release-controller/release_info.go @@ -17,6 +17,7 @@ import ( jiraBaseClient "github.com/andygrunwald/go-jira" "github.com/golang/groupcache" imagereference "github.com/openshift/library-go/pkg/image/reference" + "github.com/patrickmn/go-cache" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -42,7 +43,7 @@ const ( JiraTypeMarketProblem = "Market Problem" ) -const maxChunkSize = 500 +const maxChunkSize = 450 // this seems to be the maximum Jira can handle, currently type CachingReleaseInfo struct { cache *groupcache.Group @@ -170,6 +171,7 @@ type ExecReleaseInfo struct { name string imageNameFn func() (string, error) jiraClient jira.Client + jiraCache *cache.Cache } // NewExecReleaseInfo creates a stateful set, in the specified namespace, that provides git changelogs to the @@ -183,6 +185,7 @@ func NewExecReleaseInfo(client kubernetes.Interface, restConfig *rest.Config, na name: name, imageNameFn: imageNameFn, jiraClient: jiraClient, + jiraCache: cache.New(24*time.Hour, 1*time.Hour), } } @@ -401,7 +404,7 @@ func (r *ExecReleaseInfo) IssuesInfo(changelog string) (string, error) { if err != nil { return "", err } - issuesWithRemoteLinkDetails, err := r.GetRemoteLinksWithConcurrency(issuesWithDemoLinkList) + issuesWithRemoteLinkDetails, err := r.GetRemoteLinksWithConcurrency(issuesWithDemoLinkList, 1*time.Second) if err != nil { return "", err } @@ -600,10 +603,16 @@ func (r *ExecReleaseInfo) GetFeatureChildren(featuresList []string, validityPeri if r.jiraClient == nil { return "", fmt.Errorf("unable to communicate with Jira") } - // loop to start goroutines + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for _, feature := range featuresList { wg.Add(1) limit <- struct{}{} + + <-ticker.C + go func(id string) { defer func() { <-limit }() defer wg.Done() @@ -634,6 +643,35 @@ func (r *ExecReleaseInfo) GetFeatureChildren(featuresList []string, validityPeri return string(a), err } +func (r *ExecReleaseInfo) divideSlice(issues []string, chunk int, skipCache bool) ([][]string, []jiraBaseClient.Issue) { + + var result []jiraBaseClient.Issue + uncashedIssues := make([]string, 0) + + if !skipCache { + for _, issue := range issues { + cashedIssue, found := r.jiraCache.Get(issue) + if found { + result = append(result, cashedIssue.(jiraBaseClient.Issue)) + } else { + uncashedIssues = append(uncashedIssues, issue) + } + } + } else { + uncashedIssues = issues + } + + var divided [][]string + for index := 0; index < len(uncashedIssues); index += chunk { + end := index + chunk + if end > len(issues) { + end = len(issues) + } + divided = append(divided, issues[index:end]) + } + return divided, result +} + func (r *ExecReleaseInfo) GetIssuesWithChunks(issues []string) (result []jiraBaseClient.Issue, err error) { // This will prevent a Panic if/when the release-controller's are run without the necessary jira flags if r.jiraClient == nil { @@ -643,7 +681,8 @@ func (r *ExecReleaseInfo) GetIssuesWithChunks(issues []string) (result []jiraBas chunk := maxChunkSize // Divide issues into chunks - dividedIssues := divideSlice(issues, chunk) + dividedIssues, cashedIssues := r.divideSlice(issues, chunk, false) + result = append(result, cashedIssues...) // Search for issues in parallel var wg sync.WaitGroup @@ -723,6 +762,9 @@ func (r *ExecReleaseInfo) GetIssuesWithChunks(issues []string) (result []jiraBas } mu.Lock() defer mu.Unlock() + for _, issue := range issues { + r.jiraCache.Set(issue.Key, issue, 0) + } result = append(result, issues...) }(jql) } @@ -741,27 +783,24 @@ func (r *ExecReleaseInfo) GetIssuesWithDemoLink(issues []string) (result []jiraB if r.jiraClient == nil { return result, fmt.Errorf("unable to communicate with Jira") } - // Keep the chunk on the small side, it is much faster - // There is a limit for API calls per second in Akamai for Jira, don't chunk too much - chunk := len(issues) / 10 - // Jira can't handle more than 500 IDs at once - if chunk > maxChunkSize { - chunk = maxChunkSize - } - if chunk < 50 { - chunk = 50 - } - // Divide issues into chunks - dividedIssues := divideSlice(issues, chunk) + chunk := maxChunkSize + + dividedIssues, _ := r.divideSlice(issues, chunk, true) - // Search for issues in parallel var wg sync.WaitGroup var mu sync.Mutex var buf bytes.Buffer + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for _, parts := range dividedIssues { wg.Add(1) jql := fmt.Sprintf("issueFunction in linkedIssuesOfremote(\"demo\") AND id IN (%s)", strings.Join(parts, ",")) + + <-ticker.C + go func(jql string) { defer wg.Done() issues, _, err := r.jiraClient.SearchWithContext( @@ -793,73 +832,75 @@ func (r *ExecReleaseInfo) GetIssuesWithDemoLink(issues []string) (result []jiraB return result, err } -func (r *ExecReleaseInfo) GetRemoteLinksWithConcurrency(issues []string) (result map[string][]string, err error) { - // This will prevent a Panic if/when the release-controller's are run without the necessary jira flags +func (r *ExecReleaseInfo) GetRemoteLinksWithConcurrency(issues []string, requestInterval time.Duration) (map[string][]string, error) { + // Prevent Panic if Jira client is not set if r.jiraClient == nil { - return result, fmt.Errorf("unable to communicate with Jira") + return nil, fmt.Errorf("unable to communicate with Jira") } var ( mapIssueDemoLink = make(map[string][]string) wg sync.WaitGroup mu sync.Mutex - maxWorkers = 10 - workers = make(chan struct{}, maxWorkers) - buf bytes.Buffer + workers = make(chan struct{}, 5) // it does not make sense to have more workers since the API rate is limited + ticker = time.NewTicker(requestInterval) + errorBuilder strings.Builder + demoRegex = regexp.MustCompile(`\bdemo\b`) // Precompile regex ) + defer ticker.Stop() + worker := func(issue string) { defer wg.Done() - defer func() { <-workers }() + <-ticker.C // Wait for API call slot - links, err := r.jiraClient.GetRemoteLinks(issue) - if err != nil { - mu.Lock() - err = fmt.Errorf("search failed: %w", err) - buf.WriteString(err.Error() + "\n") - mu.Unlock() - return + var links []jiraBaseClient.RemoteLink + if cachedLinks, found := r.jiraCache.Get(fmt.Sprintf("%s_remote_link", issue)); found { + links = cachedLinks.([]jiraBaseClient.RemoteLink) + } else { + var err error + links, err = r.jiraClient.GetRemoteLinks(issue) + if err != nil { + mu.Lock() + errorBuilder.WriteString(fmt.Sprintf("search failed for issue %s: %v\n", issue, err)) + mu.Unlock() + return + } + r.jiraCache.Set(fmt.Sprintf("%s_remote_link", issue), links, 0) } - mu.Lock() - var linkUrl []string - demoRegex := regexp.MustCompile(`\bdemo\b`) + // Process links + var linkUrls []string for _, link := range links { - if matched := demoRegex.MatchString(strings.ToLower(link.Object.Title)); matched { - linkUrl = append(linkUrl, link.Object.URL) + if demoRegex.MatchString(strings.ToLower(link.Object.Title)) { + linkUrls = append(linkUrls, link.Object.URL) } } - mapIssueDemoLink[issue] = linkUrl + + // Store results + mu.Lock() + mapIssueDemoLink[issue] = linkUrls mu.Unlock() } for _, issue := range issues { wg.Add(1) - workers <- struct{}{} - go worker(issue) + workers <- struct{}{} // Acquire worker slot + + go func(issue string) { + <-workers // Release worker slot immediately + worker(issue) + }(issue) } wg.Wait() - close(workers) - if buf.Len() > 0 { - err = stdErrors.New(buf.String()) + if errorBuilder.Len() > 0 { + return mapIssueDemoLink, stdErrors.New(errorBuilder.String()) } - return mapIssueDemoLink, err -} - -func divideSlice(issues []string, chunk int) [][]string { - var divided [][]string - for index := 0; index < len(issues); index += chunk { - end := index + chunk - if end > len(issues) { - end = len(issues) - } - divided = append(divided, issues[index:end]) - } - return divided + return mapIssueDemoLink, nil } func extractIssuesFromChangeLog(changelog ChangeLog, bugSource string) map[string][]string { diff --git a/vendor/modules.txt b/vendor/modules.txt index abc34a48a..a00e7c6e7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -643,6 +643,9 @@ github.com/openshift/library-go/pkg/network github.com/openshift/library-go/pkg/operator/events github.com/openshift/library-go/pkg/operator/v1helpers github.com/openshift/library-go/pkg/serviceability +# github.com/patrickmn/go-cache v2.1.0+incompatible +## explicit +github.com/patrickmn/go-cache # github.com/peterbourgon/diskv v2.0.1+incompatible ## explicit github.com/peterbourgon/diskv From a21911d1d5aaead9431adff83909e9c908875b76 Mon Sep 17 00:00:00 2001 From: Eris Hoxha Date: Mon, 24 Feb 2025 21:24:29 +0100 Subject: [PATCH 3/5] update vendor --- .../patrickmn/go-cache/CONTRIBUTORS | 9 + vendor/github.com/patrickmn/go-cache/LICENSE | 19 + .../github.com/patrickmn/go-cache/README.md | 83 ++ vendor/github.com/patrickmn/go-cache/cache.go | 1161 +++++++++++++++++ .../github.com/patrickmn/go-cache/sharded.go | 192 +++ 5 files changed, 1464 insertions(+) create mode 100644 vendor/github.com/patrickmn/go-cache/CONTRIBUTORS create mode 100644 vendor/github.com/patrickmn/go-cache/LICENSE create mode 100644 vendor/github.com/patrickmn/go-cache/README.md create mode 100644 vendor/github.com/patrickmn/go-cache/cache.go create mode 100644 vendor/github.com/patrickmn/go-cache/sharded.go diff --git a/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS b/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS new file mode 100644 index 000000000..2b16e9974 --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/CONTRIBUTORS @@ -0,0 +1,9 @@ +This is a list of people who have contributed code to go-cache. They, or their +employers, are the copyright holders of the contributed code. Contributed code +is subject to the license restrictions listed in LICENSE (as they were when the +code was contributed.) + +Dustin Sallings +Jason Mooberry +Sergey Shepelev +Alex Edwards diff --git a/vendor/github.com/patrickmn/go-cache/LICENSE b/vendor/github.com/patrickmn/go-cache/LICENSE new file mode 100644 index 000000000..db9903c75 --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012-2017 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/patrickmn/go-cache/README.md b/vendor/github.com/patrickmn/go-cache/README.md new file mode 100644 index 000000000..c5789cc66 --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/README.md @@ -0,0 +1,83 @@ +# go-cache + +go-cache is an in-memory key:value store/cache similar to memcached that is +suitable for applications running on a single machine. Its major advantage is +that, being essentially a thread-safe `map[string]interface{}` with expiration +times, it doesn't need to serialize or transmit its contents over the network. + +Any object can be stored, for a given duration or forever, and the cache can be +safely used by multiple goroutines. + +Although go-cache isn't meant to be used as a persistent datastore, the entire +cache can be saved to and loaded from a file (using `c.Items()` to retrieve the +items map to serialize, and `NewFrom()` to create a cache from a deserialized +one) to recover from downtime quickly. (See the docs for `NewFrom()` for caveats.) + +### Installation + +`go get github.com/patrickmn/go-cache` + +### Usage + +```go +import ( + "fmt" + "github.com/patrickmn/go-cache" + "time" +) + +func main() { + // Create a cache with a default expiration time of 5 minutes, and which + // purges expired items every 10 minutes + c := cache.New(5*time.Minute, 10*time.Minute) + + // Set the value of the key "foo" to "bar", with the default expiration time + c.Set("foo", "bar", cache.DefaultExpiration) + + // Set the value of the key "baz" to 42, with no expiration time + // (the item won't be removed until it is re-set, or removed using + // c.Delete("baz") + c.Set("baz", 42, cache.NoExpiration) + + // Get the string associated with the key "foo" from the cache + foo, found := c.Get("foo") + if found { + fmt.Println(foo) + } + + // Since Go is statically typed, and cache values can be anything, type + // assertion is needed when values are being passed to functions that don't + // take arbitrary types, (i.e. interface{}). The simplest way to do this for + // values which will only be used once--e.g. for passing to another + // function--is: + foo, found := c.Get("foo") + if found { + MyFunction(foo.(string)) + } + + // This gets tedious if the value is used several times in the same function. + // You might do either of the following instead: + if x, found := c.Get("foo"); found { + foo := x.(string) + // ... + } + // or + var foo string + if x, found := c.Get("foo"); found { + foo = x.(string) + } + // ... + // foo can then be passed around freely as a string + + // Want performance? Store pointers! + c.Set("foo", &MyStruct, cache.DefaultExpiration) + if x, found := c.Get("foo"); found { + foo := x.(*MyStruct) + // ... + } +} +``` + +### Reference + +`godoc` or [http://godoc.org/github.com/patrickmn/go-cache](http://godoc.org/github.com/patrickmn/go-cache) diff --git a/vendor/github.com/patrickmn/go-cache/cache.go b/vendor/github.com/patrickmn/go-cache/cache.go new file mode 100644 index 000000000..db88d2f2c --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/cache.go @@ -0,0 +1,1161 @@ +package cache + +import ( + "encoding/gob" + "fmt" + "io" + "os" + "runtime" + "sync" + "time" +) + +type Item struct { + Object interface{} + Expiration int64 +} + +// Returns true if the item has expired. +func (item Item) Expired() bool { + if item.Expiration == 0 { + return false + } + return time.Now().UnixNano() > item.Expiration +} + +const ( + // For use with functions that take an expiration time. + NoExpiration time.Duration = -1 + // For use with functions that take an expiration time. Equivalent to + // passing in the same expiration duration as was given to New() or + // NewFrom() when the cache was created (e.g. 5 minutes.) + DefaultExpiration time.Duration = 0 +) + +type Cache struct { + *cache + // If this is confusing, see the comment at the bottom of New() +} + +type cache struct { + defaultExpiration time.Duration + items map[string]Item + mu sync.RWMutex + onEvicted func(string, interface{}) + janitor *janitor +} + +// Add an item to the cache, replacing any existing item. If the duration is 0 +// (DefaultExpiration), the cache's default expiration time is used. If it is -1 +// (NoExpiration), the item never expires. +func (c *cache) Set(k string, x interface{}, d time.Duration) { + // "Inlining" of set + var e int64 + if d == DefaultExpiration { + d = c.defaultExpiration + } + if d > 0 { + e = time.Now().Add(d).UnixNano() + } + c.mu.Lock() + c.items[k] = Item{ + Object: x, + Expiration: e, + } + // TODO: Calls to mu.Unlock are currently not deferred because defer + // adds ~200 ns (as of go1.) + c.mu.Unlock() +} + +func (c *cache) set(k string, x interface{}, d time.Duration) { + var e int64 + if d == DefaultExpiration { + d = c.defaultExpiration + } + if d > 0 { + e = time.Now().Add(d).UnixNano() + } + c.items[k] = Item{ + Object: x, + Expiration: e, + } +} + +// Add an item to the cache, replacing any existing item, using the default +// expiration. +func (c *cache) SetDefault(k string, x interface{}) { + c.Set(k, x, DefaultExpiration) +} + +// Add an item to the cache only if an item doesn't already exist for the given +// key, or if the existing item has expired. Returns an error otherwise. +func (c *cache) Add(k string, x interface{}, d time.Duration) error { + c.mu.Lock() + _, found := c.get(k) + if found { + c.mu.Unlock() + return fmt.Errorf("Item %s already exists", k) + } + c.set(k, x, d) + c.mu.Unlock() + return nil +} + +// Set a new value for the cache key only if it already exists, and the existing +// item hasn't expired. Returns an error otherwise. +func (c *cache) Replace(k string, x interface{}, d time.Duration) error { + c.mu.Lock() + _, found := c.get(k) + if !found { + c.mu.Unlock() + return fmt.Errorf("Item %s doesn't exist", k) + } + c.set(k, x, d) + c.mu.Unlock() + return nil +} + +// Get an item from the cache. Returns the item or nil, and a bool indicating +// whether the key was found. +func (c *cache) Get(k string) (interface{}, bool) { + c.mu.RLock() + // "Inlining" of get and Expired + item, found := c.items[k] + if !found { + c.mu.RUnlock() + return nil, false + } + if item.Expiration > 0 { + if time.Now().UnixNano() > item.Expiration { + c.mu.RUnlock() + return nil, false + } + } + c.mu.RUnlock() + return item.Object, true +} + +// GetWithExpiration returns an item and its expiration time from the cache. +// It returns the item or nil, the expiration time if one is set (if the item +// never expires a zero value for time.Time is returned), and a bool indicating +// whether the key was found. +func (c *cache) GetWithExpiration(k string) (interface{}, time.Time, bool) { + c.mu.RLock() + // "Inlining" of get and Expired + item, found := c.items[k] + if !found { + c.mu.RUnlock() + return nil, time.Time{}, false + } + + if item.Expiration > 0 { + if time.Now().UnixNano() > item.Expiration { + c.mu.RUnlock() + return nil, time.Time{}, false + } + + // Return the item and the expiration time + c.mu.RUnlock() + return item.Object, time.Unix(0, item.Expiration), true + } + + // If expiration <= 0 (i.e. no expiration time set) then return the item + // and a zeroed time.Time + c.mu.RUnlock() + return item.Object, time.Time{}, true +} + +func (c *cache) get(k string) (interface{}, bool) { + item, found := c.items[k] + if !found { + return nil, false + } + // "Inlining" of Expired + if item.Expiration > 0 { + if time.Now().UnixNano() > item.Expiration { + return nil, false + } + } + return item.Object, true +} + +// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, +// uint8, uint32, or uint64, float32 or float64 by n. Returns an error if the +// item's value is not an integer, if it was not found, or if it is not +// possible to increment it by n. To retrieve the incremented value, use one +// of the specialized methods, e.g. IncrementInt64. +func (c *cache) Increment(k string, n int64) error { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item %s not found", k) + } + switch v.Object.(type) { + case int: + v.Object = v.Object.(int) + int(n) + case int8: + v.Object = v.Object.(int8) + int8(n) + case int16: + v.Object = v.Object.(int16) + int16(n) + case int32: + v.Object = v.Object.(int32) + int32(n) + case int64: + v.Object = v.Object.(int64) + n + case uint: + v.Object = v.Object.(uint) + uint(n) + case uintptr: + v.Object = v.Object.(uintptr) + uintptr(n) + case uint8: + v.Object = v.Object.(uint8) + uint8(n) + case uint16: + v.Object = v.Object.(uint16) + uint16(n) + case uint32: + v.Object = v.Object.(uint32) + uint32(n) + case uint64: + v.Object = v.Object.(uint64) + uint64(n) + case float32: + v.Object = v.Object.(float32) + float32(n) + case float64: + v.Object = v.Object.(float64) + float64(n) + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s is not an integer", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Increment an item of type float32 or float64 by n. Returns an error if the +// item's value is not floating point, if it was not found, or if it is not +// possible to increment it by n. Pass a negative number to decrement the +// value. To retrieve the incremented value, use one of the specialized methods, +// e.g. IncrementFloat64. +func (c *cache) IncrementFloat(k string, n float64) error { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item %s not found", k) + } + switch v.Object.(type) { + case float32: + v.Object = v.Object.(float32) + float32(n) + case float64: + v.Object = v.Object.(float64) + n + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s does not have type float32 or float64", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Increment an item of type int by n. Returns an error if the item's value is +// not an int, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt(k string, n int) (int, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int8 by n. Returns an error if the item's value is +// not an int8, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt8(k string, n int8) (int8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int8", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int16 by n. Returns an error if the item's value is +// not an int16, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt16(k string, n int16) (int16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int16", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int32 by n. Returns an error if the item's value is +// not an int32, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt32(k string, n int32) (int32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int32", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type int64 by n. Returns an error if the item's value is +// not an int64, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementInt64(k string, n int64) (int64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int64", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint by n. Returns an error if the item's value is +// not an uint, or if it was not found. If there is no error, the incremented +// value is returned. +func (c *cache) IncrementUint(k string, n uint) (uint, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uintptr by n. Returns an error if the item's value +// is not an uintptr, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUintptr(k string, n uintptr) (uintptr, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uintptr) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uintptr", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint8 by n. Returns an error if the item's value +// is not an uint8, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint8(k string, n uint8) (uint8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint8", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint16 by n. Returns an error if the item's value +// is not an uint16, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint16(k string, n uint16) (uint16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint16", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint32 by n. Returns an error if the item's value +// is not an uint32, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint32(k string, n uint32) (uint32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint32", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type uint64 by n. Returns an error if the item's value +// is not an uint64, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementUint64(k string, n uint64) (uint64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint64", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type float32 by n. Returns an error if the item's value +// is not an float32, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementFloat32(k string, n float32) (float32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float32", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Increment an item of type float64 by n. Returns an error if the item's value +// is not an float64, or if it was not found. If there is no error, the +// incremented value is returned. +func (c *cache) IncrementFloat64(k string, n float64) (float64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float64", k) + } + nv := rv + n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int, int8, int16, int32, int64, uintptr, uint, +// uint8, uint32, or uint64, float32 or float64 by n. Returns an error if the +// item's value is not an integer, if it was not found, or if it is not +// possible to decrement it by n. To retrieve the decremented value, use one +// of the specialized methods, e.g. DecrementInt64. +func (c *cache) Decrement(k string, n int64) error { + // TODO: Implement Increment and Decrement more cleanly. + // (Cannot do Increment(k, n*-1) for uints.) + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item not found") + } + switch v.Object.(type) { + case int: + v.Object = v.Object.(int) - int(n) + case int8: + v.Object = v.Object.(int8) - int8(n) + case int16: + v.Object = v.Object.(int16) - int16(n) + case int32: + v.Object = v.Object.(int32) - int32(n) + case int64: + v.Object = v.Object.(int64) - n + case uint: + v.Object = v.Object.(uint) - uint(n) + case uintptr: + v.Object = v.Object.(uintptr) - uintptr(n) + case uint8: + v.Object = v.Object.(uint8) - uint8(n) + case uint16: + v.Object = v.Object.(uint16) - uint16(n) + case uint32: + v.Object = v.Object.(uint32) - uint32(n) + case uint64: + v.Object = v.Object.(uint64) - uint64(n) + case float32: + v.Object = v.Object.(float32) - float32(n) + case float64: + v.Object = v.Object.(float64) - float64(n) + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s is not an integer", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Decrement an item of type float32 or float64 by n. Returns an error if the +// item's value is not floating point, if it was not found, or if it is not +// possible to decrement it by n. Pass a negative number to decrement the +// value. To retrieve the decremented value, use one of the specialized methods, +// e.g. DecrementFloat64. +func (c *cache) DecrementFloat(k string, n float64) error { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return fmt.Errorf("Item %s not found", k) + } + switch v.Object.(type) { + case float32: + v.Object = v.Object.(float32) - float32(n) + case float64: + v.Object = v.Object.(float64) - n + default: + c.mu.Unlock() + return fmt.Errorf("The value for %s does not have type float32 or float64", k) + } + c.items[k] = v + c.mu.Unlock() + return nil +} + +// Decrement an item of type int by n. Returns an error if the item's value is +// not an int, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt(k string, n int) (int, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int8 by n. Returns an error if the item's value is +// not an int8, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt8(k string, n int8) (int8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int8", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int16 by n. Returns an error if the item's value is +// not an int16, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt16(k string, n int16) (int16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int16", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int32 by n. Returns an error if the item's value is +// not an int32, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt32(k string, n int32) (int32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int32", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type int64 by n. Returns an error if the item's value is +// not an int64, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementInt64(k string, n int64) (int64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(int64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an int64", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint by n. Returns an error if the item's value is +// not an uint, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementUint(k string, n uint) (uint, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uintptr by n. Returns an error if the item's value +// is not an uintptr, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUintptr(k string, n uintptr) (uintptr, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uintptr) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uintptr", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint8 by n. Returns an error if the item's value is +// not an uint8, or if it was not found. If there is no error, the decremented +// value is returned. +func (c *cache) DecrementUint8(k string, n uint8) (uint8, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint8) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint8", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint16 by n. Returns an error if the item's value +// is not an uint16, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUint16(k string, n uint16) (uint16, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint16) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint16", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint32 by n. Returns an error if the item's value +// is not an uint32, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUint32(k string, n uint32) (uint32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint32", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type uint64 by n. Returns an error if the item's value +// is not an uint64, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementUint64(k string, n uint64) (uint64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(uint64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an uint64", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type float32 by n. Returns an error if the item's value +// is not an float32, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementFloat32(k string, n float32) (float32, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float32) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float32", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Decrement an item of type float64 by n. Returns an error if the item's value +// is not an float64, or if it was not found. If there is no error, the +// decremented value is returned. +func (c *cache) DecrementFloat64(k string, n float64) (float64, error) { + c.mu.Lock() + v, found := c.items[k] + if !found || v.Expired() { + c.mu.Unlock() + return 0, fmt.Errorf("Item %s not found", k) + } + rv, ok := v.Object.(float64) + if !ok { + c.mu.Unlock() + return 0, fmt.Errorf("The value for %s is not an float64", k) + } + nv := rv - n + v.Object = nv + c.items[k] = v + c.mu.Unlock() + return nv, nil +} + +// Delete an item from the cache. Does nothing if the key is not in the cache. +func (c *cache) Delete(k string) { + c.mu.Lock() + v, evicted := c.delete(k) + c.mu.Unlock() + if evicted { + c.onEvicted(k, v) + } +} + +func (c *cache) delete(k string) (interface{}, bool) { + if c.onEvicted != nil { + if v, found := c.items[k]; found { + delete(c.items, k) + return v.Object, true + } + } + delete(c.items, k) + return nil, false +} + +type keyAndValue struct { + key string + value interface{} +} + +// Delete all expired items from the cache. +func (c *cache) DeleteExpired() { + var evictedItems []keyAndValue + now := time.Now().UnixNano() + c.mu.Lock() + for k, v := range c.items { + // "Inlining" of expired + if v.Expiration > 0 && now > v.Expiration { + ov, evicted := c.delete(k) + if evicted { + evictedItems = append(evictedItems, keyAndValue{k, ov}) + } + } + } + c.mu.Unlock() + for _, v := range evictedItems { + c.onEvicted(v.key, v.value) + } +} + +// Sets an (optional) function that is called with the key and value when an +// item is evicted from the cache. (Including when it is deleted manually, but +// not when it is overwritten.) Set to nil to disable. +func (c *cache) OnEvicted(f func(string, interface{})) { + c.mu.Lock() + c.onEvicted = f + c.mu.Unlock() +} + +// Write the cache's items (using Gob) to an io.Writer. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) Save(w io.Writer) (err error) { + enc := gob.NewEncoder(w) + defer func() { + if x := recover(); x != nil { + err = fmt.Errorf("Error registering item types with Gob library") + } + }() + c.mu.RLock() + defer c.mu.RUnlock() + for _, v := range c.items { + gob.Register(v.Object) + } + err = enc.Encode(&c.items) + return +} + +// Save the cache's items to the given filename, creating the file if it +// doesn't exist, and overwriting it if it does. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) SaveFile(fname string) error { + fp, err := os.Create(fname) + if err != nil { + return err + } + err = c.Save(fp) + if err != nil { + fp.Close() + return err + } + return fp.Close() +} + +// Add (Gob-serialized) cache items from an io.Reader, excluding any items with +// keys that already exist (and haven't expired) in the current cache. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) Load(r io.Reader) error { + dec := gob.NewDecoder(r) + items := map[string]Item{} + err := dec.Decode(&items) + if err == nil { + c.mu.Lock() + defer c.mu.Unlock() + for k, v := range items { + ov, found := c.items[k] + if !found || ov.Expired() { + c.items[k] = v + } + } + } + return err +} + +// Load and add cache items from the given filename, excluding any items with +// keys that already exist in the current cache. +// +// NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the +// documentation for NewFrom().) +func (c *cache) LoadFile(fname string) error { + fp, err := os.Open(fname) + if err != nil { + return err + } + err = c.Load(fp) + if err != nil { + fp.Close() + return err + } + return fp.Close() +} + +// Copies all unexpired items in the cache into a new map and returns it. +func (c *cache) Items() map[string]Item { + c.mu.RLock() + defer c.mu.RUnlock() + m := make(map[string]Item, len(c.items)) + now := time.Now().UnixNano() + for k, v := range c.items { + // "Inlining" of Expired + if v.Expiration > 0 { + if now > v.Expiration { + continue + } + } + m[k] = v + } + return m +} + +// Returns the number of items in the cache. This may include items that have +// expired, but have not yet been cleaned up. +func (c *cache) ItemCount() int { + c.mu.RLock() + n := len(c.items) + c.mu.RUnlock() + return n +} + +// Delete all items from the cache. +func (c *cache) Flush() { + c.mu.Lock() + c.items = map[string]Item{} + c.mu.Unlock() +} + +type janitor struct { + Interval time.Duration + stop chan bool +} + +func (j *janitor) Run(c *cache) { + ticker := time.NewTicker(j.Interval) + for { + select { + case <-ticker.C: + c.DeleteExpired() + case <-j.stop: + ticker.Stop() + return + } + } +} + +func stopJanitor(c *Cache) { + c.janitor.stop <- true +} + +func runJanitor(c *cache, ci time.Duration) { + j := &janitor{ + Interval: ci, + stop: make(chan bool), + } + c.janitor = j + go j.Run(c) +} + +func newCache(de time.Duration, m map[string]Item) *cache { + if de == 0 { + de = -1 + } + c := &cache{ + defaultExpiration: de, + items: m, + } + return c +} + +func newCacheWithJanitor(de time.Duration, ci time.Duration, m map[string]Item) *Cache { + c := newCache(de, m) + // This trick ensures that the janitor goroutine (which--granted it + // was enabled--is running DeleteExpired on c forever) does not keep + // the returned C object from being garbage collected. When it is + // garbage collected, the finalizer stops the janitor goroutine, after + // which c can be collected. + C := &Cache{c} + if ci > 0 { + runJanitor(c, ci) + runtime.SetFinalizer(C, stopJanitor) + } + return C +} + +// Return a new cache with a given default expiration duration and cleanup +// interval. If the expiration duration is less than one (or NoExpiration), +// the items in the cache never expire (by default), and must be deleted +// manually. If the cleanup interval is less than one, expired items are not +// deleted from the cache before calling c.DeleteExpired(). +func New(defaultExpiration, cleanupInterval time.Duration) *Cache { + items := make(map[string]Item) + return newCacheWithJanitor(defaultExpiration, cleanupInterval, items) +} + +// Return a new cache with a given default expiration duration and cleanup +// interval. If the expiration duration is less than one (or NoExpiration), +// the items in the cache never expire (by default), and must be deleted +// manually. If the cleanup interval is less than one, expired items are not +// deleted from the cache before calling c.DeleteExpired(). +// +// NewFrom() also accepts an items map which will serve as the underlying map +// for the cache. This is useful for starting from a deserialized cache +// (serialized using e.g. gob.Encode() on c.Items()), or passing in e.g. +// make(map[string]Item, 500) to improve startup performance when the cache +// is expected to reach a certain minimum size. +// +// Only the cache's methods synchronize access to this map, so it is not +// recommended to keep any references to the map around after creating a cache. +// If need be, the map can be accessed at a later point using c.Items() (subject +// to the same caveat.) +// +// Note regarding serialization: When using e.g. gob, make sure to +// gob.Register() the individual types stored in the cache before encoding a +// map retrieved with c.Items(), and to register those same types before +// decoding a blob containing an items map. +func NewFrom(defaultExpiration, cleanupInterval time.Duration, items map[string]Item) *Cache { + return newCacheWithJanitor(defaultExpiration, cleanupInterval, items) +} diff --git a/vendor/github.com/patrickmn/go-cache/sharded.go b/vendor/github.com/patrickmn/go-cache/sharded.go new file mode 100644 index 000000000..bcc0538bc --- /dev/null +++ b/vendor/github.com/patrickmn/go-cache/sharded.go @@ -0,0 +1,192 @@ +package cache + +import ( + "crypto/rand" + "math" + "math/big" + insecurerand "math/rand" + "os" + "runtime" + "time" +) + +// This is an experimental and unexported (for now) attempt at making a cache +// with better algorithmic complexity than the standard one, namely by +// preventing write locks of the entire cache when an item is added. As of the +// time of writing, the overhead of selecting buckets results in cache +// operations being about twice as slow as for the standard cache with small +// total cache sizes, and faster for larger ones. +// +// See cache_test.go for a few benchmarks. + +type unexportedShardedCache struct { + *shardedCache +} + +type shardedCache struct { + seed uint32 + m uint32 + cs []*cache + janitor *shardedJanitor +} + +// djb2 with better shuffling. 5x faster than FNV with the hash.Hash overhead. +func djb33(seed uint32, k string) uint32 { + var ( + l = uint32(len(k)) + d = 5381 + seed + l + i = uint32(0) + ) + // Why is all this 5x faster than a for loop? + if l >= 4 { + for i < l-4 { + d = (d * 33) ^ uint32(k[i]) + d = (d * 33) ^ uint32(k[i+1]) + d = (d * 33) ^ uint32(k[i+2]) + d = (d * 33) ^ uint32(k[i+3]) + i += 4 + } + } + switch l - i { + case 1: + case 2: + d = (d * 33) ^ uint32(k[i]) + case 3: + d = (d * 33) ^ uint32(k[i]) + d = (d * 33) ^ uint32(k[i+1]) + case 4: + d = (d * 33) ^ uint32(k[i]) + d = (d * 33) ^ uint32(k[i+1]) + d = (d * 33) ^ uint32(k[i+2]) + } + return d ^ (d >> 16) +} + +func (sc *shardedCache) bucket(k string) *cache { + return sc.cs[djb33(sc.seed, k)%sc.m] +} + +func (sc *shardedCache) Set(k string, x interface{}, d time.Duration) { + sc.bucket(k).Set(k, x, d) +} + +func (sc *shardedCache) Add(k string, x interface{}, d time.Duration) error { + return sc.bucket(k).Add(k, x, d) +} + +func (sc *shardedCache) Replace(k string, x interface{}, d time.Duration) error { + return sc.bucket(k).Replace(k, x, d) +} + +func (sc *shardedCache) Get(k string) (interface{}, bool) { + return sc.bucket(k).Get(k) +} + +func (sc *shardedCache) Increment(k string, n int64) error { + return sc.bucket(k).Increment(k, n) +} + +func (sc *shardedCache) IncrementFloat(k string, n float64) error { + return sc.bucket(k).IncrementFloat(k, n) +} + +func (sc *shardedCache) Decrement(k string, n int64) error { + return sc.bucket(k).Decrement(k, n) +} + +func (sc *shardedCache) Delete(k string) { + sc.bucket(k).Delete(k) +} + +func (sc *shardedCache) DeleteExpired() { + for _, v := range sc.cs { + v.DeleteExpired() + } +} + +// Returns the items in the cache. This may include items that have expired, +// but have not yet been cleaned up. If this is significant, the Expiration +// fields of the items should be checked. Note that explicit synchronization +// is needed to use a cache and its corresponding Items() return values at +// the same time, as the maps are shared. +func (sc *shardedCache) Items() []map[string]Item { + res := make([]map[string]Item, len(sc.cs)) + for i, v := range sc.cs { + res[i] = v.Items() + } + return res +} + +func (sc *shardedCache) Flush() { + for _, v := range sc.cs { + v.Flush() + } +} + +type shardedJanitor struct { + Interval time.Duration + stop chan bool +} + +func (j *shardedJanitor) Run(sc *shardedCache) { + j.stop = make(chan bool) + tick := time.Tick(j.Interval) + for { + select { + case <-tick: + sc.DeleteExpired() + case <-j.stop: + return + } + } +} + +func stopShardedJanitor(sc *unexportedShardedCache) { + sc.janitor.stop <- true +} + +func runShardedJanitor(sc *shardedCache, ci time.Duration) { + j := &shardedJanitor{ + Interval: ci, + } + sc.janitor = j + go j.Run(sc) +} + +func newShardedCache(n int, de time.Duration) *shardedCache { + max := big.NewInt(0).SetUint64(uint64(math.MaxUint32)) + rnd, err := rand.Int(rand.Reader, max) + var seed uint32 + if err != nil { + os.Stderr.Write([]byte("WARNING: go-cache's newShardedCache failed to read from the system CSPRNG (/dev/urandom or equivalent.) Your system's security may be compromised. Continuing with an insecure seed.\n")) + seed = insecurerand.Uint32() + } else { + seed = uint32(rnd.Uint64()) + } + sc := &shardedCache{ + seed: seed, + m: uint32(n), + cs: make([]*cache, n), + } + for i := 0; i < n; i++ { + c := &cache{ + defaultExpiration: de, + items: map[string]Item{}, + } + sc.cs[i] = c + } + return sc +} + +func unexportedNewSharded(defaultExpiration, cleanupInterval time.Duration, shards int) *unexportedShardedCache { + if defaultExpiration == 0 { + defaultExpiration = -1 + } + sc := newShardedCache(shards, defaultExpiration) + SC := &unexportedShardedCache{sc} + if cleanupInterval > 0 { + runShardedJanitor(sc, cleanupInterval) + runtime.SetFinalizer(SC, stopShardedJanitor) + } + return SC +} From 85c96ad304090967f2baa18339521321e44f662c Mon Sep 17 00:00:00 2001 From: Eris Hoxha Date: Tue, 25 Feb 2025 16:32:38 +0100 Subject: [PATCH 4/5] log access to httpFeatureInfo --- cmd/release-controller-api/http.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/release-controller-api/http.go b/cmd/release-controller-api/http.go index 1edc3f841..396dd47b6 100644 --- a/cmd/release-controller-api/http.go +++ b/cmd/release-controller-api/http.go @@ -1010,7 +1010,7 @@ func (c *Controller) httpFeatureInfo(w http.ResponseWriter, req *http.Request) { if from == "" { from = "the last version" } - + klog.V(4).Infof("running feature anaysis: Tag %s from %s at %s\n:", tagInfo.Tag, from, time.Now()) featureTrees, err := c.releaseFeatureInfo(tagInfo) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1134,6 +1134,7 @@ func (c *Controller) httpFeatureInfo(w http.ResponseWriter, req *http.Request) { if _, err := w.Write(buf.Bytes()); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } + klog.V(4).Infof("finished running feature anaysis: Tag %s from %s at %s\n:", tagInfo.Tag, from, time.Now()) } func includeKey(key string) bool { From 2b9497f9cb3b6b249f35dc53ac36aad63cb25f07 Mon Sep 17 00:00:00 2001 From: Eris Hoxha Date: Wed, 26 Feb 2025 08:56:58 +0100 Subject: [PATCH 5/5] fix based on review --- pkg/release-controller/release_info.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/release-controller/release_info.go b/pkg/release-controller/release_info.go index 5ce0efcde..1c2356e2d 100644 --- a/pkg/release-controller/release_info.go +++ b/pkg/release-controller/release_info.go @@ -763,7 +763,7 @@ func (r *ExecReleaseInfo) GetIssuesWithChunks(issues []string) (result []jiraBas mu.Lock() defer mu.Unlock() for _, issue := range issues { - r.jiraCache.Set(issue.Key, issue, 0) + r.jiraCache.Set(issue.Key, issue, cache.DefaultExpiration) } result = append(result, issues...) }(jql) @@ -855,7 +855,9 @@ func (r *ExecReleaseInfo) GetRemoteLinksWithConcurrency(issues []string, request <-ticker.C // Wait for API call slot var links []jiraBaseClient.RemoteLink - if cachedLinks, found := r.jiraCache.Get(fmt.Sprintf("%s_remote_link", issue)); found { + issueRemoteLinkKey := fmt.Sprintf("%s_remote_link", issue) + + if cachedLinks, found := r.jiraCache.Get(issueRemoteLinkKey); found { links = cachedLinks.([]jiraBaseClient.RemoteLink) } else { var err error @@ -866,7 +868,7 @@ func (r *ExecReleaseInfo) GetRemoteLinksWithConcurrency(issues []string, request mu.Unlock() return } - r.jiraCache.Set(fmt.Sprintf("%s_remote_link", issue), links, 0) + r.jiraCache.Set(issueRemoteLinkKey, links, cache.DefaultExpiration) } // Process links