Skip to content

Commit

Permalink
perf: hibernate check optimisation (#3788)
Browse files Browse the repository at this point in the history 
* check for kind

* hibernation replica parallelism

* clean dead code

* refactoring

---------

Co-authored-by: Ashish-devtron <ashish.kumar@devtron.ai>
  • Loading branch information
@kripanshdevtron @Ashish-devtron
kripanshdevtron and Ashish-devtron committed Aug 17, 2023
1 parent 6441783 commit 91efceb
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 47 deletions.
140 changes: 97 additions & 43 deletions pkg/appStore/deployment/service/InstalledAppService.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ import (
"github.com/tidwall/gjson"
"net/http"
"regexp"
"sync"
"sync/atomic"

/* #nosec */
"crypto/sha1"
Expand Down Expand Up @@ -1207,57 +1209,39 @@ func (impl InstalledAppServiceImpl) checkHibernate(resp map[string]interface{},
return resp, ""
}
responseTree := resp
canBeHibernated := 0
hibernated := 0
var canBeHibernated uint64 = 0
var hibernated uint64 = 0
responseTreeNodes, ok := responseTree["nodes"]
if !ok {
return resp, ""
}
for _, node := range responseTreeNodes.(interface{}).([]interface{}) {
currNode := node.(interface{}).(map[string]interface{})
resName := util3.InterfaceToString(currNode["name"])
resKind := util3.InterfaceToString(currNode["kind"])
resGroup := util3.InterfaceToString(currNode["group"])
resVersion := util3.InterfaceToString(currNode["version"])
resNamespace := util3.InterfaceToString(currNode["namespace"])
rQuery := &application.ApplicationResourceRequest{
Name: &deploymentAppName,
ResourceName: &resName,
Kind: &resKind,
Group: &resGroup,
Version: &resVersion,
Namespace: &resNamespace,
}
ctx, _ := context.WithTimeout(ctx, 60*time.Second)
if currNode["parentRefs"] == nil {
t0 := time.Now()
res, err := impl.acdClient.GetResource(ctx, rQuery)
if err != nil {
impl.logger.Errorw("error getting response from acdClient", "request", rQuery, "data", res, "timeTaken", time.Since(t0), "err", err)
continue
}
if res.Manifest != nil {
manifest, _ := gjson.Parse(*res.Manifest).Value().(map[string]interface{})
replicas := util3.InterfaceToMapAdapter(manifest["spec"])["replicas"]
if replicas != nil {
currNode["canBeHibernated"] = true
canBeHibernated++
replicaNodes := impl.filterOutReplicaNodes(responseTreeNodes)
batchSize := impl.aCDAuthConfig.ResourceListForReplicasBatchSize
requestsLength := len(replicaNodes)
for i := 0; i < requestsLength; {
//requests left to process
remainingBatch := requestsLength - i
if remainingBatch < batchSize {
batchSize = remainingBatch
}
var wg sync.WaitGroup
for j := 0; j < batchSize; j++ {
wg.Add(1)
go func(j int) {
defer wg.Done()
canBeHibernatedFlag, hibernatedFlag := impl.processReplicaNodeForHibernation(replicaNodes[i+j], deploymentAppName, ctx)
if canBeHibernatedFlag {
atomic.AddUint64(&canBeHibernated, 1)
}
annotations := util3.InterfaceToMapAdapter(manifest["metadata"])["annotations"]
if annotations != nil {
val := util3.InterfaceToMapAdapter(annotations)["hibernator.devtron.ai/replicas"]
if val != nil {
if util3.InterfaceToString(val) != "0" && util3.InterfaceToFloat(replicas) == 0 {
currNode["isHibernated"] = true
hibernated++
}
}
if hibernatedFlag {
atomic.AddUint64(&hibernated, 1)
}
}

}(j)
}
node = currNode
wg.Wait()
i += batchSize
}

status := ""
if hibernated > 0 && canBeHibernated > 0 {
if hibernated == canBeHibernated {
Expand All @@ -1270,6 +1254,61 @@ func (impl InstalledAppServiceImpl) checkHibernate(resp map[string]interface{},
return responseTree, status
}

func (impl InstalledAppServiceImpl) processReplicaNodeForHibernation(node interface{}, deploymentAppName string, ctx context.Context) (bool, bool) {
currNode := node.(interface{}).(map[string]interface{})
resName := util3.InterfaceToString(currNode["name"])
resKind := util3.InterfaceToString(currNode["kind"])
resGroup := util3.InterfaceToString(currNode["group"])
resVersion := util3.InterfaceToString(currNode["version"])
resNamespace := util3.InterfaceToString(currNode["namespace"])
rQuery := &application.ApplicationResourceRequest{
Name: &deploymentAppName,
ResourceName: &resName,
Kind: &resKind,
Group: &resGroup,
Version: &resVersion,
Namespace: &resNamespace,
}
canBeHibernatedFlag := false
alreadyHibernated := false

if currNode["parentRefs"] == nil {
canBeHibernatedFlag, alreadyHibernated = impl.checkForHibernation(ctx, rQuery, currNode)
}
return canBeHibernatedFlag, alreadyHibernated
}

func (impl InstalledAppServiceImpl) checkForHibernation(ctx context.Context, rQuery *application.ApplicationResourceRequest, currNode map[string]interface{}) (bool, bool) {
t0 := time.Now()
canBeHibernated := false
alreadyHibernated := false
ctx, _ = context.WithTimeout(ctx, 60*time.Second)
res, err := impl.acdClient.GetResource(ctx, rQuery)
if err != nil {
impl.logger.Errorw("error getting response from acdClient", "request", rQuery, "data", res, "timeTaken", time.Since(t0), "err", err)
return canBeHibernated, alreadyHibernated
}
if res.Manifest != nil {
manifest, _ := gjson.Parse(*res.Manifest).Value().(map[string]interface{})
replicas := util3.InterfaceToMapAdapter(manifest["spec"])["replicas"]
if replicas != nil {
currNode["canBeHibernated"] = true
canBeHibernated = true
}
annotations := util3.InterfaceToMapAdapter(manifest["metadata"])["annotations"]
if annotations != nil {
val := util3.InterfaceToMapAdapter(annotations)["hibernator.devtron.ai/replicas"]
if val != nil {
if util3.InterfaceToString(val) != "0" && util3.InterfaceToFloat(replicas) == 0 {
currNode["isHibernated"] = true
alreadyHibernated = true
}
}
}
}
return canBeHibernated, alreadyHibernated
}

func (impl InstalledAppServiceImpl) fetchResourceTreeForACD(rctx context.Context, cn http.CloseNotifier, appId int, envId, clusterId int, deploymentAppName, namespace string) (map[string]interface{}, error) {
var resourceTree map[string]interface{}
query := &application.ResourcesQuery{
Expand Down Expand Up @@ -1334,3 +1373,18 @@ func (impl InstalledAppServiceImpl) fetchResourceTreeForACD(rctx context.Context
impl.logger.Debugf("application %s in environment %s had status %+v\n", appId, envId, resp)
return resourceTree, err
}

func (impl InstalledAppServiceImpl) filterOutReplicaNodes(responseTreeNodes interface{}) []interface{} {
resourceListForReplicas := impl.aCDAuthConfig.ResourceListForReplicas
entries := strings.Split(resourceListForReplicas, ",")
resourceListMap := util3.ConvertStringSliceToMap(entries)
var replicaNodes []interface{}
for _, node := range responseTreeNodes.(interface{}).([]interface{}) {
currNode := node.(interface{}).(map[string]interface{})
resKind := util3.InterfaceToString(currNode["kind"])
if _, ok := resourceListMap[resKind]; ok {
replicaNodes = append(replicaNodes, node)
}
}
return replicaNodes
}
10 changes: 6 additions & 4 deletions pkg/util/TokenCache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,12 @@ func (impl *TokenCache) BuildACDSynchContext() (acdContext context.Context, err
}

type ACDAuthConfig struct {
ACDUsername string `env:"ACD_USERNAME" envDefault:"admin"`
ACDPassword string `env:"ACD_PASSWORD" `
ACDConfigMapName string `env:"ACD_CM" envDefault:"argocd-cm"`
ACDConfigMapNamespace string `env:"ACD_NAMESPACE" envDefault:"devtroncd"`
ACDUsername string `env:"ACD_USERNAME" envDefault:"admin"`
ACDPassword string `env:"ACD_PASSWORD" `
ACDConfigMapName string `env:"ACD_CM" envDefault:"argocd-cm"`
ACDConfigMapNamespace string `env:"ACD_NAMESPACE" envDefault:"devtroncd"`
ResourceListForReplicas string `env:"RESOURCE_LIST_FOR_REPLICAS" envDefault:"Deployment,Rollout,StatefulSet,ReplicaSet"`
ResourceListForReplicasBatchSize int `env:"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE" envDefault:"5"`
}

func GetACDAuthConfig() (*ACDAuthConfig, error) {
Expand Down

0 comments on commit 91efceb

Please sign in to comment.