From 776aa25b5868394a4020a9290862de66af3306ac Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 25 Sep 2024 15:35:25 +0800 Subject: [PATCH 1/2] feat: improve performance of the listing-scope-config-projects-and-scopes --- .../srvhelper/scope_config_service_helper.go | 103 +++++++++--------- 1 file changed, 49 insertions(+), 54 deletions(-) diff --git a/backend/helpers/srvhelper/scope_config_service_helper.go b/backend/helpers/srvhelper/scope_config_service_helper.go index c06354d25c9..0ca3904dca4 100644 --- a/backend/helpers/srvhelper/scope_config_service_helper.go +++ b/backend/helpers/srvhelper/scope_config_service_helper.go @@ -18,6 +18,9 @@ limitations under the License. package srvhelper import ( + "fmt" + "strings" + "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -57,67 +60,59 @@ func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetAllByConnectionId(conne } func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetProjectsByScopeConfig(pluginName string, scopeConfig *SC) (*models.ProjectScopeOutput, errors.Error) { - ps := &models.ProjectScopeOutput{} - projectMap := make(map[string]*models.ProjectScope) - // 1. get all scopes that are using the scopeConfigId - var scope []*S - err := scopeConfigSrv.db.All(&scope, - dal.Where("scope_config_id = ?", (*scopeConfig).ScopeConfigId()), - ) + s := new(S) + // find out the primary key of the scope model + sPk, err := dal.GetPrimarykeyColumnNames(scopeConfigSrv.db, (interface{}(s)).(dal.Tabler)) if err != nil { - return nil, err + panic(err) } - for _, s := range scope { - // 2. get blueprint id by connection id and scope id - bpScope := []*models.BlueprintScope{} - err = scopeConfigSrv.db.All(&bpScope, - dal.Where("plugin_name = ? and connection_id = ? and scope_id = ?", pluginName, (*s).ScopeConnectionId(), (*s).ScopeId()), - ) - if err != nil { - return nil, err - } - - for _, bs := range bpScope { - // 3. get project details by blueprint id - bp := models.Blueprint{} - err = scopeConfigSrv.db.All(&bp, - dal.Where("id = ?", bs.BlueprintId), - ) - if err != nil { - return nil, err - } - if project, exists := projectMap[bp.ProjectName]; exists { - project.Scopes = append(project.Scopes, struct { - ScopeID string `json:"scopeId"` - ScopeName string `json:"scopeName"` - }{ - ScopeID: bs.ScopeId, - ScopeName: (*s).ScopeName(), - }) - } else { - projectMap[bp.ProjectName] = &models.ProjectScope{ - Name: bp.ProjectName, - BlueprintId: bp.ID, - Scopes: []struct { - ScopeID string `json:"scopeId"` - ScopeName string `json:"scopeName"` - }{ - { - ScopeID: bs.ScopeId, - ScopeName: (*s).ScopeName(), - }, - }, - } + if len(sPk) != 2 { + return nil, errors.Internal.New("Scope model should have 2 primary key fields") + } + theOtherPk := sPk[0] + if strings.HasSuffix(theOtherPk, ".connection_id") { + theOtherPk = sPk[1] + } + var bpss []struct { + S *S `gorm:"embedded"` + BlueprintId uint64 + ProjectName string + ScopeId string + } + scopeTable := (*s).TableName() + errors.Must(scopeConfigSrv.db.All( + &bpss, + dal.Select(fmt.Sprintf("bp.id AS blueprint_id, bp.project_name, bps.scope_id, %s.*", scopeTable)), + dal.From("_devlake_blueprint_scopes bps"), + dal.Join("LEFT JOIN _devlake_blueprints bp ON (bp.id = bps.blueprint_id)"), + dal.Join(fmt.Sprintf("LEFT JOIN %s ON (%s.connection_id = bps.connection_id AND %s = bps.scope_id)", scopeTable, scopeTable, theOtherPk)), + dal.Where("bps.plugin_name = ? AND bps.connection_id = ?", pluginName, (*scopeConfig).ScopeConfigConnectionId()), + )) + projectScopeMap := make(map[string]*models.ProjectScope) + for _, bps := range bpss { + if _, ok := projectScopeMap[bps.ProjectName]; !ok { + projectScopeMap[bps.ProjectName] = &models.ProjectScope{ + Name: bps.ProjectName, + BlueprintId: bps.BlueprintId, } } + projectScopeMap[bps.ProjectName].Scopes = append( + projectScopeMap[bps.ProjectName].Scopes, + struct { + ScopeID string "json:\"scopeId\"" + ScopeName string "json:\"scopeName\"" + }{ + ScopeID: bps.ScopeId, + ScopeName: (*bps.S).ScopeName(), + }, + ) } - // 4. combine all projects - for _, project := range projectMap { - ps.Projects = append(ps.Projects, *project) + ps := &models.ProjectScopeOutput{} + for _, projectScope := range projectScopeMap { + ps.Projects = append(ps.Projects, *projectScope) } ps.Count = len(ps.Projects) - - return ps, err + return ps, nil } func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) DeleteScopeConfig(scopeConfig *SC) (refs []*S, err errors.Error) { From e80d086ff07d2e62d38b29db632950876ca70267 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 25 Sep 2024 15:52:44 +0800 Subject: [PATCH 2/2] fix: linting --- backend/helpers/srvhelper/scope_config_service_helper.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/helpers/srvhelper/scope_config_service_helper.go b/backend/helpers/srvhelper/scope_config_service_helper.go index 0ca3904dca4..6c7e3e05f0e 100644 --- a/backend/helpers/srvhelper/scope_config_service_helper.go +++ b/backend/helpers/srvhelper/scope_config_service_helper.go @@ -62,10 +62,7 @@ func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetAllByConnectionId(conne func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetProjectsByScopeConfig(pluginName string, scopeConfig *SC) (*models.ProjectScopeOutput, errors.Error) { s := new(S) // find out the primary key of the scope model - sPk, err := dal.GetPrimarykeyColumnNames(scopeConfigSrv.db, (interface{}(s)).(dal.Tabler)) - if err != nil { - panic(err) - } + sPk := errors.Must1(dal.GetPrimarykeyColumnNames(scopeConfigSrv.db, (interface{}(s)).(dal.Tabler))) if len(sPk) != 2 { return nil, errors.Internal.New("Scope model should have 2 primary key fields") } @@ -99,8 +96,8 @@ func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC]) GetProjectsByScopeConfig(p projectScopeMap[bps.ProjectName].Scopes = append( projectScopeMap[bps.ProjectName].Scopes, struct { - ScopeID string "json:\"scopeId\"" - ScopeName string "json:\"scopeName\"" + ScopeID string `json:"scopeId"` + ScopeName string `json:"scopeName"` }{ ScopeID: bps.ScopeId, ScopeName: (*bps.S).ScopeName(),