From 2202a364c42ccbd496d6094b67b9725426709ca0 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 28 May 2024 21:03:42 +0800 Subject: [PATCH 1/5] fix(linker): link current project's issues --- .../linker/e2e/link_pr_and_issue_test.go | 1 + .../e2e/snapshot_tables/board_issues.csv | 3 +++ .../linker/e2e/snapshot_tables/issues.csv | 1 + .../e2e/snapshot_tables/project_mapping.csv | 3 ++- .../plugins/linker/tasks/link_pr_and_issue.go | 18 +++++++++++++++++- 5 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 backend/plugins/linker/e2e/snapshot_tables/board_issues.csv diff --git a/backend/plugins/linker/e2e/link_pr_and_issue_test.go b/backend/plugins/linker/e2e/link_pr_and_issue_test.go index 321c61723c6..dbec92a600a 100644 --- a/backend/plugins/linker/e2e/link_pr_and_issue_test.go +++ b/backend/plugins/linker/e2e/link_pr_and_issue_test.go @@ -49,6 +49,7 @@ func TestLinkPrToIssue(t *testing.T) { dataflowTester.ImportCsvIntoTabler("./snapshot_tables/issues.csv", &ticket.Issue{}) dataflowTester.ImportCsvIntoTabler("./snapshot_tables/pull_requests.csv", &code.PullRequest{}) dataflowTester.ImportCsvIntoTabler("./snapshot_tables/project_mapping.csv", &crossdomain.ProjectMapping{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/board_issues.csv", &ticket.BoardIssue{}) dataflowTester.FlushTabler(&crossdomain.PullRequestIssue{}) dataflowTester.Subtask(tasks.LinkPrToIssueMeta, taskData) diff --git a/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..8c288bf48cf --- /dev/null +++ b/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,3 @@ +"board_id","issue_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" +"github:GithubRepo:1:384111310","github:GithubIssue:1:1237324696","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,"" +"github:GithubRepo:1:384111310","github:GithubIssue:1:1237324697","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,"" diff --git a/backend/plugins/linker/e2e/snapshot_tables/issues.csv b/backend/plugins/linker/e2e/snapshot_tables/issues.csv index 9ef9a334315..5dde760a080 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/issues.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/issues.csv @@ -2,3 +2,4 @@ "github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" "github:GithubIssue:1:1237324697","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1885","","1885","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" "github:GithubIssue:1:1237324698","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1886","","1886","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" +"github:GithubIssue:2:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":2,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" diff --git a/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv b/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv index 1fb85d1d19c..9b25fc033e8 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv @@ -1,2 +1,3 @@ "project_name","table","row_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" -"GitHub1","repos","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,"" \ No newline at end of file +"GitHub1","repos","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,"" +"GitHub1","boards","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,"" \ No newline at end of file diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go index 35db8cf4b35..7fcd3baa5de 100644 --- a/backend/plugins/linker/tasks/link_pr_and_issue.go +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -80,6 +80,16 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { defer cursor.Close() + var projectIssueIds []string + if err := db.All(&projectIssueIds, + dal.From(ticket.BoardIssue{}), + dal.Select("board_issues.issue_id"), + dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'boards' AND pm.row_id = board_issues.board_id)"), + dal.Where("pm.project_name = ?", data.Options.ProjectName), + ); err != nil { + return err + } + enricher, err := api.NewDataEnricher(api.DataEnricherArgs[code.PullRequest]{ Ctx: taskCtx, Name: code.PullRequest{}.TableName(), @@ -98,7 +108,13 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { } } var issues []*ticket.Issue - if err := db.All(&issues, dal.Where("issue_key in ?", issueKeys)); err != nil { + + var clauses = []dal.Clause{ + dal.From(&ticket.Issue{}), + dal.Where("issues.id in ? AND issues.issue_key in ?", projectIssueIds, issueKeys), + } + + if err := db.All(&issues, clauses...); err != nil { return nil, err } if len(issues) == 0 { From fdde31ad8fb61ba56fa23f1d8e430420cbafd881 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 4 Jun 2024 12:18:28 +0800 Subject: [PATCH 2/5] perf(framework): fetch projects in parallel --- backend/go.mod | 2 +- backend/go.sum | 2 ++ backend/server/services/pipeline.go | 26 ++++++++++++++++++-------- backend/server/services/project.go | 28 +++++++++++++++++++--------- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 8062a749ba0..c892dede1b3 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -33,7 +33,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 - golang.org/x/sync v0.3.0 + golang.org/x/sync v0.7.0 gorm.io/datatypes v1.0.1 gorm.io/driver/mysql v1.5.1 gorm.io/driver/postgres v1.5.2 diff --git a/backend/go.sum b/backend/go.sum index d48de10e0e1..dd6f20f4251 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -680,6 +680,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go index 80ce47e4e9e..13f03e268f8 100644 --- a/backend/server/services/pipeline.go +++ b/backend/server/services/pipeline.go @@ -20,6 +20,7 @@ package services import ( "context" "fmt" + "golang.org/x/sync/errgroup" "net/url" "os" "path/filepath" @@ -180,16 +181,25 @@ func GetPipelines(query *PipelineQuery, shouldSanitize bool) ([]*models.Pipeline if err != nil { return nil, 0, errors.Convert(err) } + + g := new(errgroup.Group) for _, p := range pipelines { - err = fillPipelineDetail(p) - if err != nil { - return nil, 0, err - } - if shouldSanitize { - if err := SanitizePipeline(p); err != nil { - return nil, 0, errors.Convert(err) + tmpPipeline := p + g.Go(func() error { + err = fillPipelineDetail(tmpPipeline) + if err != nil { + return err } - } + if shouldSanitize { + if err := SanitizePipeline(p); err != nil { + return err + } + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, 0, errors.Convert(err) } return pipelines, i, nil } diff --git a/backend/server/services/project.go b/backend/server/services/project.go index 754092ad2c6..81a123c48bd 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -19,6 +19,7 @@ package services import ( "fmt" + "golang.org/x/sync/errgroup" "time" "github.com/apache/incubator-devlake/core/dal" @@ -58,16 +59,25 @@ func GetProjects(query *ProjectQuery) ([]*models.ApiOutputProject, int64, errors if err != nil { return nil, 0, errors.Default.Wrap(err, "error finding DB project") } - var apiOutProjects []*models.ApiOutputProject - for _, project := range projects { - apiOutputProject, err := makeProjectOutput(project, true) - if err != nil { - logger.Error(err, "makeProjectOutput, name: %s", project.Name) - return nil, 0, errors.Default.Wrap(err, "error making project output") - } - apiOutProjects = append(apiOutProjects, apiOutputProject) - } + apiOutProjects := make([]*models.ApiOutputProject, len(projects)) + g := new(errgroup.Group) + for idx, project := range projects { + tmpProject := *project + tmpIdx := idx + g.Go(func() error { + apiOutputProject, err := makeProjectOutput(&tmpProject, true) + if err != nil { + logger.Error(err, "makeProjectOutput, name: %s", project.Name) + return errors.Default.Wrap(err, "error making project output") + } + apiOutProjects[tmpIdx] = apiOutputProject + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, 0, errors.Convert(err) + } return apiOutProjects, count, nil } From d544c63eb4ec59f5dbf1ead52ff32503aa70f6a7 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 4 Jun 2024 12:23:07 +0800 Subject: [PATCH 3/5] fix(framework): fix `copyloopvar` error --- backend/server/services/pipeline.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go index 13f03e268f8..ffd2dce81a7 100644 --- a/backend/server/services/pipeline.go +++ b/backend/server/services/pipeline.go @@ -183,10 +183,11 @@ func GetPipelines(query *PipelineQuery, shouldSanitize bool) ([]*models.Pipeline } g := new(errgroup.Group) - for _, p := range pipelines { - tmpPipeline := p + for idx, p := range pipelines { + tmpPipeline := *p + tmpIdx := idx g.Go(func() error { - err = fillPipelineDetail(tmpPipeline) + err = fillPipelineDetail(&tmpPipeline) if err != nil { return err } @@ -195,6 +196,7 @@ func GetPipelines(query *PipelineQuery, shouldSanitize bool) ([]*models.Pipeline return err } } + pipelines[tmpIdx] = &tmpPipeline return nil }) } From 077d5efea4db1b40ff3c6fe811c8f6e8ca1643b2 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 4 Jun 2024 12:26:09 +0800 Subject: [PATCH 4/5] fix(framework): fix `loopclosure` --- backend/server/services/pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go index ffd2dce81a7..733b78f7761 100644 --- a/backend/server/services/pipeline.go +++ b/backend/server/services/pipeline.go @@ -192,7 +192,7 @@ func GetPipelines(query *PipelineQuery, shouldSanitize bool) ([]*models.Pipeline return err } if shouldSanitize { - if err := SanitizePipeline(p); err != nil { + if err := SanitizePipeline(&tmpPipeline); err != nil { return err } } From b3dcfa6776a7cda23d3320893ef0286d883e9c0b Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 4 Jun 2024 12:27:10 +0800 Subject: [PATCH 5/5] fix(framework): fix lint errors --- backend/server/services/project.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/server/services/project.go b/backend/server/services/project.go index 81a123c48bd..547616d078c 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -67,13 +67,12 @@ func GetProjects(query *ProjectQuery) ([]*models.ApiOutputProject, int64, errors g.Go(func() error { apiOutputProject, err := makeProjectOutput(&tmpProject, true) if err != nil { - logger.Error(err, "makeProjectOutput, name: %s", project.Name) + logger.Error(err, "makeProjectOutput, name: %s", tmpProject.Name) return errors.Default.Wrap(err, "error making project output") } apiOutProjects[tmpIdx] = apiOutputProject return nil }) - } if err := g.Wait(); err != nil { return nil, 0, errors.Convert(err)