From bd3804dec379ae501d8ee42b081dc82c25d0d1d8 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Wed, 15 May 2024 12:03:56 +0800 Subject: [PATCH 01/13] fix(apikey): abort request if api key doesn't match --- backend/server/api/middlewares.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/server/api/middlewares.go b/backend/server/api/middlewares.go index 32ec5e85490..86c8c6a6fb7 100644 --- a/backend/server/api/middlewares.go +++ b/backend/server/api/middlewares.go @@ -121,6 +121,7 @@ func RestAuthentication(router *gin.Engine, basicRes context.BasicRes) gin.Handl authHeader := c.GetHeader("Authorization") ok := CheckAuthorizationHeader(c, logger, db, apiKeyHelper, authHeader, path) if !ok { + c.Abort() return } else { router.HandleContext(c) From 6279bce57bebc3daca7c87bdf3cc08ec7f913ef3 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Fri, 17 May 2024 11:05:45 +0800 Subject: [PATCH 02/13] feat(webhook): update deployment commit id --- backend/plugins/webhook/api/deployments.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/plugins/webhook/api/deployments.go b/backend/plugins/webhook/api/deployments.go index 47ff8e9b9d6..f49591bb336 100644 --- a/backend/plugins/webhook/api/deployments.go +++ b/backend/plugins/webhook/api/deployments.go @@ -69,9 +69,9 @@ type DeploymentCommit struct { CommitMsg string `mapstructure:"commit_msg"` } -func generateDeploymentCommitId(connectionId uint64, repoUrl string, commitSha string) string { +func GenerateDeploymentCommitId(connectionId uint64, pipelineId string, repoUrl string, commitSha string) string { urlHash16 := fmt.Sprintf("%x", md5.Sum([]byte(repoUrl)))[:16] - return fmt.Sprintf("%s:%d:%s:%s", "webhook", connectionId, urlHash16, commitSha) + return fmt.Sprintf("%s:%d:%s:%s:%s", "webhook", connectionId, pipelineId, urlHash16, commitSha) } func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, request *WebhookDeployTaskRequest, tx dal.Transaction, logger log.Logger) errors.Error { @@ -122,7 +122,7 @@ func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, // create a deployment_commit record deploymentCommit := &devops.CicdDeploymentCommit{ DomainEntity: domainlayer.DomainEntity{ - Id: generateDeploymentCommitId(connection.ID, request.RepoUrl, request.CommitSha), + Id: GenerateDeploymentCommitId(connection.ID, request.PipelineId, request.RepoUrl, request.CommitSha), }, CicdDeploymentId: request.PipelineId, CicdScopeId: scopeId, @@ -157,7 +157,7 @@ func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, // create a deployment_commit record deploymentCommit := &devops.CicdDeploymentCommit{ DomainEntity: domainlayer.DomainEntity{ - Id: generateDeploymentCommitId(connection.ID, commit.RepoUrl, commit.CommitSha), + Id: GenerateDeploymentCommitId(connection.ID, request.PipelineId, commit.RepoUrl, commit.CommitSha), }, CicdDeploymentId: request.PipelineId, CicdScopeId: scopeId, From e7df6c10bcec487d751e4dc18ba522586447aa06 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Thu, 23 May 2024 10:07:29 +0800 Subject: [PATCH 03/13] fix(plugins): update path parameter, adapt to latest plugin helper --- backend/plugins/bamboo/impl/impl.go | 2 +- backend/plugins/bitbucket/impl/impl.go | 2 +- backend/plugins/circleci/impl/impl.go | 2 +- backend/plugins/tapd/impl/impl.go | 2 +- backend/plugins/trello/impl/impl.go | 2 +- backend/plugins/zentao/impl/impl.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go index bf5b10747a3..96999a74244 100644 --- a/backend/plugins/bamboo/impl/impl.go +++ b/backend/plugins/bamboo/impl/impl.go @@ -222,7 +222,7 @@ func (p Bamboo) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "POST": api.CreateScopeConfig, "GET": api.GetScopeConfigList, }, - "connections/:connectionId/scope-configs/:id": { + "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go index 933a11a4c6d..3fd1ec6f836 100644 --- a/backend/plugins/bitbucket/impl/impl.go +++ b/backend/plugins/bitbucket/impl/impl.go @@ -233,7 +233,7 @@ func (p Bitbucket) ApiResources() map[string]map[string]plugin.ApiResourceHandle "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, }, - "connections/:connectionId/scope-configs/:id": { + "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, diff --git a/backend/plugins/circleci/impl/impl.go b/backend/plugins/circleci/impl/impl.go index 89e559ec589..4610d887da5 100644 --- a/backend/plugins/circleci/impl/impl.go +++ b/backend/plugins/circleci/impl/impl.go @@ -202,7 +202,7 @@ func (p Circleci) ApiResources() map[string]map[string]plugin.ApiResourceHandler "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, }, - "connections/:connectionId/scope-configs/:id": { + "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go index ad81640193c..9f3e932ea8e 100644 --- a/backend/plugins/tapd/impl/impl.go +++ b/backend/plugins/tapd/impl/impl.go @@ -298,7 +298,7 @@ func (p Tapd) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, }, - "connections/:connectionId/scope-configs/:id": { + "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go index 35a646b5926..f6b3ee45783 100644 --- a/backend/plugins/trello/impl/impl.go +++ b/backend/plugins/trello/impl/impl.go @@ -175,7 +175,7 @@ func (p Trello) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, }, - "connections/:connectionId/scope-configs/:id": { + "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go index 933c296ab4b..b3fe6bafb74 100644 --- a/backend/plugins/zentao/impl/impl.go +++ b/backend/plugins/zentao/impl/impl.go @@ -268,7 +268,7 @@ func (p Zentao) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, }, - "connections/:connectionId/scope-configs/:id": { + "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, From 8d7afb83d75780725db8e1829af656435bea9384 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Thu, 23 May 2024 19:05:06 +0800 Subject: [PATCH 04/13] feat(plugins): add new plugin linker --- backend/plugins/linker/README.md | 16 ++ backend/plugins/linker/impl/impl.go | 117 +++++++++++ .../models/migrationscripts/register.go | 27 +++ .../plugins/linker/tasks/link_pr_and_issue.go | 182 ++++++++++++++++++ backend/plugins/linker/tasks/task_data.go | 48 +++++ 5 files changed, 390 insertions(+) create mode 100644 backend/plugins/linker/README.md create mode 100644 backend/plugins/linker/impl/impl.go create mode 100644 backend/plugins/linker/models/migrationscripts/register.go create mode 100644 backend/plugins/linker/tasks/link_pr_and_issue.go create mode 100644 backend/plugins/linker/tasks/task_data.go diff --git a/backend/plugins/linker/README.md b/backend/plugins/linker/README.md new file mode 100644 index 00000000000..e9090fc2e55 --- /dev/null +++ b/backend/plugins/linker/README.md @@ -0,0 +1,16 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/backend/plugins/linker/impl/impl.go b/backend/plugins/linker/impl/impl.go new file mode 100644 index 00000000000..fdd315d2904 --- /dev/null +++ b/backend/plugins/linker/impl/impl.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/linker/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/linker/tasks" +) + +// make sure interface is implemented +var _ interface { + plugin.PluginMeta + plugin.PluginTask + plugin.PluginModel + plugin.PluginMetric + plugin.PluginMigration + plugin.MetricPluginBlueprintV200 +} = (*Linker)(nil) + +type Linker struct{} + +func (p Linker) Description() string { + return "link some cross table datas together" +} + +// RequiredDataEntities hasn't been used so far +func (p Linker) RequiredDataEntities() (data []map[string]interface{}, err errors.Error) { + return []map[string]interface{}{}, nil +} + +func (p Linker) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{} +} + +func (p Linker) Name() string { + return "linker" +} + +func (p Linker) IsProjectMetric() bool { + return true +} + +func (p Linker) RunAfter() ([]string, errors.Error) { + return []string{}, nil +} + +func (p Linker) Settings() interface{} { + return nil +} + +func (p Linker) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.LinkPrToIssueMeta, + } +} + +func (p Linker) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + return &tasks.LinkerTaskData{ + Options: op, + }, nil +} + +// RootPkgPath information lost when compiled as plugin(.so) +func (p Linker) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/linker" +} + +func (p Linker) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Linker) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) { + op := &tasks.LinkerOptions{} + err := json.Unmarshal(options, op) + if err != nil { + return nil, errors.Default.WrapRaw(err) + } + plan := coreModels.PipelinePlan{ + { + { + Plugin: "linker", + Options: map[string]interface{}{ + "projectName": projectName, + }, + Subtasks: []string{ + "LinkPrToIssue", + }, + }, + }, + } + return plan, nil +} diff --git a/backend/plugins/linker/models/migrationscripts/register.go b/backend/plugins/linker/models/migrationscripts/register.go new file mode 100644 index 00000000000..0aaa5373ba2 --- /dev/null +++ b/backend/plugins/linker/models/migrationscripts/register.go @@ -0,0 +1,27 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{} +} diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go new file mode 100644 index 00000000000..50466782e4d --- /dev/null +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -0,0 +1,182 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +var LinkPrToIssueMeta = plugin.SubTaskMeta{ + Name: "LinkPrToIssue", + EntryPoint: LinkPrToIssue, + EnabledByDefault: true, + Description: "Try to link pull requests to issues, according to pull requests' title and description", + DependencyTables: []string{code.PullRequest{}.TableName(), ticket.Issue{}.TableName()}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CROSS}, + ProductTables: []string{crossdomain.PullRequestIssue{}.TableName()}, +} + +func normalizeIssueKey(issueNumberStr string) string { + issueNumberStr = strings.ReplaceAll(issueNumberStr, "#", "") + issueNumberStr = strings.TrimSpace(issueNumberStr) + return issueNumberStr +} + +func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { + + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinkerTaskData) + + rawDataSubTaskArgs := &api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: data, + //Options:Link + Table: code.PullRequest{}.TableName(), + } + + //issuePattern := data.Options.ScopeConfig.IssueRegex + issuePattern := "" + issueRegex, err := errors.Convert01(regexp.Compile(issuePattern)) + if err != nil { + return errors.Default.Wrap(err, "regexp compile failed") + } + + cursor, err := db.Cursor(dal.From(&code.PullRequest{})) + if err != nil { + return err + } + + defer cursor.Close() + + // iterate all rows + enricher, err := api.NewDataEnricher(api.DataEnricherArgs[code.PullRequest]{ + Ctx: taskCtx, + Name: code.PullRequest{}.TableName(), + Input: cursor, + Enrich: func(pullRequest *code.PullRequest) ([]interface{}, errors.Error) { + + issueNumberStr := "" + for _, text := range []string{pullRequest.Title, pullRequest.Description} { + issueNumberStr = issueRegex.FindString(text) + if issueNumberStr != "" { + break + } + } + issueNumberStr = normalizeIssueKey(issueNumberStr) + if issueNumberStr == "" { + return nil, nil + } + + issue := &ticket.Issue{} + + db.F + + err = db.All( + issue, + dal.Where("issue_key = ?", + issueNumber), + dal.Limit(1), + ) + if err != nil { + return nil, err + } + + pullRequestIssue := &crossdomain.PullRequestIssue{ + PullRequestId: pullRequest.Id, + IssueId: issue.Id, + PullRequestKey: pullRequest.PullRequestKey, + IssueKey: issueNumber, + } + + return []interface{}{pullRequestIssue}, nil + }, + }) + if err != nil { + return err + } + + return enricher.Execute() + converter, err := api.NewDataConverter(api.DataConverterArgs{ + InputRowType: reflect.TypeOf(code.PullRequest{}), + Input: cursor, + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + pullRequest := inputRow.(*code.PullRequest) + + //find the issue in the body + issueNumberStr := "" + + if issueRegex != nil { + issueNumberStr = issueRegex.FindString(pullRequest.Description) + } + //find the issue in the title + if issueNumberStr == "" { + issueNumberStr = issueRegex.FindString(pullRequest.Title) + } + + if issueNumberStr == "" { + return nil, nil + } + + issueNumberStr = strings.ReplaceAll(issueNumberStr, "#", "") + issueNumberStr = strings.TrimSpace(issueNumberStr) + + issue := &ticket.Issue{} + + //change the issueNumberStr to int, if cannot be changed, just continue + issueNumber, numFormatErr := strconv.Atoi(issueNumberStr) + if numFormatErr != nil { + return nil, nil + } + err = db.All( + issue, + dal.Where("issue_key = ?", + issueNumber), + dal.Limit(1), + ) + if err != nil { + return nil, err + } + + pullRequestIssue := &crossdomain.PullRequestIssue{ + PullRequestId: pullRequest.Id, + IssueId: issue.Id, + PullRequestKey: pullRequest.PullRequestKey, + IssueKey: issueNumber, + } + + return []interface{}{pullRequestIssue}, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/linker/tasks/task_data.go b/backend/plugins/linker/tasks/task_data.go new file mode 100644 index 00000000000..4a875bed038 --- /dev/null +++ b/backend/plugins/linker/tasks/task_data.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type LinkerApiParams struct { + ProjectName string +} + +type LinkerOptions struct { + Tasks []string `json:"tasks,omitempty"` + Since string + ProjectName string `json:"projectName"` + ScopeId *string `json:"scopeId,omitempty"` +} + +type LinkerTaskData struct { + Options *LinkerOptions +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*LinkerOptions, errors.Error) { + var op LinkerOptions + err := helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "error decoding Linker task options") + } + + return &op, nil +} From c75b503175178821f4f37df60b93ffd8b3a26d20 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Thu, 23 May 2024 19:29:59 +0800 Subject: [PATCH 05/13] feat(linker): fix compile errors --- backend/plugins/linker/impl/impl.go | 13 +- backend/plugins/linker/linker.go | 42 +++++++ .../plugins/linker/tasks/link_pr_and_issue.go | 112 +++--------------- backend/plugins/linker/tasks/task_data.go | 16 ++- 4 files changed, 76 insertions(+), 107 deletions(-) create mode 100644 backend/plugins/linker/linker.go diff --git a/backend/plugins/linker/impl/impl.go b/backend/plugins/linker/impl/impl.go index fdd315d2904..338239f0da9 100644 --- a/backend/plugins/linker/impl/impl.go +++ b/backend/plugins/linker/impl/impl.go @@ -19,6 +19,7 @@ package impl import ( "encoding/json" + "regexp" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -80,9 +81,17 @@ func (p Linker) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err != nil { return nil, err } - return &tasks.LinkerTaskData{ + taskData := &tasks.LinkerTaskData{ Options: op, - }, nil + } + if op.PrToIssueRegexp != "" { + re, err := regexp.Compile(op.PrToIssueRegexp) + if err != nil { + return taskData, errors.Convert(err) + } + taskData.PrToIssueRegexp = re + } + return taskData, nil } // RootPkgPath information lost when compiled as plugin(.so) diff --git a/backend/plugins/linker/linker.go b/backend/plugins/linker/linker.go new file mode 100644 index 00000000000..30b72fa1d8f --- /dev/null +++ b/backend/plugins/linker/linker.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/linker/impl" + "github.com/spf13/cobra" +) + +// PluginEntry exports for Framework to search and load +var PluginEntry impl.Linker //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "dora"} + + projectName := cmd.Flags().StringP("projectName", "p", "", "project name") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "projectName": *projectName, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go index 50466782e4d..4456f194701 100644 --- a/backend/plugins/linker/tasks/link_pr_and_issue.go +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -21,9 +21,6 @@ import ( "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" - "reflect" - "regexp" - "strconv" "strings" "github.com/apache/incubator-devlake/core/dal" @@ -42,32 +39,21 @@ var LinkPrToIssueMeta = plugin.SubTaskMeta{ ProductTables: []string{crossdomain.PullRequestIssue{}.TableName()}, } -func normalizeIssueKey(issueNumberStr string) string { - issueNumberStr = strings.ReplaceAll(issueNumberStr, "#", "") - issueNumberStr = strings.TrimSpace(issueNumberStr) - return issueNumberStr +func normalizeIssueKey(issueKey string) string { + issueKey = strings.ReplaceAll(issueKey, "#", "") + issueKey = strings.TrimSpace(issueKey) + return issueKey } func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() data := taskCtx.GetData().(*LinkerTaskData) - - rawDataSubTaskArgs := &api.RawDataSubTaskArgs{ - Ctx: taskCtx, - Params: data, - //Options:Link - Table: code.PullRequest{}.TableName(), - } - - //issuePattern := data.Options.ScopeConfig.IssueRegex - issuePattern := "" - issueRegex, err := errors.Convert01(regexp.Compile(issuePattern)) - if err != nil { - return errors.Default.Wrap(err, "regexp compile failed") + var clauses = []dal.Clause{ + dal.From(&code.PullRequest{}), + dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = pull_requests.base_repo_id)"), + dal.Where("pm.project_name = ?", data.Options.ProjectName), } - - cursor, err := db.Cursor(dal.From(&code.PullRequest{})) + cursor, err := db.Cursor(clauses...) if err != nil { return err } @@ -81,29 +67,20 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { Input: cursor, Enrich: func(pullRequest *code.PullRequest) ([]interface{}, errors.Error) { - issueNumberStr := "" + issueKey := "" for _, text := range []string{pullRequest.Title, pullRequest.Description} { - issueNumberStr = issueRegex.FindString(text) - if issueNumberStr != "" { + issueKey = data.PrToIssueRegexp.FindString(text) + if issueKey != "" { break } } - issueNumberStr = normalizeIssueKey(issueNumberStr) - if issueNumberStr == "" { + issueKey = normalizeIssueKey(issueKey) + if issueKey == "" { return nil, nil } issue := &ticket.Issue{} - - db.F - - err = db.All( - issue, - dal.Where("issue_key = ?", - issueNumber), - dal.Limit(1), - ) - if err != nil { + if err := db.First(issue, dal.Where("issue_key = ?", issueKey)); err != nil { return nil, err } @@ -111,7 +88,7 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { PullRequestId: pullRequest.Id, IssueId: issue.Id, PullRequestKey: pullRequest.PullRequestKey, - IssueKey: issueNumber, + IssueKey: issueKey, } return []interface{}{pullRequestIssue}, nil @@ -122,61 +99,4 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { } return enricher.Execute() - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(code.PullRequest{}), - Input: cursor, - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - pullRequest := inputRow.(*code.PullRequest) - - //find the issue in the body - issueNumberStr := "" - - if issueRegex != nil { - issueNumberStr = issueRegex.FindString(pullRequest.Description) - } - //find the issue in the title - if issueNumberStr == "" { - issueNumberStr = issueRegex.FindString(pullRequest.Title) - } - - if issueNumberStr == "" { - return nil, nil - } - - issueNumberStr = strings.ReplaceAll(issueNumberStr, "#", "") - issueNumberStr = strings.TrimSpace(issueNumberStr) - - issue := &ticket.Issue{} - - //change the issueNumberStr to int, if cannot be changed, just continue - issueNumber, numFormatErr := strconv.Atoi(issueNumberStr) - if numFormatErr != nil { - return nil, nil - } - err = db.All( - issue, - dal.Where("issue_key = ?", - issueNumber), - dal.Limit(1), - ) - if err != nil { - return nil, err - } - - pullRequestIssue := &crossdomain.PullRequestIssue{ - PullRequestId: pullRequest.Id, - IssueId: issue.Id, - PullRequestKey: pullRequest.PullRequestKey, - IssueKey: issueNumber, - } - - return []interface{}{pullRequestIssue}, nil - }, - }) - if err != nil { - return err - } - - return converter.Execute() } diff --git a/backend/plugins/linker/tasks/task_data.go b/backend/plugins/linker/tasks/task_data.go index 4a875bed038..f22fae3af08 100644 --- a/backend/plugins/linker/tasks/task_data.go +++ b/backend/plugins/linker/tasks/task_data.go @@ -20,21 +20,19 @@ package tasks import ( "github.com/apache/incubator-devlake/core/errors" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "regexp" ) -type LinkerApiParams struct { - ProjectName string -} - type LinkerOptions struct { - Tasks []string `json:"tasks,omitempty"` - Since string - ProjectName string `json:"projectName"` - ScopeId *string `json:"scopeId,omitempty"` + PrToIssueRegexp string `json:"prToIssueRegexp"` + + ProjectName string `json:"projectName"` // how to get it ? + Since string `json:"since"` // how to get and use it ? } type LinkerTaskData struct { - Options *LinkerOptions + Options *LinkerOptions + PrToIssueRegexp *regexp.Regexp } func DecodeAndValidateTaskOptions(options map[string]interface{}) (*LinkerOptions, errors.Error) { From e9da2ba92c8d13456cd4e2200a8e3e3a4fdfa3ed Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Thu, 23 May 2024 19:38:38 +0800 Subject: [PATCH 06/13] feat(linker): remove `since` in options --- backend/plugins/linker/impl/impl.go | 3 ++- backend/plugins/linker/linker.go | 2 +- backend/plugins/linker/tasks/task_data.go | 7 ++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/plugins/linker/impl/impl.go b/backend/plugins/linker/impl/impl.go index 338239f0da9..917b22df9aa 100644 --- a/backend/plugins/linker/impl/impl.go +++ b/backend/plugins/linker/impl/impl.go @@ -114,7 +114,8 @@ func (p Linker) MakeMetricPluginPipelinePlanV200(projectName string, options jso { Plugin: "linker", Options: map[string]interface{}{ - "projectName": projectName, + "projectName": projectName, + "prToIssueRegexp": op.PrToIssueRegexp, }, Subtasks: []string{ "LinkPrToIssue", diff --git a/backend/plugins/linker/linker.go b/backend/plugins/linker/linker.go index 30b72fa1d8f..0f1bc456551 100644 --- a/backend/plugins/linker/linker.go +++ b/backend/plugins/linker/linker.go @@ -28,7 +28,7 @@ var PluginEntry impl.Linker //nolint // standalone mode for debugging func main() { - cmd := &cobra.Command{Use: "dora"} + cmd := &cobra.Command{Use: "linker"} projectName := cmd.Flags().StringP("projectName", "p", "", "project name") timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") diff --git a/backend/plugins/linker/tasks/task_data.go b/backend/plugins/linker/tasks/task_data.go index f22fae3af08..37a37c358d7 100644 --- a/backend/plugins/linker/tasks/task_data.go +++ b/backend/plugins/linker/tasks/task_data.go @@ -25,9 +25,7 @@ import ( type LinkerOptions struct { PrToIssueRegexp string `json:"prToIssueRegexp"` - - ProjectName string `json:"projectName"` // how to get it ? - Since string `json:"since"` // how to get and use it ? + ProjectName string `json:"projectName"` } type LinkerTaskData struct { @@ -39,8 +37,7 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*LinkerOption var op LinkerOptions err := helper.Decode(options, &op, nil) if err != nil { - return nil, errors.Default.Wrap(err, "error decoding Linker task options") + return nil, errors.Default.Wrap(err, "error decoding linker task options") } - return &op, nil } From 708addb0b1646c5c12aa64172a4b874285df4a98 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Fri, 24 May 2024 16:39:59 +0800 Subject: [PATCH 07/13] feat(linker): add e2e test, fix dora errors --- backend/core/plugin/hub.go | 8 +- backend/core/runner/loader.go | 20 +++-- backend/plugins/dora/impl/impl.go | 9 ++- .../linker/e2e/link_pr_and_issue_test.go | 74 +++++++++++++++++++ .../linker/e2e/snapshot_tables/issues.csv | 2 + .../e2e/snapshot_tables/project_mapping.csv | 2 + .../snapshot_tables/pull_request_issues.csv | 2 + .../e2e/snapshot_tables/pull_requests.csv | 2 + 8 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 backend/plugins/linker/e2e/link_pr_and_issue_test.go create mode 100644 backend/plugins/linker/e2e/snapshot_tables/issues.csv create mode 100644 backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv create mode 100644 backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv create mode 100644 backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go index 6788c905753..3feca2219ba 100644 --- a/backend/core/plugin/hub.go +++ b/backend/core/plugin/hub.go @@ -20,6 +20,7 @@ package plugin import ( "fmt" "strings" + "sync" "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" @@ -27,9 +28,14 @@ import ( // Allowing plugin to know each other -var plugins map[string]PluginMeta +var ( + plugins map[string]PluginMeta + pluginMutex sync.RWMutex +) func RegisterPlugin(name string, plugin PluginMeta) errors.Error { + pluginMutex.Lock() + defer pluginMutex.Unlock() if plugins == nil { plugins = make(map[string]PluginMeta) } diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go index 6e435a172c4..c454aeceb8a 100644 --- a/backend/core/runner/loader.go +++ b/backend/core/runner/loader.go @@ -23,6 +23,7 @@ import ( "path/filepath" goplugin "plugin" "strings" + "sync" "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" @@ -49,6 +50,7 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error { func LoadGoPlugins(basicRes context.BasicRes) errors.Error { pluginsDir := basicRes.GetConfig("PLUGIN_DIR") + var wg sync.WaitGroup walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -68,15 +70,21 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("%s PluginEntry must implement PluginMeta interface", pluginName)) } - err = plugin.RegisterPlugin(pluginName, pluginMeta) - if err != nil { - return err - } - - basicRes.GetLogger().Info(`plugin loaded %s`, pluginName) + wg.Add(1) + go func(pluginName string, pluginMeta plugin.PluginMeta) { + defer func() { + wg.Done() + }() + err = plugin.RegisterPlugin(pluginName, pluginMeta) + if err != nil { + panic(err) + } + basicRes.GetLogger().Info(`plugin loaded %s`, pluginName) + }(pluginName, pluginMeta) } return nil }) + wg.Wait() return errors.Convert(walkErr) } diff --git a/backend/plugins/dora/impl/impl.go b/backend/plugins/dora/impl/impl.go index b20ce189dda..f96a04b5ba3 100644 --- a/backend/plugins/dora/impl/impl.go +++ b/backend/plugins/dora/impl/impl.go @@ -119,10 +119,13 @@ func (p Dora) MigrationScripts() []plugin.MigrationScript { func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) { op := &tasks.DoraOptions{} - err := json.Unmarshal(options, op) - if err != nil { - return nil, errors.Default.WrapRaw(err) + if options != nil && string(options) != "\"\"" { + err := json.Unmarshal(options, op) + if err != nil { + return nil, errors.Default.WrapRaw(err) + } } + plan := coreModels.PipelinePlan{ { { diff --git a/backend/plugins/linker/e2e/link_pr_and_issue_test.go b/backend/plugins/linker/e2e/link_pr_and_issue_test.go new file mode 100644 index 00000000000..ad2ae7eec96 --- /dev/null +++ b/backend/plugins/linker/e2e/link_pr_and_issue_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "regexp" + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linker/impl" + "github.com/apache/incubator-devlake/plugins/linker/tasks" +) + +func TestLinkPrToIssue(t *testing.T) { + var plugin impl.Linker + dataflowTester := e2ehelper.NewDataFlowTester(t, "issue_linker", plugin) + + regexpStr := "#(\\d+)" + re, err := regexp.Compile(regexpStr) + if err != nil { + panic(err) + } + taskData := &tasks.LinkerTaskData{ + Options: &tasks.LinkerOptions{ + // https://docs.gitlab.com/ee/user/project/issues/crosslinking_issues.html + // For gitlab issues #xxx, GL-xxxx, projectname#xxx or https://gitlab.com///-/issues/ + // https://regex101.com/r/RteyFk/1 + PrToIssueRegexp: regexpStr, + ProjectName: "GitHub1", + }, + PrToIssueRegexp: re, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/issues.csv", &ticket.Issue{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/pull_requests.csv", &code.PullRequest{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/project_mapping.csv", &crossdomain.ProjectMapping{}) + + // verify extraction + dataflowTester.FlushTabler(&crossdomain.PullRequestIssue{}) + dataflowTester.Subtask(tasks.LinkPrToIssueMeta, taskData) + dataflowTester.VerifyTable( + crossdomain.PullRequestIssue{}, + "./snapshot_tables/pull_request_issues.csv", + []string{ + "pull_request_id", + "pull_request_key", + "issue_id", + "issue_key", + "_raw_data_params", + "_raw_data_table", + "_raw_data_id", + "_raw_data_remark", + }, + ) + +} diff --git a/backend/plugins/linker/e2e/snapshot_tables/issues.csv b/backend/plugins/linker/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..a7d1fd0f989 --- /dev/null +++ b/backend/plugins/linker/e2e/snapshot_tables/issues.csv @@ -0,0 +1,2 @@ +"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","original_type","status","original_status","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","story_point","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","original_project","urgency" +"github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" diff --git a/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv b/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv new file mode 100644 index 00000000000..70e57627e0c --- /dev/null +++ b/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv @@ -0,0 +1,2 @@ +"project_name","table","row_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" +"GitHub1","cicd_scopes","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,"" \ No newline at end of file diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv new file mode 100644 index 00000000000..4f952b6fd1f --- /dev/null +++ b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv @@ -0,0 +1,2 @@ +pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324696,7317,1884,,,0,"pull_requests," diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv new file mode 100644 index 00000000000..bf260bf0cb7 --- /dev/null +++ b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv @@ -0,0 +1,2 @@ +"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","base_repo_id","base_ref","base_commit_sha","head_repo_id","head_ref","head_commit_sha","merge_commit_sha","status","original_status","type","component","title","description","url","author_name","author_id","parent_pr_id","pull_request_key","created_date","merged_date","closed_date" +"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000" \ No newline at end of file From 0499a5cd0e3870d073116b76649fe475b80c495a Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Fri, 24 May 2024 16:48:53 +0800 Subject: [PATCH 08/13] fix(test): fix test errors --- backend/plugins/table_info_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 7360b978bb2..937ba14a848 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -39,6 +39,7 @@ import ( icla "github.com/apache/incubator-devlake/plugins/icla/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" @@ -88,6 +89,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("zentao/models", zentao.Zentao{}.GetTablesInfo) checker.FeedIn("circleci/models", circleci.Circleci{}.GetTablesInfo) checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) + checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) err := checker.Verify() if err != nil { t.Error(err) From 563b0720f08e52cd45bafb3a3d3450b491b0c081 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Fri, 24 May 2024 16:55:54 +0800 Subject: [PATCH 09/13] docs(linker): remove some comments --- backend/plugins/linker/e2e/link_pr_and_issue_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/plugins/linker/e2e/link_pr_and_issue_test.go b/backend/plugins/linker/e2e/link_pr_and_issue_test.go index ad2ae7eec96..a199764d401 100644 --- a/backend/plugins/linker/e2e/link_pr_and_issue_test.go +++ b/backend/plugins/linker/e2e/link_pr_and_issue_test.go @@ -40,9 +40,6 @@ func TestLinkPrToIssue(t *testing.T) { } taskData := &tasks.LinkerTaskData{ Options: &tasks.LinkerOptions{ - // https://docs.gitlab.com/ee/user/project/issues/crosslinking_issues.html - // For gitlab issues #xxx, GL-xxxx, projectname#xxx or https://gitlab.com///-/issues/ - // https://regex101.com/r/RteyFk/1 PrToIssueRegexp: regexpStr, ProjectName: "GitHub1", }, @@ -53,7 +50,6 @@ func TestLinkPrToIssue(t *testing.T) { dataflowTester.ImportCsvIntoTabler("./snapshot_tables/pull_requests.csv", &code.PullRequest{}) dataflowTester.ImportCsvIntoTabler("./snapshot_tables/project_mapping.csv", &crossdomain.ProjectMapping{}) - // verify extraction dataflowTester.FlushTabler(&crossdomain.PullRequestIssue{}) dataflowTester.Subtask(tasks.LinkPrToIssueMeta, taskData) dataflowTester.VerifyTable( From e9944750ee012d686759150e609d5c5aee9ead78 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Mon, 27 May 2024 14:31:49 +0800 Subject: [PATCH 10/13] fix(linker): fix plugin name --- backend/plugins/linker/e2e/link_pr_and_issue_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/plugins/linker/e2e/link_pr_and_issue_test.go b/backend/plugins/linker/e2e/link_pr_and_issue_test.go index a199764d401..321c61723c6 100644 --- a/backend/plugins/linker/e2e/link_pr_and_issue_test.go +++ b/backend/plugins/linker/e2e/link_pr_and_issue_test.go @@ -31,7 +31,7 @@ import ( func TestLinkPrToIssue(t *testing.T) { var plugin impl.Linker - dataflowTester := e2ehelper.NewDataFlowTester(t, "issue_linker", plugin) + dataflowTester := e2ehelper.NewDataFlowTester(t, "linker", plugin) regexpStr := "#(\\d+)" re, err := regexp.Compile(regexpStr) From ecad69bfa19c0750b0b33aab26d38b2cc8967eb5 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Mon, 27 May 2024 14:40:57 +0800 Subject: [PATCH 11/13] fix(linker): fix unpected error --- backend/plugins/linker/tasks/link_pr_and_issue.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go index 4456f194701..39ad5aa8e68 100644 --- a/backend/plugins/linker/tasks/link_pr_and_issue.go +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -81,6 +81,9 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { issue := &ticket.Issue{} if err := db.First(issue, dal.Where("issue_key = ?", issueKey)); err != nil { + if db.IsErrorNotFound(err) { + return nil, nil + } return nil, err } From 5b0266bb18fb2a2bfc403e5709784080d7ea46d0 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Mon, 27 May 2024 20:03:53 +0800 Subject: [PATCH 12/13] feat(linker): find all candidate issue keys from pr's title and description --- .../linker/e2e/snapshot_tables/issues.csv | 1 + .../plugins/linker/tasks/link_pr_and_issue.go | 42 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/backend/plugins/linker/e2e/snapshot_tables/issues.csv b/backend/plugins/linker/e2e/snapshot_tables/issues.csv index a7d1fd0f989..81fb251cea5 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/issues.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/issues.csv @@ -1,2 +1,3 @@ "id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","original_type","status","original_status","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","story_point","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","original_project","urgency" "github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" +"github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1885","","1885","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go index 39ad5aa8e68..c8fec33306c 100644 --- a/backend/plugins/linker/tasks/link_pr_and_issue.go +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -60,41 +60,43 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { defer cursor.Close() - // iterate all rows enricher, err := api.NewDataEnricher(api.DataEnricherArgs[code.PullRequest]{ Ctx: taskCtx, Name: code.PullRequest{}.TableName(), Input: cursor, Enrich: func(pullRequest *code.PullRequest) ([]interface{}, errors.Error) { - issueKey := "" + var issueKeys []string for _, text := range []string{pullRequest.Title, pullRequest.Description} { - issueKey = data.PrToIssueRegexp.FindString(text) - if issueKey != "" { + foundIssueKeys := data.PrToIssueRegexp.FindAllString(text, -1) + if foundIssueKeys != nil && len(foundIssueKeys) > 0 { + for _, issueKey := range foundIssueKeys { + issueKey = normalizeIssueKey(issueKey) + issueKeys = append(issueKeys, issueKey) + } break } } - issueKey = normalizeIssueKey(issueKey) - if issueKey == "" { - return nil, nil - } - issue := &ticket.Issue{} - if err := db.First(issue, dal.Where("issue_key = ?", issueKey)); err != nil { - if db.IsErrorNotFound(err) { - return nil, nil - } + var issues []*ticket.Issue + if err := db.All(&issues, dal.Where("issue_key in ?", issueKeys)); err != nil { return nil, err } - - pullRequestIssue := &crossdomain.PullRequestIssue{ - PullRequestId: pullRequest.Id, - IssueId: issue.Id, - PullRequestKey: pullRequest.PullRequestKey, - IssueKey: issueKey, + if len(issues) == 0 { + return nil, nil + } + var result []interface{} + for _, issue := range issues { + pullRequestIssue := &crossdomain.PullRequestIssue{ + PullRequestId: pullRequest.Id, + IssueId: issue.Id, + PullRequestKey: pullRequest.PullRequestKey, + IssueKey: issue.IssueKey, + } + result = append(result, pullRequestIssue) } - return []interface{}{pullRequestIssue}, nil + return result, nil }, }) if err != nil { From c471b707509a103f98b9dcd53be556adc5721d4b Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Mon, 27 May 2024 20:08:37 +0800 Subject: [PATCH 13/13] fix(test): fix lint errors --- backend/plugins/linker/e2e/snapshot_tables/issues.csv | 3 ++- .../plugins/linker/e2e/snapshot_tables/pull_request_issues.csv | 1 + backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv | 2 +- backend/plugins/linker/tasks/link_pr_and_issue.go | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/plugins/linker/e2e/snapshot_tables/issues.csv b/backend/plugins/linker/e2e/snapshot_tables/issues.csv index 81fb251cea5..9ef9a334315 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/issues.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/issues.csv @@ -1,3 +1,4 @@ "id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","original_type","status","original_status","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","story_point","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","original_project","urgency" "github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" -"github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1885","","1885","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" +"github:GithubIssue:1:1237324697","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1885","","1885","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" +"github:GithubIssue:1:1237324698","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1886","","1886","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","","" diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv index 4f952b6fd1f..6b7d30fb8a3 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv @@ -1,2 +1,3 @@ pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324696,7317,1884,,,0,"pull_requests," +github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324697,7317,1885,,,0,"pull_requests," diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv index bf260bf0cb7..22635f9b0b7 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv @@ -1,2 +1,2 @@ "id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","base_repo_id","base_ref","base_commit_sha","head_repo_id","head_ref","head_commit_sha","merge_commit_sha","status","original_status","type","component","title","description","url","author_name","author_id","parent_pr_id","pull_request_key","created_date","merged_date","closed_date" -"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000" \ No newline at end of file +"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884 #1885","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000" \ No newline at end of file diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go index b0aa9e9206e..a920c4fe719 100644 --- a/backend/plugins/linker/tasks/link_pr_and_issue.go +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -69,7 +69,7 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { var issueKeys []string for _, text := range []string{pullRequest.Title, pullRequest.Description} { foundIssueKeys := data.PrToIssueRegexp.FindAllString(text, -1) - if foundIssueKeys != nil && len(foundIssueKeys) > 0 { + if len(foundIssueKeys) > 0 { for _, issueKey := range foundIssueKeys { issueKey = normalizeIssueKey(issueKey) issueKeys = append(issueKeys, issueKey)