Add a new plugin, linker.

backend/core/plugin/hub.go

@@ -20,16 +20,22 @@ package plugin
 import (
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 )
 
 // Allowing plugin to know each other
 
-var plugins map[string]PluginMeta
+var (
+	plugins     map[string]PluginMeta
+	pluginMutex sync.RWMutex
+)
 
 func RegisterPlugin(name string, plugin PluginMeta) errors.Error {
+	pluginMutex.Lock()
+	defer pluginMutex.Unlock()
 	if plugins == nil {
 		plugins = make(map[string]PluginMeta)
 	}

backend/core/runner/loader.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	goplugin "plugin"
 	"strings"
+	"sync"
 
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
@@ -49,6 +50,7 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error {
 
 func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
 	pluginsDir := basicRes.GetConfig("PLUGIN_DIR")
+	var wg sync.WaitGroup
 	walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -68,15 +70,21 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
 			if !ok {
 				return errors.Default.New(fmt.Sprintf("%s PluginEntry must implement PluginMeta interface", pluginName))
 			}
-			err = plugin.RegisterPlugin(pluginName, pluginMeta)
-			if err != nil {
-				return err
-			}
-
-			basicRes.GetLogger().Info(`plugin loaded %s`, pluginName)
+			wg.Add(1)
+			go func(pluginName string, pluginMeta plugin.PluginMeta) {
+				defer func() {
+					wg.Done()
+				}()
+				err = plugin.RegisterPlugin(pluginName, pluginMeta)
+				if err != nil {
+					panic(err)
+				}
+				basicRes.GetLogger().Info(`plugin loaded %s`, pluginName)
+			}(pluginName, pluginMeta)
 		}
 		return nil
 	})
+	wg.Wait()
 	return errors.Convert(walkErr)
 }
 

backend/plugins/dora/impl/impl.go

@@ -119,10 +119,13 @@ func (p Dora) MigrationScripts() []plugin.MigrationScript {
 
 func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) {
 	op := &tasks.DoraOptions{}
-	err := json.Unmarshal(options, op)
-	if err != nil {
-		return nil, errors.Default.WrapRaw(err)
+	if options != nil && string(options) != "\"\"" {
+		err := json.Unmarshal(options, op)
+		if err != nil {
+			return nil, errors.Default.WrapRaw(err)
+		}
 	}
+
 	plan := coreModels.PipelinePlan{
 		{
 			{

backend/plugins/linker/README.md

@@ -0,0 +1,16 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/

backend/plugins/linker/e2e/link_pr_and_issue_test.go

@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	"github.com/apache/incubator-devlake/plugins/linker/impl"
+	"github.com/apache/incubator-devlake/plugins/linker/tasks"
+)
+
+func TestLinkPrToIssue(t *testing.T) {
+	var plugin impl.Linker
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "issue_linker", plugin)
+
+	regexpStr := "#(\\d+)"
+	re, err := regexp.Compile(regexpStr)
+	if err != nil {
+		panic(err)
+	}
+	taskData := &tasks.LinkerTaskData{
+		Options: &tasks.LinkerOptions{
+			PrToIssueRegexp: regexpStr,
+			ProjectName:     "GitHub1",
+		},
+		PrToIssueRegexp: re,
+	}
+
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/issues.csv", &ticket.Issue{})
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/pull_requests.csv", &code.PullRequest{})
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/project_mapping.csv", &crossdomain.ProjectMapping{})
+
+	dataflowTester.FlushTabler(&crossdomain.PullRequestIssue{})
+	dataflowTester.Subtask(tasks.LinkPrToIssueMeta, taskData)
+	dataflowTester.VerifyTable(
+		crossdomain.PullRequestIssue{},
+		"./snapshot_tables/pull_request_issues.csv",
+		[]string{
+			"pull_request_id",
+			"pull_request_key",
+			"issue_id",
+			"issue_key",
+			"_raw_data_params",
+			"_raw_data_table",
+			"_raw_data_id",
+			"_raw_data_remark",
+		},
+	)
+
+}

backend/plugins/linker/e2e/snapshot_tables/issues.csv

@@ -0,0 +1,2 @@
+"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","original_type","status","original_status","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","story_point","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","original_project","urgency"
+"github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add a plugin for Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","",""

backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv

@@ -0,0 +1,2 @@
+"project_name","table","row_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark"
+"GitHub1","cicd_scopes","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,""

backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv

@@ -0,0 +1,2 @@
+pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324696,7317,1884,,,0,"pull_requests,"

backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv

@@ -0,0 +1,2 @@
+"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","base_repo_id","base_ref","base_commit_sha","head_repo_id","head_ref","head_commit_sha","merge_commit_sha","status","original_status","type","component","title","description","url","author_name","author_id","parent_pr_id","pull_request_key","created_date","merged_date","closed_date"
+"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000"

backend/plugins/linker/impl/impl.go

@@ -0,0 +1,127 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+	"encoding/json"
+	"regexp"
+
+	"github.com/apache/incubator-devlake/core/dal"
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/plugins/linker/models/migrationscripts"
+	"github.com/apache/incubator-devlake/plugins/linker/tasks"
+)
+
+// make sure interface is implemented
+var _ interface {
+	plugin.PluginMeta
+	plugin.PluginTask
+	plugin.PluginModel
+	plugin.PluginMetric
+	plugin.PluginMigration
+	plugin.MetricPluginBlueprintV200
+} = (*Linker)(nil)
+
+type Linker struct{}
+
+func (p Linker) Description() string {
+	return "link some cross table datas together"
+}
+
+// RequiredDataEntities hasn't been used so far
+func (p Linker) RequiredDataEntities() (data []map[string]interface{}, err errors.Error) {
+	return []map[string]interface{}{}, nil
+}
+
+func (p Linker) GetTablesInfo() []dal.Tabler {
+	return []dal.Tabler{}
+}
+
+func (p Linker) Name() string {
+	return "linker"
+}
+
+func (p Linker) IsProjectMetric() bool {
+	return true
+}
+
+func (p Linker) RunAfter() ([]string, errors.Error) {
+	return []string{}, nil
+}
+
+func (p Linker) Settings() interface{} {
+	return nil
+}
+
+func (p Linker) SubTaskMetas() []plugin.SubTaskMeta {
+	return []plugin.SubTaskMeta{
+		tasks.LinkPrToIssueMeta,
+	}
+}
+
+func (p Linker) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
+	op, err := tasks.DecodeAndValidateTaskOptions(options)
+	if err != nil {
+		return nil, err
+	}
+	taskData := &tasks.LinkerTaskData{
+		Options: op,
+	}
+	if op.PrToIssueRegexp != "" {
+		re, err := regexp.Compile(op.PrToIssueRegexp)
+		if err != nil {
+			return taskData, errors.Convert(err)
+		}
+		taskData.PrToIssueRegexp = re
+	}
+	return taskData, nil
+}
+
+// RootPkgPath information lost when compiled as plugin(.so)
+func (p Linker) RootPkgPath() string {
+	return "github.com/apache/incubator-devlake/plugins/linker"
+}
+
+func (p Linker) MigrationScripts() []plugin.MigrationScript {
+	return migrationscripts.All()
+}
+
+func (p Linker) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) {
+	op := &tasks.LinkerOptions{}
+	err := json.Unmarshal(options, op)
+	if err != nil {
+		return nil, errors.Default.WrapRaw(err)
+	}
+	plan := coreModels.PipelinePlan{
+		{
+			{
+				Plugin: "linker",
+				Options: map[string]interface{}{
+					"projectName":     projectName,
+					"prToIssueRegexp": op.PrToIssueRegexp,
+				},
+				Subtasks: []string{
+					"LinkPrToIssue",
+				},
+			},
+		},
+	}
+	return plan, nil
+}

backend/plugins/linker/linker.go

@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/apache/incubator-devlake/core/runner"
+	"github.com/apache/incubator-devlake/plugins/linker/impl"
+	"github.com/spf13/cobra"
+)
+
+// PluginEntry exports for Framework to search and load
+var PluginEntry impl.Linker //nolint
+
+// standalone mode for debugging
+func main() {
+	cmd := &cobra.Command{Use: "linker"}
+
+	projectName := cmd.Flags().StringP("projectName", "p", "", "project name")
+	timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z")
+
+	cmd.Run = func(cmd *cobra.Command, args []string) {
+		runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+			"projectName": *projectName,
+		}, *timeAfter)
+	}
+	runner.RunCmd(cmd)
+}

backend/plugins/linker/models/migrationscripts/register.go

@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+// All return all the migration scripts
+func All() []plugin.MigrationScript {
+	return []plugin.MigrationScript{}
+}