refactor: add subtask register for github plugin (#5411)

* refactor: add subtask registert for github * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask meta register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: Update remote.go * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register * refactor: update subtask register * refactor: update subtask register * refactor: add subtask register * refactor: add subtask register * refactor: add subtask register
apache · Jul 20, 2023 · cc39ef2 · cc39ef2
1 parent f8ad90e
commit cc39ef2
Show file tree

Hide file tree

Showing 63 changed files with 897 additions and 193 deletions.
diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go
@@ -19,6 +19,7 @@ package plugin
 
 import (
 	"context"
+
 	corecontext "github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 )
@@ -98,6 +99,8 @@ type SubTaskMeta struct {
 	Description      string
 	DomainTypes      []string
 	Dependencies     []*SubTaskMeta
+	DependencyTables []string
+	ProductTables    []string
 }
 
 // PluginTask Implement this interface to let framework run tasks for you

diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -41,7 +41,7 @@ type ApiCollectorStateManager struct {
 	ExecuteStart time.Time
 }
 
-// NewApiCollectorWithState create a new ApiCollectorStateManager
+// NewStatefulApiCollector create a new ApiCollectorStateManager
 func NewStatefulApiCollector(args RawDataSubTaskArgs, timeAfter *time.Time) (*ApiCollectorStateManager, errors.Error) {
 	db := args.Ctx.GetDal()
 

diff --git a/backend/helpers/pluginhelper/api/batch_save.go b/backend/helpers/pluginhelper/api/batch_save.go
@@ -29,7 +29,7 @@ import (
 	"github.com/apache/incubator-devlake/core/log"
 )
 
-// BatchSave performs mulitple records persistence of a specific type in one sql query to improve the performance
+// BatchSave performs multiple records persistence of a specific type in one sql query to improve the performance
 type BatchSave struct {
 	basicRes context.BasicRes
 	log      log.Logger

diff --git a/backend/helpers/pluginhelper/api/connection_auths.go b/backend/helpers/pluginhelper/api/connection_auths.go
@@ -24,9 +24,8 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/apache/incubator-devlake/core/plugin"
-
 	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/go-playground/validator/v10"
 )
 

diff --git a/...ginhelper/subtaskmeta_sorter/interface.go → ...ginhelper/subtaskmeta/sorter/interface.go b/...ginhelper/subtaskmeta_sorter/interface.go → ...ginhelper/subtaskmeta/sorter/interface.go
@@ -15,10 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package subtaskmeta_sorter
+package sorter
 
-import "github.com/apache/incubator-devlake/core/plugin"
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
 
 type SubTaskMetaSorter interface {
-	Sort() ([]plugin.SubTaskMeta, error)
+	Sort() ([]plugin.SubTaskMeta, errors.Error)
 }
diff --git a/...inhelper/subtaskmeta_sorter/dependency.go → ...luginhelper/subtaskmeta/sorter/subtask.go b/...inhelper/subtaskmeta_sorter/dependency.go → ...luginhelper/subtaskmeta/sorter/subtask.go
@@ -15,12 +15,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package subtaskmeta_sorter
+package sorter
 
 import (
 	"fmt"
+
+	"github.com/apache/incubator-devlake/core/errors"
+
 	"github.com/apache/incubator-devlake/core/plugin"
-	"sort"
 )
 
 type DependencySorter struct {
@@ -31,12 +33,12 @@ func NewDependencySorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter {
 	return &DependencySorter{metas: metas}
 }
 
-func (d *DependencySorter) Sort() ([]plugin.SubTaskMeta, error) {
-	return topologicalSort(d.metas)
+func (d *DependencySorter) Sort() ([]plugin.SubTaskMeta, errors.Error) {
+	return dependenciesTopologicalSort(d.metas)
 }
 
 // stable topological sort
-func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error) {
+func dependenciesTopologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, errors.Error) {
 	// which state will make a cycle
 	dependenciesMap := make(map[string][]string)
 	nameMetaMap := make(map[string]*plugin.SubTaskMeta)
@@ -56,63 +58,25 @@ func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error)
 				dependenciesMap[item.Name] = make([]string, 0)
 			}
 		} else {
-			return nil, fmt.Errorf("duplicate subtaskmetas detected in list: %s", item.Name)
+			return nil, errors.Convert(fmt.Errorf("duplicate subtaskmetas detected in list: %s", item.Name))
 		}
 	}
 
-	orderedSubtaskList := make([]plugin.SubTaskMeta, 0)
-	for {
-		if len(dependenciesMap) == 0 {
-			break
-		}
-
-		tmpList := make([]string, 0)
-		for key, item := range dependenciesMap {
-			if len(item) == 0 {
-				tmpList = append(tmpList, key)
-			}
-		}
-		if len(tmpList) == 0 {
-			return nil, fmt.Errorf("cyclic dependency detected: %v", dependenciesMap)
-		}
-
-		// remove item in dependencies map
-		for key, value := range dependenciesMap {
-			if contains(tmpList, key) {
-				delete(dependenciesMap, key)
-			} else {
-				dependenciesMap[key] = removeElements(value, tmpList)
-			}
-		}
-
-		sort.Strings(tmpList)
-		// convert item to subtaskmeta by name, and append to orderedSubtaskList
-		for _, item := range tmpList {
-			value, ok := nameMetaMap[item]
-			if !ok {
-				return nil, fmt.Errorf("illeagal subtaskmeta detected %s", item)
-			}
-			orderedSubtaskList = append(orderedSubtaskList, *value)
-		}
+	// sort
+	orderStrList, err := topologicalSortSameElements(dependenciesMap)
+	if err != nil {
+		return nil, errors.Convert(err)
 	}
-	return orderedSubtaskList, nil
-}
 
-func contains[T comparable](itemList []T, item T) bool {
-	for _, newItem := range itemList {
-		if item == newItem {
-			return true
+	// gen list by sorted name list and return
+	orderedSubtaskList := make([]plugin.SubTaskMeta, 0)
+	for _, item := range orderStrList {
+		value, ok := nameMetaMap[item]
+		if !ok {
+			return nil, errors.Convert(fmt.Errorf("illeagal subtaskmeta detected %s", item))
 		}
+		orderedSubtaskList = append(orderedSubtaskList, *value)
 	}
-	return false
-}
 
-func removeElements[T comparable](raw, toRemove []T) []T {
-	newList := make([]T, 0)
-	for _, item := range raw {
-		if !contains(toRemove, item) {
-			newList = append(newList, item)
-		}
-	}
-	return newList
+	return orderedSubtaskList, nil
 }
diff --git a/...per/subtaskmeta_sorter/dependency_test.go → ...helper/subtaskmeta/sorter/subtask_test.go b/...per/subtaskmeta_sorter/dependency_test.go → ...helper/subtaskmeta/sorter/subtask_test.go
@@ -15,12 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package subtaskmeta_sorter
+package sorter
 
 import (
-	"github.com/apache/incubator-devlake/core/plugin"
 	"reflect"
 	"testing"
+
+	"github.com/apache/incubator-devlake/core/plugin"
 )
 
 func Test_topologicalSort(t *testing.T) {
@@ -85,13 +86,13 @@ func Test_topologicalSort(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := topologicalSort(tt.args.metas)
+			got, err := dependenciesTopologicalSort(tt.args.metas)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("topologicalSort() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("dependenciesTopologicalSort() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("topologicalSort() got = %v, want %v", got, tt.want)
+				t.Errorf("dependenciesTopologicalSort() got = %v, want %v", got, tt.want)
 			}
 		})
 	}

diff --git a/backend/helpers/pluginhelper/subtaskmeta/sorter/table.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/table.go
@@ -0,0 +1,80 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sorter
+
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+type TableSorter struct {
+	metas []*plugin.SubTaskMeta
+}
+
+func NewTableSorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter {
+	return &TableSorter{metas: metas}
+}
+
+func (d *TableSorter) Sort() ([]plugin.SubTaskMeta, errors.Error) {
+	return tableTopologicalSort(d.metas)
+}
+
+func tableTopologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, errors.Error) {
+	constructedMetas := constructDependenciesByTable(metas)
+	return dependenciesTopologicalSort(constructedMetas)
+}
+
+func constructDependenciesByTable(metas []*plugin.SubTaskMeta) []*plugin.SubTaskMeta {
+	// construct map by metas and their produced tables, the key is table, and value is metas
+	tableMetasMap := make(map[string][]*plugin.SubTaskMeta)
+	for _, item := range metas {
+		for _, tableItem := range item.ProductTables {
+			if value, ok := tableMetasMap[tableItem]; ok {
+				tableMetasMap[tableItem] = append(value, item)
+			} else {
+				tableMetasMap[tableItem] = []*plugin.SubTaskMeta{item}
+			}
+		}
+	}
+	// construct meta dependencies by meta.TableDependencies
+	// use noDupMap to deduplicate dependencies of meta
+	noDupMap := make(map[*plugin.SubTaskMeta]map[*plugin.SubTaskMeta]any)
+	for _, metaItem := range metas {
+		// convert dependency tables to dependency metas
+		dependenciesMap, ok := noDupMap[metaItem]
+		if !ok {
+			noDupMap[metaItem] = make(map[*plugin.SubTaskMeta]any)
+			dependenciesMap = noDupMap[metaItem]
+		}
+		for _, tableItem := range metaItem.DependencyTables {
+			for _, item := range tableMetasMap[tableItem] {
+				dependenciesMap[item] = ""
+			}
+		}
+		metaItem.Dependencies = keys(dependenciesMap)
+	}
+	return metas
+}
+
+func keys[T comparable](raw map[T]any) []T {
+	list := make([]T, 0)
+	for key := range raw {
+		list = append(list, key)
+	}
+	return list
+}
diff --git a/backend/helpers/pluginhelper/subtaskmeta/sorter/table_test.go b/backend/helpers/pluginhelper/subtaskmeta/sorter/table_test.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sorter
+
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"reflect"
+	"testing"
+)
+
+func Test_tableTopologicalSort(t *testing.T) {
+	pluginA := plugin.SubTaskMeta{
+		Name:             "A",
+		DependencyTables: []string{},
+		ProductTables:    []string{"_TOOL_TEST_TABLE", "_TOOL_TEST_TABLE2"},
+	}
+	pluginB := plugin.SubTaskMeta{
+		Name:             "B",
+		DependencyTables: []string{"_TOOL_TEST_TABLE"},
+		ProductTables:    []string{"_TOOL_TEST_TABLE2", "_TOOL_TEST_TABLE3"},
+	}
+	pluginC := plugin.SubTaskMeta{
+		Name:             "C",
+		DependencyTables: []string{"_TOOL_TEST_TABLE"},
+		ProductTables:    []string{"_TOOL_TEST_TABLE3"},
+	}
+	pluginD := plugin.SubTaskMeta{
+		Name:             "D",
+		DependencyTables: []string{"_TOOL_TEST_TABLE2", "_TOOL_TEST_TABLE3"},
+		ProductTables:    []string{"_TOOL_TEST_TABLE4"},
+	}
+
+	type args struct {
+		metas []*plugin.SubTaskMeta
+	}
+	tests := []struct {
+		name  string
+		args  args
+		want  []plugin.SubTaskMeta
+		want1 errors.Error
+	}{
+		{
+			name:  "test sorter",
+			args:  args{[]*plugin.SubTaskMeta{&pluginA, &pluginB, &pluginC, &pluginD}},
+			want:  []plugin.SubTaskMeta{pluginA, pluginB, pluginC, pluginD},
+			want1: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, got1 := tableTopologicalSort(tt.args.metas)
+			if len(got) != len(tt.want) {
+				t.Errorf("tableTopologicalSort() got = %v, want %v", got, tt.want)
+			}
+			for index, item := range got {
+				if item.Name != tt.want[index].Name {
+					t.Errorf("tableTopologicalSort() got = %v, want %v, not equal with index = %d", got, tt.want, index)
+				}
+			}
+			if !reflect.DeepEqual(got1, tt.want1) {
+				t.Errorf("tableTopologicalSort() got1 = %v, want %v", got1, tt.want1)
+			}
+		})
+	}
+}