diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go index 9baf274e301..e19f75e97c5 100644 --- a/backend/core/plugin/plugin_task.go +++ b/backend/core/plugin/plugin_task.go @@ -19,6 +19,7 @@ package plugin import ( "context" + corecontext "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" ) @@ -98,6 +99,7 @@ type SubTaskMeta struct { Description string DomainTypes []string Dependencies []*SubTaskMeta + DependencyTables []string } // PluginTask Implement this interface to let framework run tasks for you diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/subtask.go similarity index 59% rename from backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go rename to backend/helpers/pluginhelper/subtaskmeta_sorter/subtask.go index d310c1fe56b..62ecf2bf176 100644 --- a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go +++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/subtask.go @@ -19,8 +19,8 @@ package subtaskmeta_sorter import ( "fmt" + "github.com/apache/incubator-devlake/core/plugin" - "sort" ) type DependencySorter struct { @@ -32,11 +32,12 @@ func NewDependencySorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter { } func (d *DependencySorter) Sort() ([]plugin.SubTaskMeta, error) { - return topologicalSort(d.metas) + return dependenciesTopologicalSort(d.metas) } // stable topological sort -func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error) { +func dependenciesTopologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error) { + // 1. construct data // which state will make a cycle dependenciesMap := make(map[string][]string) nameMetaMap := make(map[string]*plugin.SubTaskMeta) @@ -60,59 +61,20 @@ func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error) } } - orderedSubtaskList := make([]plugin.SubTaskMeta, 0) - for { - if len(dependenciesMap) == 0 { - break - } - - tmpList := make([]string, 0) - for key, item := range dependenciesMap { - if len(item) == 0 { - tmpList = append(tmpList, key) - } - } - if len(tmpList) == 0 { - return nil, fmt.Errorf("cyclic dependency detected: %v", dependenciesMap) - } - - // remove item in dependencies map - for key, value := range dependenciesMap { - if contains(tmpList, key) { - delete(dependenciesMap, key) - } else { - dependenciesMap[key] = removeElements(value, tmpList) - } - } - - sort.Strings(tmpList) - // convert item to subtaskmeta by name, and append to orderedSubtaskList - for _, item := range tmpList { - value, ok := nameMetaMap[item] - if !ok { - return nil, fmt.Errorf("illeagal subtaskmeta detected %s", item) - } - orderedSubtaskList = append(orderedSubtaskList, *value) - } - } - return orderedSubtaskList, nil -} - -func contains[T comparable](itemList []T, item T) bool { - for _, newItem := range itemList { - if item == newItem { - return true - } + // sort + orderStrList, err := topologicalSortSameElements(dependenciesMap) + if err != nil { + return nil, err } - return false -} -func removeElements[T comparable](raw, toRemove []T) []T { - newList := make([]T, 0) - for _, item := range raw { - if !contains(toRemove, item) { - newList = append(newList, item) + // gen list by sorted name list and return + orderedSubtaskList := make([]plugin.SubTaskMeta, 0) + for _, item := range orderStrList { + value, ok := nameMetaMap[item] + if !ok { + return nil, fmt.Errorf("illeagal subtaskmeta detected %s", item) } + orderedSubtaskList = append(orderedSubtaskList, *value) } - return newList + return orderedSubtaskList, nil } diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/subtask_test.go similarity index 91% rename from backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go rename to backend/helpers/pluginhelper/subtaskmeta_sorter/subtask_test.go index 924b3be7995..02328425a77 100644 --- a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go +++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/subtask_test.go @@ -18,9 +18,10 @@ limitations under the License. package subtaskmeta_sorter import ( - "github.com/apache/incubator-devlake/core/plugin" "reflect" "testing" + + "github.com/apache/incubator-devlake/core/plugin" ) func Test_topologicalSort(t *testing.T) { @@ -85,13 +86,13 @@ func Test_topologicalSort(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := topologicalSort(tt.args.metas) + got, err := dependenciesTopologicalSort(tt.args.metas) if (err != nil) != tt.wantErr { - t.Errorf("topologicalSort() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("dependenciesTopologicalSort() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("topologicalSort() got = %v, want %v", got, tt.want) + t.Errorf("dependenciesTopologicalSort() got = %v, want %v", got, tt.want) } }) } diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/table.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/table.go new file mode 100644 index 00000000000..490dd032210 --- /dev/null +++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/table.go @@ -0,0 +1,141 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subtaskmeta_sorter + +import ( + "fmt" + "github.com/apache/incubator-devlake/core/plugin" + "sort" +) + +type TableSorter struct { + metas []*plugin.SubTaskMeta +} + +func NewTableSorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter { + return &TableSorter{metas: metas} +} + +func (d *TableSorter) Sort() ([]plugin.SubTaskMeta, error) { + return dependencyTableTopologicalSort(d.metas) +} + +type SubtaskPrefix string + +const ( + prefixCollect SubtaskPrefix = "collect" + prefixExtract SubtaskPrefix = "extract" + prefixConvert SubtaskPrefix = "convert" +) + +func genClassNameByMetaName(rawName string) (string, error) { + if len(rawName) > 7 { + return rawName[7:], nil + } + return "", fmt.Errorf("got illeagal raw name = %s", rawName) +} + +// stable topological sort +func dependencyTableTopologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta, error) { + // TODO 1. can i use reflect to realize collect, extractor, converter ? + // first process same class data + // suppose different class has no dependency relation + // construct class name list and table list meta + // sort different metas + // add list by convert and + + // 1. construct data to sort + classNameToSubtaskListMap := make(map[string][]*plugin.SubTaskMeta) // use subtask class name to get meta list + classNameToTableListMap := make(map[string][]string) // use class name get meta name list + subtaskNameToDataMap := make(map[string]*plugin.SubTaskMeta) // use name to get meta + + for _, metaItem := range metas { + taskClassName, err := genClassNameByMetaName(metaItem.Name) + if err != nil { + return nil, err + } + if value, ok := classNameToSubtaskListMap[taskClassName]; ok { + classNameToSubtaskListMap[taskClassName] = append(value, metaItem) + } else { + classNameToSubtaskListMap[taskClassName] = []*plugin.SubTaskMeta{metaItem} + } + if value, ok := classNameToTableListMap[taskClassName]; ok { + // check if subtask in one class has different tables define + if len(value) != len(metaItem.DependencyTables) { + return nil, fmt.Errorf("got different table list in class %s", taskClassName) + } + // check list item in value and metaItem.DependencyTables, make sure it's equal + sort.Strings(value) + sort.Strings(metaItem.DependencyTables) + for index, valueItem := range value { + if valueItem != metaItem.DependencyTables[index] { + return nil, fmt.Errorf("got different table list in class %s", taskClassName) + } + } + } else { + classNameToTableListMap[taskClassName] = metaItem.DependencyTables + } + subtaskNameToDataMap[metaItem.Name] = metaItem + } + + // 2. sort + sortedNameList, err := topologicalSortDifferentElements(classNameToTableListMap) + if err != nil { + return nil, err + } + + // 3. gen subtaskmeta list by sorted data and return + sortedSubtaskMetaList := make([]plugin.SubTaskMeta, 0) + for _, nameItem := range sortedNameList { + value, ok := classNameToSubtaskListMap[nameItem] + if !ok { + return nil, fmt.Errorf("failed get subtask list by class name = %s", nameItem) + } + tmpList := make([]plugin.SubTaskMeta, len(value)) + for _, subtaskItem := range value { + if len(value) >= 1 && len(subtaskItem.Name) > 7 { + switch SubtaskPrefix(subtaskItem.Name[:7]) { + case prefixCollect: + tmpList[0] = *subtaskItem + case prefixExtract: + tmpList[1] = *subtaskItem + case prefixConvert: + if len(value) == 3 { + tmpList[2] = *subtaskItem + } else { + return nil, fmt.Errorf("got wrong length of list with extrac subtask") + } + default: + return nil, fmt.Errorf("got wrong length of subtask %v", subtaskItem) + } + } + } + sortedSubtaskMetaList = append(sortedSubtaskMetaList, tmpList...) + } + return sortedSubtaskMetaList, nil +} + +// TODO get subtask class list, different class can task concurrency +func GetSortedClassName() []string { + return nil +} + +// TODO get subtask list by class name, this subtask list should run sequentially +func GetSubtaskMetasByClassName(className string) []*plugin.SubTaskMeta { + return nil +} diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/table_test.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/table_test.go new file mode 100644 index 00000000000..294582abf05 --- /dev/null +++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/table_test.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subtaskmeta_sorter + +import ( + "github.com/apache/incubator-devlake/core/plugin" + "reflect" + "testing" +) + +func Test_dependencyTableTopologicalSort(t *testing.T) { + pluginA := plugin.SubTaskMeta{ + Name: string(prefixCollect) + "A", + DependencyTables: []string{"Table1"}, + } + pluginB := plugin.SubTaskMeta{ + Name: string(prefixCollect) + "B", + DependencyTables: []string{"table2"}, + } + pluginC := plugin.SubTaskMeta{ + Name: string(prefixCollect) + "C", + DependencyTables: []string{"table1", "table2"}, + } + pluginD := plugin.SubTaskMeta{ + Name: string(prefixCollect) + "D", + DependencyTables: []string{"table1", "table2"}, + } + type args struct { + metas []*plugin.SubTaskMeta + } + tests := []struct { + name string + args args + want []plugin.SubTaskMeta + wantErr bool + }{ + { + name: "correct stable sort", + args: args{ + metas: []*plugin.SubTaskMeta{ + &pluginA, &pluginB, &pluginC, + }, + }, + want: []plugin.SubTaskMeta{pluginA, pluginB, pluginC}, + wantErr: false, + }, + { + name: "cycle error", + args: args{ + metas: []*plugin.SubTaskMeta{ + &pluginC, &pluginD, + }, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := dependencyTableTopologicalSort(tt.args.metas) + if (err != nil) != tt.wantErr { + t.Errorf("dependencyTableTopologicalSort() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("dependencyTableTopologicalSort() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/utils.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/utils.go new file mode 100644 index 00000000000..8a98ec71ee0 --- /dev/null +++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/utils.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subtaskmeta_sorter + +import ( + "fmt" + "sort" +) + +// topologicalSortSameElements +func topologicalSortSameElements(sameElementsDependencyMap map[string][]string) ([]string, error) { + sortedKeyList := make([]string, 0) + for { + if len(sameElementsDependencyMap) == 0 { + break + } + tmpList := make([]string, 0) + for key, item := range sameElementsDependencyMap { + if len(item) == 0 { + tmpList = append(tmpList, key) + } + } + if len(tmpList) == 0 { + return nil, fmt.Errorf("cyclic dependency detected: %v", sameElementsDependencyMap) + } + // remove item in dependencies map + for key, value := range sameElementsDependencyMap { + if contains(tmpList, key) { + delete(sameElementsDependencyMap, key) + } else { + sameElementsDependencyMap[key] = removeElements(value, tmpList) + } + } + sort.Strings(tmpList) + sortedKeyList = append(sortedKeyList, tmpList...) + } + return sortedKeyList, nil +} + +// topologicalSortDifferentElements +func topologicalSortDifferentElements(differentElementsDependenciesMap map[string][]string) ([]string, error) { + sortedKeyList := make([]string, 0) + for { + if len(differentElementsDependenciesMap) == 0 { + break + } + tmpKeyList := make([]string, 0) + tmpValueItemList := make([]string, 0) + for key, item := range differentElementsDependenciesMap { + if len(item) == 0 || len(item) == 1 { + tmpKeyList = append(tmpKeyList, key) + if len(item) == 1 { + tmpValueItemList = append(tmpValueItemList, item[0]) + } + delete(differentElementsDependenciesMap, key) + } + } + if len(tmpKeyList) == 0 { + return nil, fmt.Errorf("cyclic dependency detected: %v", differentElementsDependenciesMap) + } + for key, item := range differentElementsDependenciesMap { + differentElementsDependenciesMap[key] = removeElements(item, tmpValueItemList) + } + sort.Strings(tmpKeyList) + sortedKeyList = append(sortedKeyList, tmpKeyList...) + } + return sortedKeyList, nil +} + +func contains[T comparable](itemList []T, item T) bool { + for _, newItem := range itemList { + if item == newItem { + return true + } + } + return false +} + +func removeElements[T comparable](raw, toRemove []T) []T { + newList := make([]T, 0) + for _, item := range raw { + if !contains(toRemove, item) { + newList = append(newList, item) + } + } + return newList +} diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/utils_test.go b/backend/helpers/pluginhelper/subtaskmeta_sorter/utils_test.go new file mode 100644 index 00000000000..939e4e307ff --- /dev/null +++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/utils_test.go @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subtaskmeta_sorter + +import ( + "reflect" + "testing" +) + +func Test_topologicalSortSameElements(t *testing.T) { + type args struct { + dependenciesMap map[string][]string + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "correct stable sort", + args: args{ + map[string][]string{ + "Aa": []string{"B", "C"}, + "Ac": []string{"B", "C"}, + "B": []string{"C"}, + "C": []string{}, + }, + }, + want: []string{"C", "B", "Aa", "Ac"}, + wantErr: false, + }, + { + name: "cyclic error", + args: args{ + map[string][]string{ + "A": []string{"B", "C"}, + "B": []string{"C"}, + "C": []string{"A"}, + }, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := topologicalSortSameElements(tt.args.dependenciesMap) + if (err != nil) != tt.wantErr { + t.Errorf("topologicalSortSameElements() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("topologicalSortSameElements() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_topologicalSortDifferentElements(t *testing.T) { + type args struct { + dependenciesMap map[string][]string + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "correct stable sort", + args: args{ + dependenciesMap: map[string][]string{ + "A": []string{"table1", "table2"}, + "B": []string{"table1"}, + "C": []string{"table2"}, + }, + }, + want: []string{"B", "C", "A"}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := topologicalSortDifferentElements(tt.args.dependenciesMap) + if (err != nil) != tt.wantErr { + t.Errorf("topologicalSortDifferentElements() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("topologicalSortDifferentElements() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/plugins/feishu/impl/impl.go b/backend/plugins/feishu/impl/impl.go index 799466f40dd..061c297b9ab 100644 --- a/backend/plugins/feishu/impl/impl.go +++ b/backend/plugins/feishu/impl/impl.go @@ -19,9 +19,9 @@ package impl import ( "fmt" + "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" diff --git a/backend/plugins/feishu/tasks/chat_extractor.go b/backend/plugins/feishu/tasks/chat_extractor.go index abad57fe589..e5c7cd8579a 100644 --- a/backend/plugins/feishu/tasks/chat_extractor.go +++ b/backend/plugins/feishu/tasks/chat_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -55,7 +56,7 @@ func ExtractChatItem(taskCtx plugin.SubTaskContext) errors.Error { } var ExtractChatItemMeta = plugin.SubTaskMeta{ - Name: "extractChatItem", + Name: "extractChat", EntryPoint: ExtractChatItem, EnabledByDefault: true, Description: "Extract raw chats data into tool layer table feishu_meeting_top_user_item", diff --git a/backend/plugins/feishu/tasks/meeting_top_user_item_extractor.go b/backend/plugins/feishu/tasks/meeting_top_user_item_extractor.go index 6c3e147bf55..7181edea789 100644 --- a/backend/plugins/feishu/tasks/meeting_top_user_item_extractor.go +++ b/backend/plugins/feishu/tasks/meeting_top_user_item_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" diff --git a/backend/plugins/feishu/tasks/message_collector.go b/backend/plugins/feishu/tasks/message_collector.go index 3823f7f099a..fef70ae09ce 100644 --- a/backend/plugins/feishu/tasks/message_collector.go +++ b/backend/plugins/feishu/tasks/message_collector.go @@ -19,15 +19,16 @@ package tasks import ( "encoding/json" + "net/http" + "net/url" + "reflect" + "strconv" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/feishu/apimodels" - "net/http" - "net/url" - "reflect" - "strconv" ) const RAW_MESSAGE_TABLE = "feishu_message" @@ -112,7 +113,7 @@ func CollectMessage(taskCtx plugin.SubTaskContext) errors.Error { } var CollectMessageMeta = plugin.SubTaskMeta{ - Name: "collectMeesage", + Name: "collectMessage", EntryPoint: CollectMessage, EnabledByDefault: true, Description: "Collect message from Feishu api", diff --git a/backend/plugins/feishu/tasks/message_extractor.go b/backend/plugins/feishu/tasks/message_extractor.go index f075b61f186..bf320d114cb 100644 --- a/backend/plugins/feishu/tasks/message_extractor.go +++ b/backend/plugins/feishu/tasks/message_extractor.go @@ -19,13 +19,14 @@ package tasks import ( "encoding/json" + "strconv" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/feishu/apimodels" "github.com/apache/incubator-devlake/plugins/feishu/models" - "strconv" - "time" ) var _ plugin.SubTaskEntryPoint = ExtractMessage @@ -80,7 +81,7 @@ func ExtractMessage(taskCtx plugin.SubTaskContext) errors.Error { } var ExtractMessageMeta = plugin.SubTaskMeta{ - Name: "extractChatItem", + Name: "extractMessage", EntryPoint: ExtractMessage, EnabledByDefault: true, Description: "Extract raw messages data into tool layer table feishu_meeting_top_user_item", diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index 2e354bf9900..5e9e45e837b 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -21,11 +21,12 @@ import ( "fmt" "time" - "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta_sorter" "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/api" @@ -45,12 +46,23 @@ var _ interface { plugin.CloseablePluginTask } = (*Github)(nil) +var sortedSubtaskList []plugin.SubTaskMeta + type Github struct{} func (p Github) Connection() dal.Tabler { return &models.GithubConnection{} } +func init() { + var err error + // check subtask meta loop and gen subtask list when init subtask meta + sortedSubtaskList, err = subtaskmeta_sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort() + if err != nil { + panic(err) + } +} + func (p Github) Scope() plugin.ToolLayerScope { return &models.GithubRepo{} } @@ -101,53 +113,7 @@ func (p Github) Name() string { } func (p Github) SubTaskMetas() []plugin.SubTaskMeta { - return []plugin.SubTaskMeta{ - tasks.CollectApiIssuesMeta, - tasks.ExtractApiIssuesMeta, - tasks.CollectApiPullRequestsMeta, - tasks.ExtractApiPullRequestsMeta, - tasks.CollectApiCommentsMeta, - tasks.ExtractApiCommentsMeta, - tasks.CollectApiEventsMeta, - tasks.ExtractApiEventsMeta, - tasks.CollectApiPullRequestCommitsMeta, - tasks.ExtractApiPullRequestCommitsMeta, - tasks.CollectApiPullRequestReviewsMeta, - tasks.ExtractApiPullRequestReviewsMeta, - tasks.CollectApiPrReviewCommentsMeta, - tasks.ExtractApiPrReviewCommentsMeta, - tasks.CollectApiCommitsMeta, - tasks.ExtractApiCommitsMeta, - tasks.CollectApiCommitStatsMeta, - tasks.ExtractApiCommitStatsMeta, - tasks.CollectMilestonesMeta, - tasks.ExtractMilestonesMeta, - tasks.CollectAccountsMeta, - tasks.ExtractAccountsMeta, - tasks.CollectAccountOrgMeta, - tasks.ExtractAccountOrgMeta, - tasks.CollectRunsMeta, - tasks.ExtractRunsMeta, - tasks.ConvertRunsMeta, - tasks.CollectJobsMeta, - tasks.ExtractJobsMeta, - tasks.ConvertJobsMeta, - tasks.EnrichPullRequestIssuesMeta, - tasks.ConvertRepoMeta, - tasks.ConvertIssuesMeta, - tasks.ConvertIssueAssigneeMeta, - tasks.ConvertCommitsMeta, - tasks.ConvertIssueLabelsMeta, - tasks.ConvertPullRequestCommitsMeta, - tasks.ConvertPullRequestsMeta, - tasks.ConvertPullRequestReviewsMeta, - tasks.ConvertPullRequestLabelsMeta, - tasks.ConvertPullRequestIssuesMeta, - tasks.ConvertIssueCommentsMeta, - tasks.ConvertPullRequestCommentsMeta, - tasks.ConvertMilestonesMeta, - tasks.ConvertAccountsMeta, - } + return sortedSubtaskList } func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { diff --git a/backend/plugins/github/tasks/account_collector.go b/backend/plugins/github/tasks/account_collector.go index 2aa8beada87..63e97416a4a 100644 --- a/backend/plugins/github/tasks/account_collector.go +++ b/backend/plugins/github/tasks/account_collector.go @@ -30,6 +30,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectAccountsMeta) +} + const RAW_ACCOUNT_TABLE = "github_api_accounts" type SimpleAccount struct { @@ -92,4 +96,5 @@ var CollectAccountsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect accounts data from Github api, does not support either timeFilter or diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&ExtractMilestonesMeta}, } diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go index 53ad1563d85..1d44f84659a 100644 --- a/backend/plugins/github/tasks/account_convertor.go +++ b/backend/plugins/github/tasks/account_convertor.go @@ -18,6 +18,9 @@ limitations under the License. package tasks import ( + "reflect" + "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" @@ -27,16 +30,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" - "strings" ) +func init() { + RegisterSubtaskMeta(&ConvertAccountsMeta) +} + var ConvertAccountsMeta = plugin.SubTaskMeta{ Name: "convertAccounts", EntryPoint: ConvertAccounts, EnabledByDefault: true, Description: "Convert tool layer table github_accounts into domain layer table accounts", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&ConvertMilestonesMeta}, } type GithubAccountWithOrg struct { diff --git a/backend/plugins/github/tasks/account_extractor.go b/backend/plugins/github/tasks/account_extractor.go index 3c2f6896b21..11b23c410ba 100644 --- a/backend/plugins/github/tasks/account_extractor.go +++ b/backend/plugins/github/tasks/account_extractor.go @@ -19,19 +19,25 @@ package tasks import ( "encoding/json" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "time" ) +func init() { + RegisterSubtaskMeta(&ExtractAccountsMeta) +} + var ExtractAccountsMeta = plugin.SubTaskMeta{ Name: "extractAccounts", EntryPoint: ExtractAccounts, EnabledByDefault: true, Description: "Extract raw account data into tool layer table github_accounts", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectAccountsMeta}, } type DetailGithubAccountResponse struct { diff --git a/backend/plugins/github/tasks/account_org_collector.go b/backend/plugins/github/tasks/account_org_collector.go index ae8c569ad58..9dc72c58b0f 100644 --- a/backend/plugins/github/tasks/account_org_collector.go +++ b/backend/plugins/github/tasks/account_org_collector.go @@ -19,16 +19,21 @@ package tasks import ( "encoding/json" + "io" + "net/http" + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "io" - "net/http" - "reflect" ) +func init() { + RegisterSubtaskMeta(&CollectAccountOrgMeta) +} + const RAW_ACCOUNT_ORG_TABLE = "github_api_account_orgs" type SimpleAccountWithId struct { @@ -92,4 +97,5 @@ var CollectAccountOrgMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect accounts org data from Github api, does not support either timeFilter or diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&ExtractAccountsMeta}, } diff --git a/backend/plugins/github/tasks/account_org_extractor.go b/backend/plugins/github/tasks/account_org_extractor.go index 1f37c1041ee..e42f696f302 100644 --- a/backend/plugins/github/tasks/account_org_extractor.go +++ b/backend/plugins/github/tasks/account_org_extractor.go @@ -19,18 +19,24 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractAccountOrgMeta) +} + var ExtractAccountOrgMeta = plugin.SubTaskMeta{ Name: "ExtractAccountOrg", EntryPoint: ExtractAccountOrg, EnabledByDefault: true, Description: "Extract raw account org data into tool layer table github_account_orgs", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectAccountOrgMeta}, } type GithubAccountOrgsResponse struct { diff --git a/backend/plugins/github/tasks/cicd_job_collector.go b/backend/plugins/github/tasks/cicd_job_collector.go index 4dd95c5c534..086e37c29e9 100644 --- a/backend/plugins/github/tasks/cicd_job_collector.go +++ b/backend/plugins/github/tasks/cicd_job_collector.go @@ -31,6 +31,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectJobsMeta) +} + const RAW_JOB_TABLE = "github_api_jobs" var CollectJobsMeta = plugin.SubTaskMeta{ @@ -39,6 +43,7 @@ var CollectJobsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect Jobs data from Github action api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + Dependencies: []*plugin.SubTaskMeta{&ConvertRunsMeta}, } func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go index b59ae50da8a..cd92151a0d0 100644 --- a/backend/plugins/github/tasks/cicd_job_convertor.go +++ b/backend/plugins/github/tasks/cicd_job_convertor.go @@ -18,6 +18,9 @@ limitations under the License. package tasks import ( + "reflect" + "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,16 +29,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" - "strings" ) +func init() { + RegisterSubtaskMeta(&ConvertJobsMeta) +} + var ConvertJobsMeta = plugin.SubTaskMeta{ Name: "convertJobs", EntryPoint: ConvertJobs, EnabledByDefault: true, Description: "Convert tool layer table github_jobs into domain layer table cicd_tasks", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + Dependencies: []*plugin.SubTaskMeta{&ExtractJobsMeta}, } type SimpleBranch struct { diff --git a/backend/plugins/github/tasks/cicd_job_extractor.go b/backend/plugins/github/tasks/cicd_job_extractor.go index edb96cccf06..9bfb71fff65 100644 --- a/backend/plugins/github/tasks/cicd_job_extractor.go +++ b/backend/plugins/github/tasks/cicd_job_extractor.go @@ -28,12 +28,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractJobsMeta) +} + var ExtractJobsMeta = plugin.SubTaskMeta{ Name: "extractJobs", EntryPoint: ExtractJobs, EnabledByDefault: true, Description: "Extract raw run data into tool layer table github_jobs", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + Dependencies: []*plugin.SubTaskMeta{&CollectJobsMeta}, } func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go index 3af4a66ce04..e29c005d165 100644 --- a/backend/plugins/github/tasks/cicd_run_collector.go +++ b/backend/plugins/github/tasks/cicd_run_collector.go @@ -33,6 +33,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectRunsMeta) +} + const RAW_RUN_TABLE = "github_api_runs" // Although the API accepts a maximum of 100 entries per page, sometimes @@ -56,6 +60,7 @@ var CollectRunsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect Runs data from Github action api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + Dependencies: []*plugin.SubTaskMeta{&CollectAccountOrgMeta}, } func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go index 09bccc84ff0..b04ed800977 100644 --- a/backend/plugins/github/tasks/cicd_run_convertor.go +++ b/backend/plugins/github/tasks/cicd_run_convertor.go @@ -31,12 +31,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ConvertRunsMeta) +} + var ConvertRunsMeta = plugin.SubTaskMeta{ Name: "convertRuns", EntryPoint: ConvertRuns, EnabledByDefault: true, Description: "Convert tool layer table github_runs into domain layer table cicd_pipeline", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + Dependencies: []*plugin.SubTaskMeta{&ExtractRunsMeta}, } func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/cicd_run_extractor.go b/backend/plugins/github/tasks/cicd_run_extractor.go index de3326f09ad..8991743460f 100644 --- a/backend/plugins/github/tasks/cicd_run_extractor.go +++ b/backend/plugins/github/tasks/cicd_run_extractor.go @@ -27,12 +27,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractRunsMeta) +} + var ExtractRunsMeta = plugin.SubTaskMeta{ Name: "extractRuns", EntryPoint: ExtractRuns, EnabledByDefault: true, Description: "Extract raw run data into tool layer table github_runs", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + Dependencies: []*plugin.SubTaskMeta{&CollectRunsMeta}, } func ExtractRuns(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go index 709a7dc3564..ecbcf243357 100644 --- a/backend/plugins/github/tasks/comment_collector.go +++ b/backend/plugins/github/tasks/comment_collector.go @@ -28,6 +28,10 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +func init() { + RegisterSubtaskMeta(&CollectApiCommentsMeta) +} + const RAW_COMMENTS_TABLE = "github_api_comments" func CollectApiComments(taskCtx plugin.SubTaskContext) errors.Error { @@ -94,4 +98,5 @@ var CollectApiCommentsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect comments data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiPullRequestsMeta}, } diff --git a/backend/plugins/github/tasks/comment_extractor.go b/backend/plugins/github/tasks/comment_extractor.go index d0bc66a3e7e..60d528a954b 100644 --- a/backend/plugins/github/tasks/comment_extractor.go +++ b/backend/plugins/github/tasks/comment_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -27,13 +28,18 @@ import ( githubUtils "github.com/apache/incubator-devlake/plugins/github/utils" ) +func init() { + RegisterSubtaskMeta(&ExtractApiCommentsMeta) +} + var ExtractApiCommentsMeta = plugin.SubTaskMeta{ Name: "extractApiComments", EntryPoint: ExtractApiComments, EnabledByDefault: true, Description: "Extract raw comment data into tool layer table github_pull_request_comments" + "and github_issue_comments", - DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiCommentsMeta}, } type IssueComment struct { diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go index 35f7e95da22..72467e7acbe 100644 --- a/backend/plugins/github/tasks/commit_collector.go +++ b/backend/plugins/github/tasks/commit_collector.go @@ -28,6 +28,10 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +func init() { + RegisterSubtaskMeta(&CollectApiCommitsMeta) +} + const RAW_COMMIT_TABLE = "github_api_commits" var CollectApiCommitsMeta = plugin.SubTaskMeta{ @@ -36,6 +40,7 @@ var CollectApiCommitsMeta = plugin.SubTaskMeta{ EnabledByDefault: false, Description: "Collect commits data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiPrReviewCommentsMeta}, } func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/commit_convertor.go b/backend/plugins/github/tasks/commit_convertor.go index 5b85893741a..bb734146b64 100644 --- a/backend/plugins/github/tasks/commit_convertor.go +++ b/backend/plugins/github/tasks/commit_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,15 +27,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertCommitsMeta) +} + var ConvertCommitsMeta = plugin.SubTaskMeta{ Name: "convertCommits", EntryPoint: ConvertCommits, EnabledByDefault: false, Description: "Convert tool layer table github_commits into domain layer table commits", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&ConvertIssueAssigneeMeta}, } func ConvertCommits(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/commit_extractor.go b/backend/plugins/github/tasks/commit_extractor.go index b1d48610530..6d39b88fbb0 100644 --- a/backend/plugins/github/tasks/commit_extractor.go +++ b/backend/plugins/github/tasks/commit_extractor.go @@ -19,18 +19,24 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiCommitsMeta) +} + var ExtractApiCommitsMeta = plugin.SubTaskMeta{ Name: "extractApiCommits", EntryPoint: ExtractApiCommits, EnabledByDefault: false, Description: "Extract raw commit data into tool layer table github_commits", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiCommitsMeta}, } type CommitsResponse struct { diff --git a/backend/plugins/github/tasks/commit_stats_collector.go b/backend/plugins/github/tasks/commit_stats_collector.go index 6d2cbc41087..ba8e406c13a 100644 --- a/backend/plugins/github/tasks/commit_stats_collector.go +++ b/backend/plugins/github/tasks/commit_stats_collector.go @@ -32,6 +32,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectApiCommitStatsMeta) +} + const RAW_COMMIT_STATS_TABLE = "github_api_commit_stats" var CollectApiCommitStatsMeta = plugin.SubTaskMeta{ @@ -40,6 +44,7 @@ var CollectApiCommitStatsMeta = plugin.SubTaskMeta{ EnabledByDefault: false, Description: "Collect commitStats data from Github api, does not support either timeFilter or diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommitsMeta}, } func CollectApiCommitStats(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/commit_stats_extractor.go b/backend/plugins/github/tasks/commit_stats_extractor.go index 0cbe309938d..3a8abc3c830 100644 --- a/backend/plugins/github/tasks/commit_stats_extractor.go +++ b/backend/plugins/github/tasks/commit_stats_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -26,12 +27,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiCommitStatsMeta) +} + var ExtractApiCommitStatsMeta = plugin.SubTaskMeta{ Name: "extractApiCommitStats", EntryPoint: ExtractApiCommitStats, EnabledByDefault: false, Description: "Extract raw commit stats data into tool layer table github_commit_stats", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiCommitStatsMeta}, } type ApiSingleCommitResponse struct { diff --git a/backend/plugins/github/tasks/event_collector.go b/backend/plugins/github/tasks/event_collector.go index da15a98de09..79f7f67c670 100644 --- a/backend/plugins/github/tasks/event_collector.go +++ b/backend/plugins/github/tasks/event_collector.go @@ -33,6 +33,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectApiEventsMeta) +} + const RAW_EVENTS_TABLE = "github_api_events" type SimpleGithubApiEvents struct { @@ -46,6 +50,7 @@ var CollectApiEventsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect Events data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommentsMeta}, } func CollectApiEvents(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/event_extractor.go b/backend/plugins/github/tasks/event_extractor.go index 94e3ddeb73b..f0f76667cd1 100644 --- a/backend/plugins/github/tasks/event_extractor.go +++ b/backend/plugins/github/tasks/event_extractor.go @@ -19,18 +19,24 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiEventsMeta) +} + var ExtractApiEventsMeta = plugin.SubTaskMeta{ Name: "extractApiEvents", EntryPoint: ExtractApiEvents, EnabledByDefault: true, Description: "Extract raw Events data into tool layer table github_issue_events", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiEventsMeta}, } type IssueEvent struct { diff --git a/backend/plugins/github/tasks/issue_assignee_convertor.go b/backend/plugins/github/tasks/issue_assignee_convertor.go index 373bec2b548..b23c46d4dcd 100644 --- a/backend/plugins/github/tasks/issue_assignee_convertor.go +++ b/backend/plugins/github/tasks/issue_assignee_convertor.go @@ -29,12 +29,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ConvertIssueAssigneeMeta) +} + var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{ Name: "convertIssueAssignee", EntryPoint: ConvertIssueAssignee, EnabledByDefault: true, Description: "Convert tool layer table _tool_github_issue_assignees into domain layer table issue_assignees", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ConvertIssuesMeta}, } func ConvertIssueAssignee(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go index e646f7e0874..81ac95e73df 100644 --- a/backend/plugins/github/tasks/issue_collector.go +++ b/backend/plugins/github/tasks/issue_collector.go @@ -28,6 +28,10 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +func init() { + RegisterSubtaskMeta(&CollectApiIssuesMeta) +} + const RAW_ISSUE_TABLE = "github_api_issues" var CollectApiIssuesMeta = plugin.SubTaskMeta{ @@ -36,6 +40,7 @@ var CollectApiIssuesMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect issues data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{}, } func CollectApiIssues(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/issue_comment_convertor.go b/backend/plugins/github/tasks/issue_comment_convertor.go index d1d7f70e456..d86849e3252 100644 --- a/backend/plugins/github/tasks/issue_comment_convertor.go +++ b/backend/plugins/github/tasks/issue_comment_convertor.go @@ -30,12 +30,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ConvertIssueCommentsMeta) +} + var ConvertIssueCommentsMeta = plugin.SubTaskMeta{ Name: "convertIssueComments", EntryPoint: ConvertIssueComments, EnabledByDefault: true, Description: "ConvertIssueComments data from Github api", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ConvertPullRequestIssuesMeta}, } func ConvertIssueComments(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/issue_convertor.go b/backend/plugins/github/tasks/issue_convertor.go index 9aa6cccf409..7cc3f914351 100644 --- a/backend/plugins/github/tasks/issue_convertor.go +++ b/backend/plugins/github/tasks/issue_convertor.go @@ -18,6 +18,10 @@ limitations under the License. package tasks import ( + "reflect" + "strconv" + "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,17 +30,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" - "strconv" - "strings" ) +func init() { + RegisterSubtaskMeta(&ConvertIssuesMeta) +} + var ConvertIssuesMeta = plugin.SubTaskMeta{ Name: "convertIssues", EntryPoint: ConvertIssues, EnabledByDefault: true, Description: "Convert tool layer table github_issues into domain layer table issues", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ConvertRepoMeta}, } func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/issue_extractor.go b/backend/plugins/github/tasks/issue_extractor.go index 29f6c43e2d0..891f801316f 100644 --- a/backend/plugins/github/tasks/issue_extractor.go +++ b/backend/plugins/github/tasks/issue_extractor.go @@ -29,12 +29,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiIssuesMeta) +} + var ExtractApiIssuesMeta = plugin.SubTaskMeta{ Name: "extractApiIssues", EntryPoint: ExtractApiIssues, EnabledByDefault: true, Description: "Extract raw Issues data into tool layer table github_issues", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiIssuesMeta}, } type IssuesResponse struct { diff --git a/backend/plugins/github/tasks/issue_label_convertor.go b/backend/plugins/github/tasks/issue_label_convertor.go index 87ac3ba8b37..0ebfa88d5e2 100644 --- a/backend/plugins/github/tasks/issue_label_convertor.go +++ b/backend/plugins/github/tasks/issue_label_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -25,15 +27,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertIssueLabelsMeta) +} + var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ Name: "convertIssueLabels", EntryPoint: ConvertIssueLabels, EnabledByDefault: true, Description: "Convert tool layer table github_issue_labels into domain layer table issue_labels", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ConvertCommitsMeta}, } func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/milestone_collector.go b/backend/plugins/github/tasks/milestone_collector.go index 51d4d3439a3..c5781bec2f4 100644 --- a/backend/plugins/github/tasks/milestone_collector.go +++ b/backend/plugins/github/tasks/milestone_collector.go @@ -28,6 +28,10 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +func init() { + RegisterSubtaskMeta(&CollectMilestonesMeta) +} + const RAW_MILESTONE_TABLE = "github_milestones" var CollectMilestonesMeta = plugin.SubTaskMeta{ @@ -36,6 +40,7 @@ var CollectMilestonesMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect milestone data from Github api, does not support either timeFilter or diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommitStatsMeta}, } func CollectApiMilestones(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/milestone_converter.go b/backend/plugins/github/tasks/milestone_converter.go index 57adddd7332..e9dc6e62bfb 100644 --- a/backend/plugins/github/tasks/milestone_converter.go +++ b/backend/plugins/github/tasks/milestone_converter.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" @@ -27,15 +29,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertMilestonesMeta) +} + var ConvertMilestonesMeta = plugin.SubTaskMeta{ Name: "convertMilestones", EntryPoint: ConvertMilestones, EnabledByDefault: true, Description: "Convert tool layer table github_milestones into domain layer table milestones", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ConvertPullRequestCommentsMeta}, } type MilestoneConverterModel struct { diff --git a/backend/plugins/github/tasks/milestone_extractor.go b/backend/plugins/github/tasks/milestone_extractor.go index f5dd8f16d4a..47d64568a4d 100644 --- a/backend/plugins/github/tasks/milestone_extractor.go +++ b/backend/plugins/github/tasks/milestone_extractor.go @@ -19,18 +19,24 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractMilestonesMeta) +} + var ExtractMilestonesMeta = plugin.SubTaskMeta{ Name: "extractMilestones", EntryPoint: ExtractMilestones, EnabledByDefault: true, Description: "Extract raw milestone data into tool layer table github_milestones", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&CollectMilestonesMeta}, } type MilestonesResponse struct { diff --git a/backend/plugins/github/tasks/pr_collector.go b/backend/plugins/github/tasks/pr_collector.go index d15bf3db19a..e3606946077 100644 --- a/backend/plugins/github/tasks/pr_collector.go +++ b/backend/plugins/github/tasks/pr_collector.go @@ -33,6 +33,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectApiPullRequestsMeta) +} + const RAW_PULL_REQUEST_TABLE = "github_api_pull_requests" var CollectApiPullRequestsMeta = plugin.SubTaskMeta{ @@ -41,6 +45,7 @@ var CollectApiPullRequestsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect PullRequests data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiIssuesMeta}, } type SimpleGithubPr struct { diff --git a/backend/plugins/github/tasks/pr_comment_convertor.go b/backend/plugins/github/tasks/pr_comment_convertor.go index 0b7b77bcad0..94b6f277a5f 100644 --- a/backend/plugins/github/tasks/pr_comment_convertor.go +++ b/backend/plugins/github/tasks/pr_comment_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,15 +28,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertPullRequestCommentsMeta) +} + var ConvertPullRequestCommentsMeta = plugin.SubTaskMeta{ Name: "convertPullRequestComments", EntryPoint: ConvertPullRequestComments, EnabledByDefault: true, Description: "ConvertPullRequestComments data from Github api", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ConvertIssueCommentsMeta}, } func ConvertPullRequestComments(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_commit_collector.go b/backend/plugins/github/tasks/pr_commit_collector.go index 2ee08cf762f..e250d424741 100644 --- a/backend/plugins/github/tasks/pr_commit_collector.go +++ b/backend/plugins/github/tasks/pr_commit_collector.go @@ -34,6 +34,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectApiPullRequestCommitsMeta) +} + const RAW_PR_COMMIT_TABLE = "github_api_pull_request_commits" // this struct should be moved to `gitub_api_common.go` @@ -44,6 +48,7 @@ var CollectApiPullRequestCommitsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect PullRequestCommits data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiEventsMeta}, } type SimplePr struct { diff --git a/backend/plugins/github/tasks/pr_commit_convertor.go b/backend/plugins/github/tasks/pr_commit_convertor.go index 267d343e7fa..af8fdf54120 100644 --- a/backend/plugins/github/tasks/pr_commit_convertor.go +++ b/backend/plugins/github/tasks/pr_commit_convertor.go @@ -29,12 +29,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ConvertPullRequestCommitsMeta) +} + var ConvertPullRequestCommitsMeta = plugin.SubTaskMeta{ Name: "convertPullRequestCommits", EntryPoint: ConvertPullRequestCommits, EnabledByDefault: true, Description: "Convert tool layer table github_pull_request_commits into domain layer table pull_request_commits", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ConvertIssueLabelsMeta}, } func ConvertPullRequestCommits(taskCtx plugin.SubTaskContext) (err errors.Error) { diff --git a/backend/plugins/github/tasks/pr_commit_extractor.go b/backend/plugins/github/tasks/pr_commit_extractor.go index 8418bbc8e47..d74c1a89e01 100644 --- a/backend/plugins/github/tasks/pr_commit_extractor.go +++ b/backend/plugins/github/tasks/pr_commit_extractor.go @@ -28,12 +28,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiPullRequestCommitsMeta) +} + var ExtractApiPullRequestCommitsMeta = plugin.SubTaskMeta{ Name: "extractApiPullRequestCommits", EntryPoint: ExtractApiPullRequestCommits, EnabledByDefault: true, Description: "Extract raw PullRequestCommits data into tool layer table github_commits", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiPullRequestCommitsMeta}, } type PrCommitsResponse struct { diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go index 4bee55443b3..90e66b7015e 100644 --- a/backend/plugins/github/tasks/pr_convertor.go +++ b/backend/plugins/github/tasks/pr_convertor.go @@ -30,12 +30,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ConvertPullRequestsMeta) +} + var ConvertPullRequestsMeta = plugin.SubTaskMeta{ Name: "convertPullRequests", EntryPoint: ConvertPullRequests, EnabledByDefault: true, Description: "ConvertPullRequests data from Github api", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ConvertPullRequestCommitsMeta}, } func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go index 8a03cdabce4..3ef0957a03d 100644 --- a/backend/plugins/github/tasks/pr_extractor.go +++ b/backend/plugins/github/tasks/pr_extractor.go @@ -27,12 +27,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiPullRequestsMeta) +} + var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{ Name: "extractApiPullRequests", EntryPoint: ExtractApiPullRequests, EnabledByDefault: true, Description: "Extract raw PullRequests data into tool layer table github_pull_requests", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiPullRequestsMeta}, } type GithubApiPullRequest struct { diff --git a/backend/plugins/github/tasks/pr_issue_convertor.go b/backend/plugins/github/tasks/pr_issue_convertor.go index e8611c88b8a..e52d9c16806 100644 --- a/backend/plugins/github/tasks/pr_issue_convertor.go +++ b/backend/plugins/github/tasks/pr_issue_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" @@ -25,15 +27,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertPullRequestIssuesMeta) +} + var ConvertPullRequestIssuesMeta = plugin.SubTaskMeta{ Name: "convertPullRequestIssues", EntryPoint: ConvertPullRequestIssues, EnabledByDefault: true, Description: "Convert tool layer table github_pull_request_issues into domain layer table pull_request_issues", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&ConvertPullRequestReviewsMeta}, } func ConvertPullRequestIssues(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_issue_enricher.go b/backend/plugins/github/tasks/pr_issue_enricher.go index ed0cbacae72..bc7c6c928c3 100644 --- a/backend/plugins/github/tasks/pr_issue_enricher.go +++ b/backend/plugins/github/tasks/pr_issue_enricher.go @@ -30,12 +30,17 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&EnrichPullRequestIssuesMeta) +} + var EnrichPullRequestIssuesMeta = plugin.SubTaskMeta{ Name: "enrichPullRequestIssues", EntryPoint: EnrichPullRequestIssues, EnabledByDefault: true, Description: "Create tool layer table github_pull_request_issues from github_pull_reqeusts", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&ConvertJobsMeta}, } func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext) (err errors.Error) { diff --git a/backend/plugins/github/tasks/pr_label_convertor.go b/backend/plugins/github/tasks/pr_label_convertor.go index 2c8b443cc78..419ad3bdc73 100644 --- a/backend/plugins/github/tasks/pr_label_convertor.go +++ b/backend/plugins/github/tasks/pr_label_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,15 +27,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertPullRequestLabelsMeta) +} + var ConvertPullRequestLabelsMeta = plugin.SubTaskMeta{ Name: "convertPullRequestLabels", EntryPoint: ConvertPullRequestLabels, EnabledByDefault: true, Description: "Convert tool layer table github_pull_request_labels into domain layer table pull_request_labels", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ConvertPullRequestReviewsMeta}, } func ConvertPullRequestLabels(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_review_collector.go b/backend/plugins/github/tasks/pr_review_collector.go index 888186cea08..d330d3e81c1 100644 --- a/backend/plugins/github/tasks/pr_review_collector.go +++ b/backend/plugins/github/tasks/pr_review_collector.go @@ -31,6 +31,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&CollectApiPullRequestReviewsMeta) +} + const RAW_PR_REVIEW_TABLE = "github_api_pull_request_reviews" // this struct should be moved to `gitub_api_common.go` @@ -41,6 +45,7 @@ var CollectApiPullRequestReviewsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect PullRequestReviews data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiPullRequestCommitsMeta}, } func CollectApiPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go index 2468b1b932d..bd669b03281 100644 --- a/backend/plugins/github/tasks/pr_review_comment_collector.go +++ b/backend/plugins/github/tasks/pr_review_comment_collector.go @@ -28,6 +28,10 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +func init() { + RegisterSubtaskMeta(&CollectApiPrReviewCommentsMeta) +} + const RAW_PR_REVIEW_COMMENTS_TABLE = "github_api_pull_request_review_comments" // this struct should be moved to `github_api_common.go` @@ -95,4 +99,5 @@ var CollectApiPrReviewCommentsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Collect pr review comments data from Github api, supports both timeFilter and diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiPullRequestCommitsMeta}, } diff --git a/backend/plugins/github/tasks/pr_review_comment_extractor.go b/backend/plugins/github/tasks/pr_review_comment_extractor.go index fa7181e7c26..9fe8de42c34 100644 --- a/backend/plugins/github/tasks/pr_review_comment_extractor.go +++ b/backend/plugins/github/tasks/pr_review_comment_extractor.go @@ -30,13 +30,18 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ExtractApiPrReviewCommentsMeta) +} + var ExtractApiPrReviewCommentsMeta = plugin.SubTaskMeta{ Name: "extractApiPrReviewComments", EntryPoint: ExtractApiPrReviewComments, EnabledByDefault: true, Description: "Extract raw comment data into tool layer table github_pull_request_comments" + "and github_issue_comments", - DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiPrReviewCommentsMeta}, } func ExtractApiPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_review_convertor.go b/backend/plugins/github/tasks/pr_review_convertor.go index f526c159205..f040debc3e2 100644 --- a/backend/plugins/github/tasks/pr_review_convertor.go +++ b/backend/plugins/github/tasks/pr_review_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,15 +28,19 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" ) +func init() { + RegisterSubtaskMeta(&ConvertPullRequestReviewsMeta) +} + var ConvertPullRequestReviewsMeta = plugin.SubTaskMeta{ Name: "convertPullRequestReviews", EntryPoint: ConvertPullRequestReviews, EnabledByDefault: true, Description: "ConvertPullRequestReviews data from Github api", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&ConvertPullRequestsMeta}, } func ConvertPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go index 665ed3a5d4a..75f0f5e8e6e 100644 --- a/backend/plugins/github/tasks/pr_review_extractor.go +++ b/backend/plugins/github/tasks/pr_review_extractor.go @@ -19,19 +19,25 @@ package tasks import ( "encoding/json" + "strings" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "strings" ) +func init() { + RegisterSubtaskMeta(&ExtractApiPullRequestReviewsMeta) +} + var ExtractApiPullRequestReviewsMeta = plugin.SubTaskMeta{ Name: "extractApiPullRequestReviews", EntryPoint: ExtractApiPullRequestReviews, EnabledByDefault: true, Description: "Extract raw PullRequestReviewers data into tool layer table github_reviewers", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiPullRequestReviewsMeta}, } type PullRequestReview struct { diff --git a/backend/plugins/github/tasks/register.go b/backend/plugins/github/tasks/register.go new file mode 100644 index 00000000000..51b12a76c1a --- /dev/null +++ b/backend/plugins/github/tasks/register.go @@ -0,0 +1,26 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/core/plugin" + +var SubTaskMetaList []*plugin.SubTaskMeta + +func RegisterSubtaskMeta(meta *plugin.SubTaskMeta) { + SubTaskMetaList = append(SubTaskMetaList, meta) +} diff --git a/backend/plugins/github/tasks/repo_convertor.go b/backend/plugins/github/tasks/repo_convertor.go index 418ef7a0025..5c60832b9fb 100644 --- a/backend/plugins/github/tasks/repo_convertor.go +++ b/backend/plugins/github/tasks/repo_convertor.go @@ -34,6 +34,10 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +func init() { + RegisterSubtaskMeta(&ConvertRepoMeta) +} + const RAW_REPOSITORIES_TABLE = "github_api_repositories" type ApiRepoResponse GithubApiRepo @@ -58,6 +62,7 @@ var ConvertRepoMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Convert tool layer table github_repos into domain layer table repos and boards", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&EnrichPullRequestIssuesMeta}, } func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/account_convertor.go b/backend/plugins/gitlab/tasks/account_convertor.go index 36bc314c8e5..98a54ae02c1 100644 --- a/backend/plugins/gitlab/tasks/account_convertor.go +++ b/backend/plugins/gitlab/tasks/account_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,7 +28,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go b/backend/plugins/gitlab/tasks/commit_convertor.go index c285a785e67..e74d5758daf 100644 --- a/backend/plugins/gitlab/tasks/commit_convertor.go +++ b/backend/plugins/gitlab/tasks/commit_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go b/backend/plugins/gitlab/tasks/commit_extractor.go index 7cbacf99754..07dd6109b22 100644 --- a/backend/plugins/gitlab/tasks/commit_extractor.go +++ b/backend/plugins/gitlab/tasks/commit_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index 6d09b4b19c9..81f3de71cf2 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -18,6 +18,10 @@ limitations under the License. package tasks import ( + "reflect" + "strconv" + "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,9 +30,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" - "strconv" - "strings" ) func init() { diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go index c79ce159082..4e22f2f7aab 100644 --- a/backend/plugins/gitlab/tasks/issue_label_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go index bbf795a51f0..6be8d5bd284 100644 --- a/backend/plugins/gitlab/tasks/job_extractor.go +++ b/backend/plugins/gitlab/tasks/job_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_comment_convertor.go index 1ea38c9c68a..eef12851b09 100644 --- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_comment_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,7 +28,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go index 751e84a96a1..b6001fe5b91 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go index f711075320d..3f82429b99f 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go b/backend/plugins/gitlab/tasks/mr_enricher.go index 256dffb7e47..e2fd8f6dd62 100644 --- a/backend/plugins/gitlab/tasks/mr_enricher.go +++ b/backend/plugins/gitlab/tasks/mr_enricher.go @@ -18,13 +18,14 @@ limitations under the License. package tasks import ( + "reflect" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" - "time" ) func init() { diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index 60333243848..fd025dbc0d2 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go index 80f01e708a4..0017625df0e 100644 --- a/backend/plugins/gitlab/tasks/mr_note_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" diff --git a/backend/plugins/gitlab/tasks/tag_extractor.go b/backend/plugins/gitlab/tasks/tag_extractor.go index 9a94de337cc..6cae658e566 100644 --- a/backend/plugins/gitlab/tasks/tag_extractor.go +++ b/backend/plugins/gitlab/tasks/tag_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api"