From 63cc04ede28c242564a5b8e7257f8672d1f8ceb8 Mon Sep 17 00:00:00 2001 From: Nddtfjiang Date: Tue, 9 Aug 2022 15:01:51 +0000 Subject: [PATCH 1/4] feat: table info regiest and get Add some struct for table info. Add RegisterTablesInfo for regist the table info. Add GetTotalInfo,TraversalTable,Traversal for get the table info. Nddtfjiang --- plugins/core/plugin_info.go | 218 ++++++++++++++++++++++++++++++++++++ runner/directrun.go | 19 +++- runner/loader.go | 8 ++ 3 files changed, 240 insertions(+), 5 deletions(-) create mode 100644 plugins/core/plugin_info.go diff --git a/plugins/core/plugin_info.go b/plugins/core/plugin_info.go new file mode 100644 index 00000000000..86394d92178 --- /dev/null +++ b/plugins/core/plugin_info.go @@ -0,0 +1,218 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "fmt" + "reflect" + "sync" + + "gorm.io/gorm/schema" +) + +type TableInfo struct { + name string + describe string + table schema.Tabler + mutex *sync.Mutex +} +type TableInfoCallBack func(pluginName string, name string, describe *string, table schema.Tabler) (err error) + +type TablesInfo struct { + name string + tableInfo map[string]*TableInfo + mutex *sync.Mutex +} +type TablesInfoCallBack func(pluginName string, info *TablesInfo) (err error) + +type TotalInfo struct { + tablesInfo map[string]*TablesInfo +} + +var totalInfo TotalInfo = TotalInfo{tablesInfo: make(map[string]*TablesInfo)} + +type PluginInfo interface { + RegisterTablesInfo(ts *TablesInfo) error + //ReleaseModelsInfo() error +} + +func GetTotalInfo() *TotalInfo { + return &totalInfo +} + +func RegisterPluginInfo(pi PluginInfo) error { + piType := reflect.ValueOf(pi) + if piType.Kind() == reflect.Ptr { + piType = piType.Elem() + } + ts := &TablesInfo{ + name: piType.Type().Name(), + tableInfo: make(map[string]*TableInfo), + mutex: &sync.Mutex{}, + } + // regist model info + if ts.name == "" { + return fmt.Errorf("can not use empty string on name to RegisterModelsInfo() for plugin") + } + + if ts, ok := totalInfo.tablesInfo[ts.name]; ok { + return fmt.Errorf("name [%s] has been used for RegisterModelsInfo()", ts.name) + } + + totalInfo.tablesInfo[ts.name] = ts + err := pi.RegisterTablesInfo(ts) + if err != nil { + return err + } + + return nil +} + +func (ts *TablesInfo) Add(table schema.Tabler, describe ...string) error { + modelType := reflect.ValueOf(table) + if modelType.Kind() == reflect.Ptr { + modelType = modelType.Elem() + } + + desc := "[" + table.TableName() + "]" + for _, d := range describe { + desc += d + } + + m := &TableInfo{ + name: modelType.Type().Name(), + describe: desc, + table: table, + mutex: &sync.Mutex{}, + } + if _, ok := ts.tableInfo[m.name]; ok { + return fmt.Errorf("the table name [%s] has been used in plugin [%s]", m.name, ts.name) + } + ts.tableInfo[m.name] = m + return nil +} + +func (ts *TablesInfo) Adds(tables ...schema.Tabler) error { + for _, table := range tables { + err := ts.Add(table) + if err != nil { + return err + } + } + return nil +} + +func (ts *TableInfo) Name() string { + return ts.name +} + +func (ts *TablesInfo) Visit(name string, handle TableInfoCallBack) error { + if m, ok := ts.tableInfo[name]; ok { + m.mutex.Lock() + err := handle(ts.name, name, &m.describe, m.table) + m.mutex.Unlock() + if err != nil { + return err + } + } else { + return fmt.Errorf("can not find out the table with name [%s] in plugin [%s]", name, ts.name) + } + return nil +} + +func (ts *TablesInfo) VisitWithOutLock(name string, handle TableInfoCallBack) error { + if m, ok := ts.tableInfo[name]; ok { + err := handle(ts.name, name, &m.describe, m.table) + if err != nil { + return err + } + } else { + return fmt.Errorf("can not find out the table with name [%s] in plugin [%s]", name, ts.name) + } + return nil +} + +func (ts *TablesInfo) Traversal(handle TableInfoCallBack) error { + for name, m := range ts.tableInfo { + m.mutex.Lock() + err := handle(ts.name, name, &m.describe, m.table) + m.mutex.Unlock() + if err != nil { + return err + } + } + return nil +} + +func (ts *TablesInfo) TraversalWithOutLock(handle TableInfoCallBack) error { + for name, m := range ts.tableInfo { + err := handle(ts.name, name, &m.describe, m.table) + if err != nil { + return err + } + } + return nil +} + +func (t *TotalInfo) VisitTable(name string, handle TablesInfoCallBack) error { + if ts, ok := t.tablesInfo[name]; ok { + ts.mutex.Lock() + err := handle(name, ts) + ts.mutex.Unlock() + if err != nil { + return err + } + } else { + return fmt.Errorf("can not find out the plugin with name [%s] for table", name) + } + return nil +} + +func (t *TotalInfo) VisitTableWithOutLock(name string, handle TablesInfoCallBack) error { + if ts, ok := t.tablesInfo[name]; ok { + err := handle(name, ts) + if err != nil { + return err + } + } else { + return fmt.Errorf("can not find out the plugin with name [%s] for table", name) + } + return nil +} + +func (t *TotalInfo) TraversalTable(handle TablesInfoCallBack) error { + for name, ts := range t.tablesInfo { + ts.mutex.Lock() + err := handle(name, ts) + ts.mutex.Unlock() + if err != nil { + return err + } + } + return nil +} + +func (t *TotalInfo) TraversalTableWithOutLock(handle TablesInfoCallBack) error { + for name, ts := range t.tablesInfo { + err := handle(name, ts) + if err != nil { + return err + } + } + return nil +} diff --git a/runner/directrun.go b/runner/directrun.go index b93dd643351..93c5b2364a4 100644 --- a/runner/directrun.go +++ b/runner/directrun.go @@ -21,16 +21,17 @@ import ( "context" "errors" "fmt" - "github.com/apache/incubator-devlake/config" - "github.com/apache/incubator-devlake/logger" - "github.com/apache/incubator-devlake/migration" - "github.com/apache/incubator-devlake/plugins/core" - "github.com/spf13/cobra" "io" "os" "os/signal" "runtime" "syscall" + + "github.com/apache/incubator-devlake/config" + "github.com/apache/incubator-devlake/logger" + "github.com/apache/incubator-devlake/migration" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/spf13/cobra" ) // RunCmd FIXME ... @@ -64,11 +65,19 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op panic(err) } } + err = core.RegisterPlugin(cmd.Use, pluginTask.(core.PluginMeta)) if err != nil { panic(err) } + if PluginInfo, ok := pluginTask.(core.PluginInfo); ok { + err := core.RegisterPluginInfo(PluginInfo) + if err != nil { + panic(err) + } + } + // collect migration and run migration.Init(db) if migratable, ok := pluginTask.(core.Migratable); ok { diff --git a/runner/loader.go b/runner/loader.go index 4952aa7b3d8..0d65c64e6fb 100644 --- a/runner/loader.go +++ b/runner/loader.go @@ -63,6 +63,14 @@ func LoadPlugins(pluginsDir string, config *viper.Viper, logger core.Logger, db if err != nil { return nil } + + if PluginInfo, ok := symPluginEntry.(core.PluginInfo); ok { + err := core.RegisterPluginInfo(PluginInfo) + if err != nil { + return err + } + } + logger.Info(`plugin loaded %s`, pluginName) } return nil From b7d39dff08ac24e16cef605a0dc1d1a29ba01212 Mon Sep 17 00:00:00 2001 From: Nddtfjiang Date: Tue, 9 Aug 2022 15:03:49 +0000 Subject: [PATCH 2/4] feat: gitlab regist table models Add RegisterTablesInfo for gitlab. Add GetTableInfo api to get the table info for gitlab. Nddtfjiang --- plugins/gitlab/api/connection.go | 32 ++++++++++++++++++++++++++++++++ plugins/gitlab/impl/impl.go | 27 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/plugins/gitlab/api/connection.go b/plugins/gitlab/api/connection.go index 3771bef45f1..4a4c22d13ac 100644 --- a/plugins/gitlab/api/connection.go +++ b/plugins/gitlab/api/connection.go @@ -21,12 +21,15 @@ import ( "context" "fmt" "net/http" + "reflect" "time" "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/utils" "github.com/mitchellh/mapstructure" + "gorm.io/gorm/schema" ) // @Summary test gitlab connection @@ -161,3 +164,32 @@ func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error err := connectionHelper.First(connection, input.Params) return &core.ApiResourceOutput{Body: connection}, err } + +// @Summary get table info +// @Description Get table info map +// @Tags plugins/gitlab +// @Success 200 +// @Router /plugins/gitlab/table [GET] +func GetTableInfo(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + info := make(map[string]interface{}) + + err := core.GetTotalInfo().VisitTable("Gitlab", func(pluginName string, ts *core.TablesInfo) (err error) { + + tableInfo := make(map[string]interface{}) + info[pluginName] = &tableInfo + + return ts.Traversal(func(pluginName, name string, describe *string, table schema.Tabler) (err error) { + tableData := make(map[string]reflect.StructField) + tableInfo[name] = &tableData + + sf := utils.WalkFields(reflect.TypeOf(table), nil) + for _, f := range sf { + tableData[f.Name] = f + } + + return + }) + }) + + return &core.ApiResourceOutput{Body: info}, err +} diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go index c3c67fe25c3..313c4291c79 100644 --- a/plugins/gitlab/impl/impl.go +++ b/plugins/gitlab/impl/impl.go @@ -33,6 +33,7 @@ import ( var _ core.PluginMeta = (*Gitlab)(nil) var _ core.PluginInit = (*Gitlab)(nil) +var _ core.PluginInfo = (*Gitlab)(nil) var _ core.PluginTask = (*Gitlab)(nil) var _ core.PluginApi = (*Gitlab)(nil) var _ core.Migratable = (*Gitlab)(nil) @@ -46,6 +47,29 @@ func (plugin Gitlab) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) return nil } +func (plugin Gitlab) RegisterTablesInfo(ms *core.TablesInfo) error { + ms.Adds( + &models.GitlabConnection{}, + &models.GitlabAccount{}, + &models.GitlabCommit{}, + &models.GitlabIssue{}, + &models.GitlabIssueLabel{}, + &models.GitlabJob{}, + &models.GitlabMergeRequest{}, + &models.GitlabMrComment{}, + &models.GitlabMrCommit{}, + &models.GitlabMrLabel{}, + &models.GitlabMrNote{}, + &models.GitlabPipeline{}, + &models.GitlabProject{}, + &models.GitlabProjectCommit{}, + // &models.GitlabResponse{}, + &models.GitlabReviewer{}, + &models.GitlabTag{}, + ) + return nil +} + func (plugin Gitlab) Description() string { return "To collect and enrich data from Gitlab" } @@ -144,6 +168,9 @@ func (plugin Gitlab) ApiResources() map[string]map[string]core.ApiResourceHandle "connections/:connectionId/proxy/rest/*path": { "GET": api.Proxy, }, + "table": { + "GET": api.GetTableInfo, + }, } } From a885fd8dd0c2ec562b28cc116698ff9f838416b9 Mon Sep 17 00:00:00 2001 From: Nddtfjiang Date: Wed, 10 Aug 2022 12:04:18 +0000 Subject: [PATCH 3/4] refactor: remove table data saved remove the table data saved. Nddtfjiang --- plugins/core/plugin_info.go | 218 ------------------------------- plugins/core/plugin_model.go | 26 ++++ plugins/gitlab/api/connection.go | 34 ++--- plugins/gitlab/impl/impl.go | 9 +- runner/directrun.go | 7 - runner/loader.go | 7 - 6 files changed, 48 insertions(+), 253 deletions(-) delete mode 100644 plugins/core/plugin_info.go create mode 100644 plugins/core/plugin_model.go diff --git a/plugins/core/plugin_info.go b/plugins/core/plugin_info.go deleted file mode 100644 index 86394d92178..00000000000 --- a/plugins/core/plugin_info.go +++ /dev/null @@ -1,218 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package core - -import ( - "fmt" - "reflect" - "sync" - - "gorm.io/gorm/schema" -) - -type TableInfo struct { - name string - describe string - table schema.Tabler - mutex *sync.Mutex -} -type TableInfoCallBack func(pluginName string, name string, describe *string, table schema.Tabler) (err error) - -type TablesInfo struct { - name string - tableInfo map[string]*TableInfo - mutex *sync.Mutex -} -type TablesInfoCallBack func(pluginName string, info *TablesInfo) (err error) - -type TotalInfo struct { - tablesInfo map[string]*TablesInfo -} - -var totalInfo TotalInfo = TotalInfo{tablesInfo: make(map[string]*TablesInfo)} - -type PluginInfo interface { - RegisterTablesInfo(ts *TablesInfo) error - //ReleaseModelsInfo() error -} - -func GetTotalInfo() *TotalInfo { - return &totalInfo -} - -func RegisterPluginInfo(pi PluginInfo) error { - piType := reflect.ValueOf(pi) - if piType.Kind() == reflect.Ptr { - piType = piType.Elem() - } - ts := &TablesInfo{ - name: piType.Type().Name(), - tableInfo: make(map[string]*TableInfo), - mutex: &sync.Mutex{}, - } - // regist model info - if ts.name == "" { - return fmt.Errorf("can not use empty string on name to RegisterModelsInfo() for plugin") - } - - if ts, ok := totalInfo.tablesInfo[ts.name]; ok { - return fmt.Errorf("name [%s] has been used for RegisterModelsInfo()", ts.name) - } - - totalInfo.tablesInfo[ts.name] = ts - err := pi.RegisterTablesInfo(ts) - if err != nil { - return err - } - - return nil -} - -func (ts *TablesInfo) Add(table schema.Tabler, describe ...string) error { - modelType := reflect.ValueOf(table) - if modelType.Kind() == reflect.Ptr { - modelType = modelType.Elem() - } - - desc := "[" + table.TableName() + "]" - for _, d := range describe { - desc += d - } - - m := &TableInfo{ - name: modelType.Type().Name(), - describe: desc, - table: table, - mutex: &sync.Mutex{}, - } - if _, ok := ts.tableInfo[m.name]; ok { - return fmt.Errorf("the table name [%s] has been used in plugin [%s]", m.name, ts.name) - } - ts.tableInfo[m.name] = m - return nil -} - -func (ts *TablesInfo) Adds(tables ...schema.Tabler) error { - for _, table := range tables { - err := ts.Add(table) - if err != nil { - return err - } - } - return nil -} - -func (ts *TableInfo) Name() string { - return ts.name -} - -func (ts *TablesInfo) Visit(name string, handle TableInfoCallBack) error { - if m, ok := ts.tableInfo[name]; ok { - m.mutex.Lock() - err := handle(ts.name, name, &m.describe, m.table) - m.mutex.Unlock() - if err != nil { - return err - } - } else { - return fmt.Errorf("can not find out the table with name [%s] in plugin [%s]", name, ts.name) - } - return nil -} - -func (ts *TablesInfo) VisitWithOutLock(name string, handle TableInfoCallBack) error { - if m, ok := ts.tableInfo[name]; ok { - err := handle(ts.name, name, &m.describe, m.table) - if err != nil { - return err - } - } else { - return fmt.Errorf("can not find out the table with name [%s] in plugin [%s]", name, ts.name) - } - return nil -} - -func (ts *TablesInfo) Traversal(handle TableInfoCallBack) error { - for name, m := range ts.tableInfo { - m.mutex.Lock() - err := handle(ts.name, name, &m.describe, m.table) - m.mutex.Unlock() - if err != nil { - return err - } - } - return nil -} - -func (ts *TablesInfo) TraversalWithOutLock(handle TableInfoCallBack) error { - for name, m := range ts.tableInfo { - err := handle(ts.name, name, &m.describe, m.table) - if err != nil { - return err - } - } - return nil -} - -func (t *TotalInfo) VisitTable(name string, handle TablesInfoCallBack) error { - if ts, ok := t.tablesInfo[name]; ok { - ts.mutex.Lock() - err := handle(name, ts) - ts.mutex.Unlock() - if err != nil { - return err - } - } else { - return fmt.Errorf("can not find out the plugin with name [%s] for table", name) - } - return nil -} - -func (t *TotalInfo) VisitTableWithOutLock(name string, handle TablesInfoCallBack) error { - if ts, ok := t.tablesInfo[name]; ok { - err := handle(name, ts) - if err != nil { - return err - } - } else { - return fmt.Errorf("can not find out the plugin with name [%s] for table", name) - } - return nil -} - -func (t *TotalInfo) TraversalTable(handle TablesInfoCallBack) error { - for name, ts := range t.tablesInfo { - ts.mutex.Lock() - err := handle(name, ts) - ts.mutex.Unlock() - if err != nil { - return err - } - } - return nil -} - -func (t *TotalInfo) TraversalTableWithOutLock(handle TablesInfoCallBack) error { - for name, ts := range t.tablesInfo { - err := handle(name, ts) - if err != nil { - return err - } - } - return nil -} diff --git a/plugins/core/plugin_model.go b/plugins/core/plugin_model.go new file mode 100644 index 00000000000..5ddec7f6129 --- /dev/null +++ b/plugins/core/plugin_model.go @@ -0,0 +1,26 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +type Tabler interface { + TableName() string +} + +type PluginModel interface { + GetTablesInfo() []Tabler +} diff --git a/plugins/gitlab/api/connection.go b/plugins/gitlab/api/connection.go index 4a4c22d13ac..0b8d43c8bf7 100644 --- a/plugins/gitlab/api/connection.go +++ b/plugins/gitlab/api/connection.go @@ -29,7 +29,6 @@ import ( "github.com/apache/incubator-devlake/plugins/helper" "github.com/apache/incubator-devlake/utils" "github.com/mitchellh/mapstructure" - "gorm.io/gorm/schema" ) // @Summary test gitlab connection @@ -171,25 +170,28 @@ func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error // @Success 200 // @Router /plugins/gitlab/table [GET] func GetTableInfo(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - info := make(map[string]interface{}) - - err := core.GetTotalInfo().VisitTable("Gitlab", func(pluginName string, ts *core.TablesInfo) (err error) { + plugin, err := core.GetPlugin("gitlab") + if err != nil { + return nil, err + } + pm, ok := plugin.(core.PluginModel) + if !ok { + return nil, fmt.Errorf("plugin Gitlab can not change to PluginModel") + } - tableInfo := make(map[string]interface{}) - info[pluginName] = &tableInfo + tables := pm.GetTablesInfo() - return ts.Traversal(func(pluginName, name string, describe *string, table schema.Tabler) (err error) { - tableData := make(map[string]reflect.StructField) - tableInfo[name] = &tableData + info := make(map[string]interface{}) - sf := utils.WalkFields(reflect.TypeOf(table), nil) - for _, f := range sf { - tableData[f.Name] = f - } + for _, table := range tables { + sf := utils.WalkFields(reflect.TypeOf(table), nil) + tableInfo := make(map[string]string) + info[table.TableName()] = tableInfo - return - }) - }) + for _, s := range sf { + tableInfo[s.Name] = string(s.Tag) + } + } return &core.ApiResourceOutput{Body: info}, err } diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go index 313c4291c79..7928e6ec252 100644 --- a/plugins/gitlab/impl/impl.go +++ b/plugins/gitlab/impl/impl.go @@ -33,7 +33,7 @@ import ( var _ core.PluginMeta = (*Gitlab)(nil) var _ core.PluginInit = (*Gitlab)(nil) -var _ core.PluginInfo = (*Gitlab)(nil) +var _ core.PluginModel = (*Gitlab)(nil) var _ core.PluginTask = (*Gitlab)(nil) var _ core.PluginApi = (*Gitlab)(nil) var _ core.Migratable = (*Gitlab)(nil) @@ -47,8 +47,8 @@ func (plugin Gitlab) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) return nil } -func (plugin Gitlab) RegisterTablesInfo(ms *core.TablesInfo) error { - ms.Adds( +func (plugin Gitlab) GetTablesInfo() []core.Tabler { + return []core.Tabler{ &models.GitlabConnection{}, &models.GitlabAccount{}, &models.GitlabCommit{}, @@ -66,8 +66,7 @@ func (plugin Gitlab) RegisterTablesInfo(ms *core.TablesInfo) error { // &models.GitlabResponse{}, &models.GitlabReviewer{}, &models.GitlabTag{}, - ) - return nil + } } func (plugin Gitlab) Description() string { diff --git a/runner/directrun.go b/runner/directrun.go index 93c5b2364a4..df88530dfaa 100644 --- a/runner/directrun.go +++ b/runner/directrun.go @@ -71,13 +71,6 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask core.PluginTask, op panic(err) } - if PluginInfo, ok := pluginTask.(core.PluginInfo); ok { - err := core.RegisterPluginInfo(PluginInfo) - if err != nil { - panic(err) - } - } - // collect migration and run migration.Init(db) if migratable, ok := pluginTask.(core.Migratable); ok { diff --git a/runner/loader.go b/runner/loader.go index 0d65c64e6fb..bcc6eb5bb9a 100644 --- a/runner/loader.go +++ b/runner/loader.go @@ -64,13 +64,6 @@ func LoadPlugins(pluginsDir string, config *viper.Viper, logger core.Logger, db return nil } - if PluginInfo, ok := symPluginEntry.(core.PluginInfo); ok { - err := core.RegisterPluginInfo(PluginInfo) - if err != nil { - return err - } - } - logger.Info(`plugin loaded %s`, pluginName) } return nil From bdc8ce0f9214c36bceb950673bbe38e6cf0815b6 Mon Sep 17 00:00:00 2001 From: Nddtfjiang Date: Thu, 11 Aug 2022 09:51:33 +0000 Subject: [PATCH 4/4] feat: add tables info for plugin remote api to show tables info at gitlab. add GetTablesInfo for all plugin Nddtfjiang --- plugins/ae/impl/impl.go | 8 ++++++ plugins/dbt/dbt.go | 4 +++ plugins/feishu/impl/impl.go | 7 +++++ plugins/gitee/impl/impl.go | 21 ++++++++++++++ plugins/gitextractor/gitextractor.go | 7 ++++- plugins/github/impl/impl.go | 28 +++++++++++++++++++ plugins/gitlab/api/connection.go | 34 ---------------------- plugins/gitlab/impl/impl.go | 4 --- plugins/icla/plugin_main.go | 6 ++++ plugins/jenkins/impl/impl.go | 14 ++++++++++ plugins/jira/impl/impl.go | 23 +++++++++++++++ plugins/org/impl/impl.go | 4 +++ plugins/refdiff/refdiff.go | 4 +++ plugins/starrocks/starrocks.go | 4 +++ plugins/tapd/impl/impl.go | 42 ++++++++++++++++++++++++++++ 15 files changed, 171 insertions(+), 39 deletions(-) diff --git a/plugins/ae/impl/impl.go b/plugins/ae/impl/impl.go index cfc747a4eb3..54bbee8fbff 100644 --- a/plugins/ae/impl/impl.go +++ b/plugins/ae/impl/impl.go @@ -45,6 +45,14 @@ func (plugin AE) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) erro api.Init(config, logger, db) return nil } +func (plugin AE) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.AECommit{}, + &models.AEProject{}, + &models.AeConnection{}, + &models.AeResponse{}, + } +} func (plugin AE) Description() string { return "To collect and enrich data from AE" diff --git a/plugins/dbt/dbt.go b/plugins/dbt/dbt.go index c3ce3d3e173..5fdbabd4dbf 100644 --- a/plugins/dbt/dbt.go +++ b/plugins/dbt/dbt.go @@ -42,6 +42,10 @@ func (plugin Dbt) SubTaskMetas() []core.SubTaskMeta { } } +func (plugin Dbt) GetTablesInfo() []core.Tabler { + return []core.Tabler{} +} + func (plugin Dbt) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) { var op tasks.DbtOptions err := mapstructure.Decode(options, &op) diff --git a/plugins/feishu/impl/impl.go b/plugins/feishu/impl/impl.go index e987077dc88..0b748cd4598 100644 --- a/plugins/feishu/impl/impl.go +++ b/plugins/feishu/impl/impl.go @@ -76,6 +76,13 @@ func (plugin Feishu) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) return nil } +func (plugin Feishu) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.FeishuConnection{}, + &models.FeishuMeetingTopUserItem{}, + } +} + func (plugin Feishu) Description() string { return "To collect and enrich data from Feishu" } diff --git a/plugins/gitee/impl/impl.go b/plugins/gitee/impl/impl.go index 44f8662a708..57bb4bae0a0 100644 --- a/plugins/gitee/impl/impl.go +++ b/plugins/gitee/impl/impl.go @@ -46,6 +46,27 @@ func (plugin Gitee) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) e return nil } +func (plugin Gitee) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.GiteeConnection{}, + &models.GiteeAccount{}, + &models.GiteeCommit{}, + &models.GiteeCommitStat{}, + &models.GiteeIssue{}, + &models.GiteeIssueComment{}, + &models.GiteeIssueLabel{}, + &models.GiteePullRequest{}, + &models.GiteePullRequestComment{}, + &models.GiteePullRequestCommit{}, + &models.GiteePullRequestIssue{}, + &models.GiteePullRequestLabel{}, + &models.GiteeRepo{}, + &models.GiteeRepoCommit{}, + &models.GiteeResponse{}, + &models.GiteeReviewer{}, + } +} + func (plugin Gitee) Description() string { return "To collect and enrich data from Gitee" } diff --git a/plugins/gitextractor/gitextractor.go b/plugins/gitextractor/gitextractor.go index 146e869da01..81686d99659 100644 --- a/plugins/gitextractor/gitextractor.go +++ b/plugins/gitextractor/gitextractor.go @@ -18,13 +18,14 @@ limitations under the License. package main import ( + "strings" + "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/gitextractor/models" "github.com/apache/incubator-devlake/plugins/gitextractor/parser" "github.com/apache/incubator-devlake/plugins/gitextractor/store" "github.com/apache/incubator-devlake/plugins/gitextractor/tasks" "github.com/mitchellh/mapstructure" - "strings" ) var _ core.PluginMeta = (*GitExtractor)(nil) @@ -32,6 +33,10 @@ var _ core.PluginTask = (*GitExtractor)(nil) type GitExtractor struct{} +func (plugin GitExtractor) GetTablesInfo() []core.Tabler { + return []core.Tabler{} +} + func (plugin GitExtractor) Description() string { return "extract infos from git repository" } diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go index 94dc18dea0b..92f88214ee0 100644 --- a/plugins/github/impl/impl.go +++ b/plugins/github/impl/impl.go @@ -46,6 +46,34 @@ func (plugin Github) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) return nil } +func (plugin Github) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.GithubConnection{}, + &models.GithubAccount{}, + &models.GithubAccountOrg{}, + &models.GithubCommit{}, + &models.GithubCommitStat{}, + &models.GithubIssue{}, + &models.GithubIssueComment{}, + &models.GithubIssueEvent{}, + &models.GithubIssueLabel{}, + &models.GithubJob{}, + &models.GithubMilestone{}, + &models.GithubPipeline{}, + &models.GithubPrComment{}, + &models.GithubPrCommit{}, + &models.GithubPrIssue{}, + &models.GithubPrLabel{}, + &models.GithubPrReview{}, + &models.GithubPullRequest{}, + &models.GithubRepo{}, + &models.GithubRepoAccount{}, + &models.GithubRepoCommit{}, + &models.GithubReviewer{}, + &models.GithubRun{}, + } +} + func (plugin Github) Description() string { return "To collect and enrich data from GitHub" } diff --git a/plugins/gitlab/api/connection.go b/plugins/gitlab/api/connection.go index 0b8d43c8bf7..3771bef45f1 100644 --- a/plugins/gitlab/api/connection.go +++ b/plugins/gitlab/api/connection.go @@ -21,13 +21,11 @@ import ( "context" "fmt" "net/http" - "reflect" "time" "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/apache/incubator-devlake/plugins/helper" - "github.com/apache/incubator-devlake/utils" "github.com/mitchellh/mapstructure" ) @@ -163,35 +161,3 @@ func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error err := connectionHelper.First(connection, input.Params) return &core.ApiResourceOutput{Body: connection}, err } - -// @Summary get table info -// @Description Get table info map -// @Tags plugins/gitlab -// @Success 200 -// @Router /plugins/gitlab/table [GET] -func GetTableInfo(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { - plugin, err := core.GetPlugin("gitlab") - if err != nil { - return nil, err - } - pm, ok := plugin.(core.PluginModel) - if !ok { - return nil, fmt.Errorf("plugin Gitlab can not change to PluginModel") - } - - tables := pm.GetTablesInfo() - - info := make(map[string]interface{}) - - for _, table := range tables { - sf := utils.WalkFields(reflect.TypeOf(table), nil) - tableInfo := make(map[string]string) - info[table.TableName()] = tableInfo - - for _, s := range sf { - tableInfo[s.Name] = string(s.Tag) - } - } - - return &core.ApiResourceOutput{Body: info}, err -} diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go index 7928e6ec252..514d47bb229 100644 --- a/plugins/gitlab/impl/impl.go +++ b/plugins/gitlab/impl/impl.go @@ -63,7 +63,6 @@ func (plugin Gitlab) GetTablesInfo() []core.Tabler { &models.GitlabPipeline{}, &models.GitlabProject{}, &models.GitlabProjectCommit{}, - // &models.GitlabResponse{}, &models.GitlabReviewer{}, &models.GitlabTag{}, } @@ -167,9 +166,6 @@ func (plugin Gitlab) ApiResources() map[string]map[string]core.ApiResourceHandle "connections/:connectionId/proxy/rest/*path": { "GET": api.Proxy, }, - "table": { - "GET": api.GetTableInfo, - }, } } diff --git a/plugins/icla/plugin_main.go b/plugins/icla/plugin_main.go index 57972adf636..5bfcaa76acc 100644 --- a/plugins/icla/plugin_main.go +++ b/plugins/icla/plugin_main.go @@ -42,6 +42,12 @@ var PluginEntry Icla //nolint type Icla struct{} +func (plugin Icla) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.IclaCommitter{}, + } +} + func (plugin Icla) Description() string { return "collect some Icla data" } diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go index 299a43a176c..677a8d076e2 100644 --- a/plugins/jenkins/impl/impl.go +++ b/plugins/jenkins/impl/impl.go @@ -45,6 +45,20 @@ func (plugin Jenkins) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) return nil } +func (plugin Jenkins) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.JenkinsBuild{}, + &models.JenkinsBuildRepo{}, + &models.JenkinsConnection{}, + &models.JenkinsJob{}, + &models.JenkinsJobDag{}, + &models.JenkinsPipeline{}, + &models.JenkinsResponse{}, + &models.JenkinsStage{}, + &models.JenkinsTask{}, + } +} + func (plugin Jenkins) Description() string { return "To collect and enrich data from Jenkins" } diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go index cedd9f29577..2c38afc9870 100644 --- a/plugins/jira/impl/impl.go +++ b/plugins/jira/impl/impl.go @@ -48,6 +48,29 @@ func (plugin Jira) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) er api.Init(config, logger, db) return nil } +func (plugin Jira) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.ApiMyselfResponse{}, + &models.JiraAccount{}, + &models.JiraBoard{}, + &models.JiraBoardIssue{}, + &models.JiraBoardSprint{}, + &models.JiraConnection{}, + &models.JiraIssue{}, + &models.JiraIssueChangelogItems{}, + &models.JiraIssueChangelogs{}, + &models.JiraIssueCommit{}, + &models.JiraIssueLabel{}, + &models.JiraIssueType{}, + &models.JiraProject{}, + &models.JiraRemotelink{}, + &models.JiraServerInfo{}, + &models.JiraSprint{}, + &models.JiraSprintIssue{}, + &models.JiraStatus{}, + &models.JiraWorklog{}, + } +} func (plugin Jira) Description() string { return "To collect and enrich data from JIRA" diff --git a/plugins/org/impl/impl.go b/plugins/org/impl/impl.go index f99ce1af5c5..daa272239ff 100644 --- a/plugins/org/impl/impl.go +++ b/plugins/org/impl/impl.go @@ -42,6 +42,10 @@ func (plugin *Org) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) er return nil } +func (plugin Org) GetTablesInfo() []core.Tabler { + return []core.Tabler{} +} + func (plugin Org) Description() string { return "collect data related to team and organization" } diff --git a/plugins/refdiff/refdiff.go b/plugins/refdiff/refdiff.go index 3409818296a..4e57b48dc93 100644 --- a/plugins/refdiff/refdiff.go +++ b/plugins/refdiff/refdiff.go @@ -43,6 +43,10 @@ func (plugin RefDiff) Description() string { return "Calculate commits diff for specified ref pairs based on `commits` and `commit_parents` tables" } +func (plugin RefDiff) GetTablesInfo() []core.Tabler { + return []core.Tabler{} +} + func (plugin RefDiff) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error { return nil } diff --git a/plugins/starrocks/starrocks.go b/plugins/starrocks/starrocks.go index 99f062b7f24..f22903de319 100644 --- a/plugins/starrocks/starrocks.go +++ b/plugins/starrocks/starrocks.go @@ -40,6 +40,10 @@ func (s StarRocks) PrepareTaskData(taskCtx core.TaskContext, options map[string] return &op, nil } +func (plugin StarRocks) GetTablesInfo() []core.Tabler { + return []core.Tabler{} +} + func (s StarRocks) Description() string { return "Sync data from database to StarRocks" } diff --git a/plugins/tapd/impl/impl.go b/plugins/tapd/impl/impl.go index 43df5152eda..fc26f21df18 100644 --- a/plugins/tapd/impl/impl.go +++ b/plugins/tapd/impl/impl.go @@ -49,6 +49,48 @@ func (plugin Tapd) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) er return nil } +func (plugin Tapd) GetTablesInfo() []core.Tabler { + return []core.Tabler{ + &models.TapdAccount{}, + &models.TapdBug{}, + &models.TapdBugChangelog{}, + &models.TapdBugChangelogItem{}, + &models.TapdBugCommit{}, + &models.TapdBugCustomFields{}, + &models.TapdBugLabel{}, + &models.TapdBugStatus{}, + &models.TapdConnection{}, + &models.TapdConnectionDetail{}, + &models.TapdIssue{}, + &models.TapdIteration{}, + &models.TapdIterationBug{}, + &models.TapdIterationStory{}, + &models.TapdIterationTask{}, + &models.TapdStory{}, + &models.TapdStoryBug{}, + &models.TapdStoryCategory{}, + &models.TapdStoryChangelog{}, + &models.TapdStoryChangelogItem{}, + &models.TapdStoryCommit{}, + &models.TapdStoryCustomFields{}, + &models.TapdStoryLabel{}, + &models.TapdStoryStatus{}, + &models.TapdSubWorkspace{}, + &models.TapdTask{}, + &models.TapdTaskChangelog{}, + &models.TapdTaskChangelogItem{}, + &models.TapdTaskCommit{}, + &models.TapdTaskCustomFields{}, + &models.TapdTaskLabel{}, + &models.TapdWorkSpaceBug{}, + &models.TapdWorkSpaceStory{}, + &models.TapdWorkSpaceTask{}, + &models.TapdWorklog{}, + &models.TapdWorkspace{}, + &models.TapdWorkspaceIteration{}, + } +} + func (plugin Tapd) Description() string { return "To collect and enrich data from Tapd" }