From 46713da0fcf3e4e8ba15b4aaa7d5b5f4cb9dd200 Mon Sep 17 00:00:00 2001 From: Yingchu Chen Date: Sat, 30 Jul 2022 15:32:18 +0800 Subject: [PATCH] fix(framework): update code generator templates closes #2640 --- generator/cmd/create_collector.go | 5 + generator/cmd/create_extractor.go | 5 + generator/cmd/create_plugin.go | 35 ++-- .../archived/connection.go-template | 0 .../template/plugin/api/blueprint.go-template | 69 ++++++++ .../plugin/api/connection.go-template | 155 ++++++++++++++++++ .../template/plugin/api/init.go-template | 39 +++++ .../impl/impl_complete_plugin.go-template | 123 ++++++++++++++ .../plugin/models/connection.go-template | 51 ++++++ .../plugin_main_complete_plugin.go-template | 43 +++++ .../plugin_main_with_api_client.go-template | 100 ----------- .../plugin/tasks/api_client.go-template | 71 ++++---- ... => task_data_complete_plugin.go-template} | 18 ++ 13 files changed, 558 insertions(+), 156 deletions(-) create mode 100644 generator/template/migrationscripts/archived/connection.go-template create mode 100644 generator/template/plugin/api/blueprint.go-template create mode 100644 generator/template/plugin/api/connection.go-template create mode 100644 generator/template/plugin/api/init.go-template create mode 100644 generator/template/plugin/impl/impl_complete_plugin.go-template create mode 100644 generator/template/plugin/models/connection.go-template create mode 100644 generator/template/plugin/plugin_main_complete_plugin.go-template delete mode 100644 generator/template/plugin/plugin_main_with_api_client.go-template rename generator/template/plugin/tasks/{task_data_with_api_client.go-template => task_data_complete_plugin.go-template} (70%) diff --git a/generator/cmd/create_collector.go b/generator/cmd/create_collector.go index 6c5f3d78703..bef7385f56c 100644 --- a/generator/cmd/create_collector.go +++ b/generator/cmd/create_collector.go @@ -134,6 +134,11 @@ Type in what the name of collector is, then generator will create a new collecto util.ReplaceVarInTemplates(templates, values) util.WriteTemplates(filepath.Join(`plugins`, pluginName, `tasks`), templates) if modifyExistCode { + util.ReplaceVarInFile( + filepath.Join(`plugins`, pluginName, `impl/impl.go`), + regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`), + fmt.Sprintf("$1$2\t\ttasks.Collect%sMeta,\n$4", collectorDataNameUpperCamel), + ) util.ReplaceVarInFile( filepath.Join(`plugins`, pluginName, `plugin_main.go`), regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`), diff --git a/generator/cmd/create_extractor.go b/generator/cmd/create_extractor.go index b63ac6a600f..d16c3d7fbd2 100644 --- a/generator/cmd/create_extractor.go +++ b/generator/cmd/create_extractor.go @@ -121,6 +121,11 @@ Type in what the name of extractor is, then generator will create a new extracto regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`), fmt.Sprintf("$1$2\t\ttasks.Extract%sMeta,\n$4", extractorDataNameUpperCamel), ) + util.ReplaceVarInFile( + filepath.Join(`plugins`, pluginName, `impl/impl.go`), + regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`), + fmt.Sprintf("$1$2\t\ttasks.Extract%sMeta,\n$4", extractorDataNameUpperCamel), + ) } }, } diff --git a/generator/cmd/create_plugin.go b/generator/cmd/create_plugin.go index 59a1ad2f5e3..23b9d02b99a 100644 --- a/generator/cmd/create_plugin.go +++ b/generator/cmd/create_plugin.go @@ -27,6 +27,7 @@ import ( "os" "regexp" "strings" + "time" ) func init() { @@ -103,7 +104,7 @@ Type in what the name of plugin is, then generator will create a new plugin in p } prompt := promptui.Select{ - Label: "with_api_client (Will this plugin request HTTP APIs?)", + Label: "complete_plugin (Will this plugin request HTTP APIs?)", Items: []string{"Yes", "No"}, } _, withApiClient, err := prompt.Run() @@ -112,30 +113,22 @@ Type in what the name of plugin is, then generator will create a new plugin in p values := map[string]string{} templates := map[string]string{} if withApiClient == `Yes` { - prompt := promptui.Prompt{ - Label: "Endpoint (which host to request)", - Default: `https://open.example.cn/api/v1`, - Validate: func(input string) error { - if input == `` { - return errors.New("endpoint require") - } - if !strings.HasPrefix(input, `http`) { - return errors.New("endpoint should start with http") - } - return nil - }, - } - endpoint, err := prompt.Run() - cobra.CheckErr(err) - + versionTimestamp := time.Now().Format(`20060102`) + values[`Date`] = versionTimestamp // read template templates = map[string]string{ - `plugin_main.go`: util.ReadTemplate("generator/template/plugin/plugin_main_with_api_client.go-template"), - `tasks/api_client.go`: util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"), - `tasks/task_data.go`: util.ReadTemplate("generator/template/plugin/tasks/task_data_with_api_client.go-template"), + fmt.Sprintf(`%s.go`, pluginName): util.ReadTemplate("generator/template/plugin/plugin_main_complete_plugin.go-template"), + `impl/impl.go`: util.ReadTemplate("generator/template/plugin/impl/impl_complete_plugin.go-template"), + `tasks/api_client.go`: util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"), + `tasks/task_data.go`: util.ReadTemplate("generator/template/plugin/tasks/task_data_complete_plugin.go-template"), + `api/connection.go`: util.ReadTemplate("generator/template/plugin/api/connection.go-template"), + `models/connection.go`: util.ReadTemplate("generator/template/plugin/models/connection.go-template"), + fmt.Sprintf("models/migrationscripts/%s_add_init_tables.go", versionTimestamp): util.ReadTemplate("generator/template/migrationscripts/add_init_tables.go-template"), + `models/migrationscripts/register.go`: util.ReadTemplate("generator/template/migrationscripts/register.go-template"), + `api/init.go`: util.ReadTemplate("generator/template/plugin/api/init.go-template"), + `api/blueprint.go`: util.ReadTemplate("generator/template/plugin/api/blueprint.go-template"), } util.GenerateAllFormatVar(values, `plugin_name`, pluginName) - values[`Endpoint`] = endpoint } else if withApiClient == `No` { // read template templates = map[string]string{ diff --git a/generator/template/migrationscripts/archived/connection.go-template b/generator/template/migrationscripts/archived/connection.go-template new file mode 100644 index 00000000000..e69de29bb2d diff --git a/generator/template/plugin/api/blueprint.go-template b/generator/template/plugin/api/blueprint.go-template new file mode 100644 index 00000000000..cdf40067c59 --- /dev/null +++ b/generator/template/plugin/api/blueprint.go-template @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks" +) + +func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) { + var err error + plan := make(core.PipelinePlan, len(scope)) + for i, scopeElem := range scope { + taskOptions := make(map[string]interface{}) + err = json.Unmarshal(scopeElem.Options, &taskOptions) + if err != nil { + return nil, err + } + taskOptions["connectionId"] = connectionId + + //TODO Add transformation rules to task options + + /* + var transformationRules tasks.TransformationRules + if len(scopeElem.Transformation) > 0 { + err = json.Unmarshal(scopeElem.Transformation, &transformationRules) + if err != nil { + return nil, err + } + } + */ + //taskOptions["transformationRules"] = transformationRules + _, err := tasks.DecodeAndValidateTaskOptions(taskOptions) + if err != nil { + return nil, err + } + // subtasks + subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities) + if err != nil { + return nil, err + } + plan[i] = core.PipelineStage{ + { + Plugin: "{{ .plugin_name }}", + Subtasks: subtasks, + Options: taskOptions, + }, + } + } + return plan, nil +} diff --git a/generator/template/plugin/api/connection.go-template b/generator/template/plugin/api/connection.go-template new file mode 100644 index 00000000000..0e781d2789f --- /dev/null +++ b/generator/template/plugin/api/connection.go-template @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models" + "github.com/mitchellh/mapstructure" +) + +//TODO Please modify the following code to fit your needs +func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + // decode + var err error + var connection models.TestConnectionRequest + err = mapstructure.Decode(input.Body, &connection) + if err != nil { + return nil, err + } + // validate + err = vld.Struct(connection) + if err != nil { + return nil, err + } + // test connection + apiClient, err := helper.NewApiClient( + context.TODO(), + connection.Endpoint, + map[string]string{ + "Authorization": fmt.Sprintf("Bearer %v", connection.Token), + }, + 3*time.Second, + connection.Proxy, + basicRes, + ) + if err != nil { + return nil, err + } + + res, err := apiClient.Get("user", nil, nil) + if err != nil { + return nil, err + } + resBody := &models.ApiUserResponse{} + err = helper.UnmarshalResponse(res, resBody) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + return nil, nil +} + +//TODO Please modify the folowing code to adapt to your plugin +/* +POST /plugins/{{ .PluginName }}/connections +{ + "name": "{{ .PluginName }} data connection name", + "endpoint": "{{ .PluginName }} api endpoint, i.e. https://example.com", + "username": "username, usually should be email address", + "password": "{{ .PluginName }} api access token" +} +*/ +func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + // update from request and save to database + connection := &models.{{ .PluginName }}Connection{} + err := connectionHelper.Create(connection, input) + if err != nil { + return nil, err + } + return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil +} + +//TODO Please modify the folowing code to adapt to your plugin +/* +PATCH /plugins/{{ .PluginName }}/connections/:connectionId +{ + "name": "{{ .PluginName }} data connection name", + "endpoint": "{{ .PluginName }} api endpoint, i.e. https://example.com", + "username": "username, usually should be email address", + "password": "{{ .PluginName }} api access token" +} +*/ +func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + connection := &models.{{ .PluginName }}Connection{} + err := connectionHelper.Patch(connection, input) + if err != nil { + return nil, err + } + return &core.ApiResourceOutput{Body: connection}, nil +} + +/* +DELETE /plugins/{{ .PluginName }}/connections/:connectionId +*/ +func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + connection := &models.{{ .PluginName }}Connection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + err = connectionHelper.Delete(connection) + return &core.ApiResourceOutput{Body: connection}, err +} + +/* +GET /plugins/{{ .PluginName }}/connections +*/ +func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + var connections []models.{{ .PluginName }}Connection + err := connectionHelper.List(&connections) + if err != nil { + return nil, err + } + return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil +} + +//TODO Please modify the folowing code to adapt to your plugin +/* +GET /plugins/{{ .PluginName }}/connections/:connectionId +{ + "name": "{{ .PluginName }} data connection name", + "endpoint": "{{ .PluginName }} api endpoint, i.e. https://merico.atlassian.net/rest", + "username": "username, usually should be email address", + "password": "{{ .PluginName }} api access token" +} +*/ +func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + connection := &models.{{ .PluginName }}Connection{} + err := connectionHelper.First(connection, input.Params) + return &core.ApiResourceOutput{Body: connection}, err +} \ No newline at end of file diff --git a/generator/template/plugin/api/init.go-template b/generator/template/plugin/api/init.go-template new file mode 100644 index 00000000000..6774e148218 --- /dev/null +++ b/generator/template/plugin/api/init.go-template @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/go-playground/validator/v10" + "github.com/spf13/viper" + "gorm.io/gorm" +) + +var vld *validator.Validate +var connectionHelper *helper.ConnectionApiHelper +var basicRes core.BasicRes + +func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) { + basicRes = helper.NewDefaultBasicRes(config, logger, database) + vld = validator.New() + connectionHelper = helper.NewConnectionHelper( + basicRes, + vld, + ) +} diff --git a/generator/template/plugin/impl/impl_complete_plugin.go-template b/generator/template/plugin/impl/impl_complete_plugin.go-template new file mode 100644 index 00000000000..8258b138111 --- /dev/null +++ b/generator/template/plugin/impl/impl_complete_plugin.go-template @@ -0,0 +1,123 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "github.com/apache/incubator-devlake/migration" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/api" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/spf13/viper" + "gorm.io/gorm" +) + +// make sure interface is implemented +var _ core.PluginMeta = (*{{ .PluginName }})(nil) +var _ core.PluginInit = (*{{ .PluginName }})(nil) +var _ core.PluginTask = (*{{ .PluginName }})(nil) +var _ core.PluginApi = (*{{ .PluginName }})(nil) +var _ core.PluginBlueprintV100 = (*{{ .PluginName }})(nil) +var _ core.CloseablePluginTask = (*{{ .PluginName }})(nil) + + + +type {{ .PluginName }} struct{} + +func (plugin {{ .PluginName }}) Description() string { + return "collect some {{ .PluginName }} data" +} + +func (plugin {{ .PluginName }}) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error { + api.Init(config, logger, db) + return nil +} + +func (plugin {{ .PluginName }}) SubTaskMetas() []core.SubTaskMeta { + // TODO add your sub task here + return []core.SubTaskMeta{ + } +} + +func (plugin {{ .PluginName }}) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + ) + connection := &models.{{ .PluginName }}Connection{} + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, fmt.Errorf("unable to get {{ .PluginName }} connection by the given connection ID: %v", err) + } + + apiClient, err := tasks.New{{ .PluginName }}ApiClient(taskCtx, connection) + if err != nil { + return nil, fmt.Errorf("unable to get {{ .PluginName }} API client instance: %v", err) + } + + return &tasks.{{ .PluginName }}TaskData{ + Options: op, + ApiClient: apiClient, + }, nil +} + +// PkgPath information lost when compiled as plugin(.so) +func (plugin {{ .PluginName }}) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}" +} + +func (plugin {{ .PluginName }}) MigrationScripts() []migration.Script { + return migrationscripts.All() +} + +func (plugin {{ .PluginName }}) ApiResources() map[string]map[string]core.ApiResourceHandler { + return map[string]map[string]core.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + } +} + +func (plugin {{ .PluginName }}) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) { + return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope) +} + +func (plugin {{ .PluginName }}) Close(taskCtx core.TaskContext) error { + data, ok := taskCtx.GetData().(*tasks.{{ .PluginName }}TaskData) + if !ok { + return fmt.Errorf("GetData failed when try to close %+v", taskCtx) + } + data.ApiClient.Release() + return nil +} diff --git a/generator/template/plugin/models/connection.go-template b/generator/template/plugin/models/connection.go-template new file mode 100644 index 00000000000..72133f1036c --- /dev/null +++ b/generator/template/plugin/models/connection.go-template @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/plugins/helper" + +//TODO Please modify the following code to fit your needs +// This object conforms to what the frontend currently sends. +type {{ .PluginName }}Connection struct { + helper.RestConnection `mapstructure:",squash"` + //TODO you may need to use helper.BasicAuth instead of helper.AccessToken + helper.AccessToken `mapstructure:",squash"` +} + +type TestConnectionRequest struct { + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy"` + helper.AccessToken `mapstructure:",squash"` +} + +// This object conforms to what the frontend currently expects. +type {{ .PluginName }}Response struct { + Name string `json:"name"` + ID int `json:"id"` + {{ .PluginName }}Connection +} + +// Using User because it requires authentication. +type ApiUserResponse struct { + Id int + Name string `json:"name"` +} + +func ({{ .PluginName }}Connection) TableName() string { + return "_tool_{{ .plugin_name }}_connections" +} diff --git a/generator/template/plugin/plugin_main_complete_plugin.go-template b/generator/template/plugin/plugin_main_complete_plugin.go-template new file mode 100644 index 00000000000..ee2d4a564dc --- /dev/null +++ b/generator/template/plugin/plugin_main_complete_plugin.go-template @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/plugins/{{ .PluginName }}/impl" + "github.com/apache/incubator-devlake/runner" + "github.com/spf13/cobra" +) + +// Export a variable named PluginEntry for Framework to search and load +var PluginEntry impl.{{ .PluginName }} //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "{{ .pluginName }}"} + + // TODO add your cmd flag if necessary + // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add description here") + // _ = cmd.MarkFlagRequired("yourFlag") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + // TODO add more custom params here + }) + } + runner.RunCmd(cmd) +} diff --git a/generator/template/plugin/plugin_main_with_api_client.go-template b/generator/template/plugin/plugin_main_with_api_client.go-template deleted file mode 100644 index 1ab8333dd34..00000000000 --- a/generator/template/plugin/plugin_main_with_api_client.go-template +++ /dev/null @@ -1,100 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "github.com/apache/incubator-devlake/plugins/core" - "github.com/apache/incubator-devlake/plugins/{{ .pluginName }}/tasks" - "github.com/apache/incubator-devlake/runner" - "github.com/mitchellh/mapstructure" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "gorm.io/gorm" -) - -// make sure interface is implemented -var _ core.PluginMeta = (*{{ .PluginName }})(nil) -var _ core.PluginInit = (*{{ .PluginName }})(nil) -var _ core.PluginTask = (*{{ .PluginName }})(nil) -var _ core.PluginApi = (*{{ .PluginName }})(nil) - -// Export a variable named PluginEntry for Framework to search and load -var PluginEntry {{ .PluginName }} //nolint - -type {{ .PluginName }} struct{} - -func (plugin {{ .PluginName }}) Description() string { - return "collect some {{ .PluginName }} data" -} - -func (plugin {{ .PluginName }}) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error { - // AutoSchemas is a **develop** script to auto migrate models easily. - // FIXME Don't submit it as a open source plugin - return db.Migrator().AutoMigrate( - // TODO add your models in here - ) -} - -func (plugin {{ .PluginName }}) SubTaskMetas() []core.SubTaskMeta { - // TODO add your sub task here - return []core.SubTaskMeta{ - } -} - -func (plugin {{ .PluginName }}) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) { - var op tasks.{{ .PluginName }}Options - err := mapstructure.Decode(options, &op) - if err != nil { - return nil, err - } - - apiClient, err := tasks.New{{ .PluginName }}ApiClient(taskCtx) - if err != nil { - return nil, err - } - - return &tasks.{{ .PluginName }}TaskData{ - Options: &op, - ApiClient: apiClient, - }, nil -} - -// PkgPath information lost when compiled as plugin(.so) -func (plugin {{ .PluginName }}) RootPkgPath() string { - return "github.com/apache/incubator-devlake/plugins/{{ .pluginName }}" -} - -func (plugin {{ .PluginName }}) ApiResources() map[string]map[string]core.ApiResourceHandler { - return nil -} - -// standalone mode for debugging -func main() { - cmd := &cobra.Command{Use: "{{ .pluginName }}"} - - // TODO add your cmd flag if necessary - // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add description here") - // _ = cmd.MarkFlagRequired("yourFlag") - - cmd.Run = func(cmd *cobra.Command, args []string) { - runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ - // TODO add more custom params here - }) - } - runner.RunCmd(cmd) -} diff --git a/generator/template/plugin/tasks/api_client.go-template b/generator/template/plugin/tasks/api_client.go-template index a79e1312e79..296131fb96f 100644 --- a/generator/template/plugin/tasks/api_client.go-template +++ b/generator/template/plugin/tasks/api_client.go-template @@ -19,54 +19,55 @@ package tasks import ( "fmt" + "net/http" + "strconv" + "time" + "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models" "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/helper" - "github.com/apache/incubator-devlake/utils" ) -// TODO add what host would want to requist -const ENDPOINT = "{{ .Endpoint }}" - -func New{{ .PluginName }}ApiClient(taskCtx core.TaskContext) (*helper.ApiAsyncClient, error) { - // load and process configuration - token := taskCtx.GetConfig("{{ .PLUGIN_NAME }}_TOKEN") - if token == "" { - println("invalid {{ .PLUGIN_NAME }}_TOKEN, but ignore this error now") +func New{{ .PluginName }}ApiClient(taskCtx core.TaskContext, connection *models.{{ .PluginName }}Connection) (*helper.ApiAsyncClient, error) { + // create synchronize api client so we can calculate api rate limit dynamically + headers := map[string]string{ + "Authorization": fmt.Sprintf("Bearer %v", connection.Token), } - userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("{{ .PLUGIN_NAME }}_API_REQUESTS_PER_HOUR"), 18000) + apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, headers, 0, connection.Proxy, taskCtx) if err != nil { return nil, err } - proxy := taskCtx.GetConfig("{{ .PLUGIN_NAME }}_PROXY") + apiClient.SetAfterFunction(func(res *http.Response) error { + if res.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("authentication failed, please check your AccessToken") + } + return nil + }) - // real request apiClient - apiClient, err := helper.NewApiClient(ENDPOINT, nil, 0, proxy, taskCtx.GetContext()) - if err != nil { - return nil, err - } - // set token - if token != "" { - apiClient.SetHeaders(map[string]string{ - "Authorization": fmt.Sprintf("Bearer %v", token), - }) + // create rate limit calculator + rateLimiter := &helper.ApiRateLimitCalculator{ + UserRateLimitPerHour: connection.RateLimitPerHour, + DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) { + rateLimitHeader := res.Header.Get("RateLimit-Limit") + if rateLimitHeader == "" { + // use default + return 0, 0, nil + } + rateLimit, err := strconv.Atoi(rateLimitHeader) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse RateLimit-Limit header: %w", err) + } + // seems like {{ .plugin-ame }} rate limit is on minute basis + return rateLimit, 1 * time.Minute, nil + }, } - - // TODO add some check after request if necessary - // apiClient.SetAfterFunction(func(res *http.Response) error { - // if res.StatusCode == http.StatusUnauthorized { - // return fmt.Errorf("authentication failed, please check your Bearer Auth Token") - // } - // return nil - // }) - - // create async api client - asyncApiClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient, &helper.ApiRateLimitCalculator{ - UserRateLimitPerHour: userRateLimit, - }) + asyncApiClient, err := helper.CreateAsyncApiClient( + taskCtx, + apiClient, + rateLimiter, + ) if err != nil { return nil, err } - return asyncApiClient, nil } diff --git a/generator/template/plugin/tasks/task_data_with_api_client.go-template b/generator/template/plugin/tasks/task_data_complete_plugin.go-template similarity index 70% rename from generator/template/plugin/tasks/task_data_with_api_client.go-template rename to generator/template/plugin/tasks/task_data_complete_plugin.go-template index 5dec766587a..3284d2496a6 100644 --- a/generator/template/plugin/tasks/task_data_with_api_client.go-template +++ b/generator/template/plugin/tasks/task_data_complete_plugin.go-template @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "fmt" + "github.com/mitchellh/mapstructure" "github.com/apache/incubator-devlake/plugins/helper" ) @@ -29,9 +31,25 @@ type {{ .PluginName }}Options struct { // options means some custom params required by plugin running. // Such As How many rows do your want // You can use it in sub tasks and you need pass it in main.go and pipelines. + ConnectionId uint64 `json:"connectionId"` + Tasks []string `json:"tasks,omitempty"` + Since string } type {{ .PluginName }}TaskData struct { Options *{{ .PluginName }}Options ApiClient *helper.ApiAsyncClient } + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*{{ .PluginName }}Options, error) { + var op {{ .PluginName }}Options + err := mapstructure.Decode(options, &op) + if err != nil { + return nil, err + } + + if op.ConnectionId == 0 { + return nil, fmt.Errorf("connectionId is invalid") + } + return &op, nil +} \ No newline at end of file