Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions generator/cmd/create_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,11 @@ Type in what the name of collector is, then generator will create a new collecto
util.ReplaceVarInTemplates(templates, values)
util.WriteTemplates(filepath.Join(`plugins`, pluginName, `tasks`), templates)
if modifyExistCode {
util.ReplaceVarInFile(
filepath.Join(`plugins`, pluginName, `impl/impl.go`),
regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
fmt.Sprintf("$1$2\t\ttasks.Collect%sMeta,\n$4", collectorDataNameUpperCamel),
)
util.ReplaceVarInFile(
filepath.Join(`plugins`, pluginName, `plugin_main.go`),
regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
Expand Down
5 changes: 5 additions & 0 deletions generator/cmd/create_extractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ Type in what the name of extractor is, then generator will create a new extracto
regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
fmt.Sprintf("$1$2\t\ttasks.Extract%sMeta,\n$4", extractorDataNameUpperCamel),
)
util.ReplaceVarInFile(
filepath.Join(`plugins`, pluginName, `impl/impl.go`),
regexp.MustCompile(`(return +\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
fmt.Sprintf("$1$2\t\ttasks.Extract%sMeta,\n$4", extractorDataNameUpperCamel),
)
}
},
}
35 changes: 14 additions & 21 deletions generator/cmd/create_plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"os"
"regexp"
"strings"
"time"
)

func init() {
Expand Down Expand Up @@ -103,7 +104,7 @@ Type in what the name of plugin is, then generator will create a new plugin in p
}

prompt := promptui.Select{
Label: "with_api_client (Will this plugin request HTTP APIs?)",
Label: "complete_plugin (Will this plugin request HTTP APIs?)",
Items: []string{"Yes", "No"},
}
_, withApiClient, err := prompt.Run()
Expand All @@ -112,30 +113,22 @@ Type in what the name of plugin is, then generator will create a new plugin in p
values := map[string]string{}
templates := map[string]string{}
if withApiClient == `Yes` {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

change withApiClient to generateCompletePlugin or other appropriate name?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good idea

prompt := promptui.Prompt{
Label: "Endpoint (which host to request)",
Default: `https://open.example.cn/api/v1`,
Validate: func(input string) error {
if input == `` {
return errors.New("endpoint require")
}
if !strings.HasPrefix(input, `http`) {
return errors.New("endpoint should start with http")
}
return nil
},
}
endpoint, err := prompt.Run()
cobra.CheckErr(err)

versionTimestamp := time.Now().Format(`20060102`)
values[`Date`] = versionTimestamp
// read template
templates = map[string]string{
`plugin_main.go`: util.ReadTemplate("generator/template/plugin/plugin_main_with_api_client.go-template"),
`tasks/api_client.go`: util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"),
`tasks/task_data.go`: util.ReadTemplate("generator/template/plugin/tasks/task_data_with_api_client.go-template"),
fmt.Sprintf(`%s.go`, pluginName): util.ReadTemplate("generator/template/plugin/plugin_main_complete_plugin.go-template"),
`impl/impl.go`: util.ReadTemplate("generator/template/plugin/impl/impl_complete_plugin.go-template"),
`tasks/api_client.go`: util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"),
`tasks/task_data.go`: util.ReadTemplate("generator/template/plugin/tasks/task_data_complete_plugin.go-template"),
`api/connection.go`: util.ReadTemplate("generator/template/plugin/api/connection.go-template"),
`models/connection.go`: util.ReadTemplate("generator/template/plugin/models/connection.go-template"),
fmt.Sprintf("models/migrationscripts/%s_add_init_tables.go", versionTimestamp): util.ReadTemplate("generator/template/migrationscripts/add_init_tables.go-template"),
`models/migrationscripts/register.go`: util.ReadTemplate("generator/template/migrationscripts/register.go-template"),
`api/init.go`: util.ReadTemplate("generator/template/plugin/api/init.go-template"),
`api/blueprint.go`: util.ReadTemplate("generator/template/plugin/api/blueprint.go-template"),
}
util.GenerateAllFormatVar(values, `plugin_name`, pluginName)
values[`Endpoint`] = endpoint
} else if withApiClient == `No` {
// read template
templates = map[string]string{
Expand Down
Empty file.
69 changes: 69 additions & 0 deletions generator/template/plugin/api/blueprint.go-template
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file seems not used.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
"encoding/json"

"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks"
)

func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
var err error
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
taskOptions := make(map[string]interface{})
err = json.Unmarshal(scopeElem.Options, &taskOptions)
if err != nil {
return nil, err
}
taskOptions["connectionId"] = connectionId

//TODO Add transformation rules to task options

/*
var transformationRules tasks.TransformationRules
if len(scopeElem.Transformation) > 0 {
err = json.Unmarshal(scopeElem.Transformation, &transformationRules)
if err != nil {
return nil, err
}
}
*/
//taskOptions["transformationRules"] = transformationRules
_, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
if err != nil {
return nil, err
}
// subtasks
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
if err != nil {
return nil, err
}
plan[i] = core.PipelineStage{
{
Plugin: "{{ .plugin_name }}",
Subtasks: subtasks,
Options: taskOptions,
},
}
}
return plan, nil
}
155 changes: 155 additions & 0 deletions generator/template/plugin/api/connection.go-template
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
"context"
"fmt"
"net/http"
"time"

"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
"github.com/mitchellh/mapstructure"
)

//TODO Please modify the following code to fit your needs
func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
// decode
var err error
var connection models.TestConnectionRequest
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is TestConnectionRequest ?

err = mapstructure.Decode(input.Body, &connection)
if err != nil {
return nil, err
}
// validate
err = vld.Struct(connection)
if err != nil {
return nil, err
}
// test connection
apiClient, err := helper.NewApiClient(
context.TODO(),
connection.Endpoint,
map[string]string{
"Authorization": fmt.Sprintf("Bearer %v", connection.Token),
},
3*time.Second,
connection.Proxy,
basicRes,
)
if err != nil {
return nil, err
}

res, err := apiClient.Get("user", nil, nil)
if err != nil {
return nil, err
}
resBody := &models.ApiUserResponse{}
err = helper.UnmarshalResponse(res, resBody)
if err != nil {
return nil, err
}

if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
return nil, nil
}

//TODO Please modify the folowing code to adapt to your plugin
/*
POST /plugins/{{ .PluginName }}/connections
{
"name": "{{ .PluginName }} data connection name",
"endpoint": "{{ .PluginName }} api endpoint, i.e. https://example.com",
"username": "username, usually should be email address",
"password": "{{ .PluginName }} api access token"
}
*/
func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
// update from request and save to database
connection := &models.{{ .PluginName }}Connection{}
err := connectionHelper.Create(connection, input)
if err != nil {
return nil, err
}
return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil
}

//TODO Please modify the folowing code to adapt to your plugin
/*
PATCH /plugins/{{ .PluginName }}/connections/:connectionId
{
"name": "{{ .PluginName }} data connection name",
"endpoint": "{{ .PluginName }} api endpoint, i.e. https://example.com",
"username": "username, usually should be email address",
"password": "{{ .PluginName }} api access token"
}
*/
func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
connection := &models.{{ .PluginName }}Connection{}
err := connectionHelper.Patch(connection, input)
if err != nil {
return nil, err
}
return &core.ApiResourceOutput{Body: connection}, nil
}

/*
DELETE /plugins/{{ .PluginName }}/connections/:connectionId
*/
func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
connection := &models.{{ .PluginName }}Connection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
err = connectionHelper.Delete(connection)
return &core.ApiResourceOutput{Body: connection}, err
}

/*
GET /plugins/{{ .PluginName }}/connections
*/
func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
var connections []models.{{ .PluginName }}Connection
err := connectionHelper.List(&connections)
if err != nil {
return nil, err
}
return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
}

//TODO Please modify the folowing code to adapt to your plugin
/*
GET /plugins/{{ .PluginName }}/connections/:connectionId
{
"name": "{{ .PluginName }} data connection name",
"endpoint": "{{ .PluginName }} api endpoint, i.e. https://merico.atlassian.net/rest",
"username": "username, usually should be email address",
"password": "{{ .PluginName }} api access token"
}
*/
func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) {
connection := &models.{{ .PluginName }}Connection{}
err := connectionHelper.First(connection, input.Params)
return &core.ApiResourceOutput{Body: connection}, err
}
39 changes: 39 additions & 0 deletions generator/template/plugin/api/init.go-template
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/go-playground/validator/v10"
"github.com/spf13/viper"
"gorm.io/gorm"
)

var vld *validator.Validate
var connectionHelper *helper.ConnectionApiHelper
var basicRes core.BasicRes

func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
basicRes = helper.NewDefaultBasicRes(config, logger, database)
vld = validator.New()
connectionHelper = helper.NewConnectionHelper(
basicRes,
vld,
)
}
Loading