Skip to content

cherry-pick fix(zentao): remove execution's product stories to v0.18(beta7) #5944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abeizn merged 1 commit into from
Aug 23, 2023
Merged

cherry-pick fix(zentao): remove execution's product stories to v0.18(beta7) #5944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions backend/helpers/pluginhelper/api/api_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/apache/incubator-devlake/core/plugin"
"io"
"net/http"
"net/url"
Expand All @@ -30,10 +31,11 @@ import (

"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/common"
)

var _ plugin.SubTask = (*ApiCollector)(nil)

// Pager contains pagination information for a api request
type Pager struct {
Page int
Expand Down Expand Up @@ -241,7 +243,7 @@ func (collector *ApiCollector) exec(input interface{}) {
Page: 1,
Size: collector.args.PageSize,
}
// featch the detail
// fetch the detail
if collector.args.PageSize <= 0 {
collector.fetchAsync(reqData, nil)
// fetch pages sequentially
Expand Down Expand Up @@ -324,7 +326,7 @@ func (collector *ApiCollector) fetchPagesUndetermined(reqData *RequestData) {
concurrency := collector.args.Concurrency
if concurrency == 0 {
// normally when a multi-pages api depends on a another resource, like jira changelogs depend on issue ids
// it tend to have less page, like 1 or 2 pages in total
// it tends to have less page, like 1 or 2 pages in total
if collector.args.Input != nil {
concurrency = 2
} else {
Expand Down Expand Up @@ -484,5 +486,3 @@ func (collector *ApiCollector) fetchAsync(reqData *RequestData, handler func(int
}
logger.Debug("fetchAsync === enqueued for %s %v", apiUrl, apiQuery)
}

var _ plugin.SubTask = (*ApiCollector)(nil)
2 changes: 1 addition & 1 deletion backend/helpers/pluginhelper/api/data_convertor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type DataConvertHandler func(row interface{}) ([]interface{}, errors.Error)
// BatchSize: batch size
type DataConverterArgs struct {
RawDataSubTaskArgs
// Domain layer entity Id prefix, i.e. `jira:JiraIssue:1`, `github:GithubIssue`
// Domain layer entity ID prefix, i.e. `jira:JiraIssue:1`, `github:GithubIssue`
InputRowType reflect.Type
Input dal.Rows
Convert DataConvertHandler
Expand Down
1 change: 0 additions & 1 deletion backend/plugins/zentao/tasks/department_convertor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ func ConvertDepartment(taskCtx plugin.SubTaskContext) errors.Error {
},
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
toolEntity := inputRow.(*models.ZentaoDepartment)

domainEntity := &crossdomain.Team{
DomainEntity: domainlayer.DomainEntity{
Id: departmentIdGen.Generate(toolEntity.ConnectionId, toolEntity.ID),
Expand Down
4 changes: 3 additions & 1 deletion backend/plugins/zentao/tasks/iterator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ type iteratorConcator struct {
}

func newIteratorConcator(iterators ...api.Iterator) *iteratorConcator {
return &iteratorConcator{iterators: iterators}
return &iteratorConcator{
iterators: iterators,
}
}

func (w *iteratorConcator) HasNext() bool {
Expand Down
2 changes: 0 additions & 2 deletions backend/plugins/zentao/tasks/project_convertor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,7 @@ func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error {
},
Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
toolProject := inputRow.(*models.ZentaoProject)

data.ProjectName = toolProject.Name

domainBoard := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
Id: boardIdGen.Generate(toolProject.ConnectionId, toolProject.Id),
Expand Down
42 changes: 20 additions & 22 deletions backend/plugins/zentao/tasks/story_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ const RAW_STORY_TABLE = "zentao_api_stories"

var _ plugin.SubTaskEntryPoint = CollectStory

var CollectStoryMeta = plugin.SubTaskMeta{
Name: "collectStory",
EntryPoint: CollectStory,
EnabledByDefault: true,
Description: "Collect Story data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}

type storyInput struct {
Path string
ProjectId int64
Expand All @@ -42,16 +50,11 @@ type storyInput struct {
func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
// project iterator
iter0 := newIteratorFromSlice([]interface{}{&storyInput{ProjectId: data.Options.ProjectId, Path: fmt.Sprintf("/projects/%d", data.Options.ProjectId)}})

// product iterator
productCursor, productIterator, err := getProductIterator(taskCtx)
if err != nil {
return err
}
defer productCursor.Close()
iter1 := newIteratorWrapper(productIterator, func(arg interface{}) interface{} {
return &storyInput{ProductId: arg.(*input).Id, Path: fmt.Sprintf("/products/%d", arg.(*input).Id)}
projectStoryIter := newIteratorFromSlice([]interface{}{
&storyInput{
ProjectId: data.Options.ProjectId,
Path: fmt.Sprintf("/projects/%d", data.Options.ProjectId),
},
})

// execution iterator
Expand All @@ -60,8 +63,11 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {
return err
}
defer executionCursor.Close()
iter2 := newIteratorWrapper(executionIterator, func(arg interface{}) interface{} {
return &storyInput{ExecutionId: arg.(*input).Id, Path: fmt.Sprintf("/executions/%d", arg.(*input).Id)}
executionStoryIter := newIteratorWrapper(executionIterator, func(arg interface{}) interface{} {
return &storyInput{
ExecutionId: arg.(*input).Id,
Path: fmt.Sprintf("/executions/%d", arg.(*input).Id),
}
})

collector, err := api.NewApiCollector(api.ApiCollectorArgs{
Expand All @@ -70,7 +76,7 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {
Options: data.Options,
Table: RAW_STORY_TABLE,
},
Input: newIteratorConcator(iter0, iter1, iter2),
Input: newIteratorConcator(projectStoryIter, executionStoryIter),
ApiClient: data.ApiClient,
PageSize: 100,
UrlTemplate: "{{ .Input.Path }}/stories",
Expand All @@ -91,7 +97,7 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {
return nil, nil
}
if err != nil {
return nil, errors.Default.Wrap(err, "error reading endpoint response by Zentao bug collector")
return nil, errors.Default.Wrap(err, "error reading endpoint response by Zentao story collector")
}
return data.Story, nil
},
Expand All @@ -102,11 +108,3 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {

return collector.Execute()
}

var CollectStoryMeta = plugin.SubTaskMeta{
Name: "collectStory",
EntryPoint: CollectStory,
EnabledByDefault: true,
Description: "Collect Story data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}