diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index a45891e9976..686b99816b0 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -57,6 +57,7 @@ import ( taiga "github.com/apache/incubator-devlake/plugins/taiga/impl" tapd "github.com/apache/incubator-devlake/plugins/tapd/impl" teambition "github.com/apache/incubator-devlake/plugins/teambition/impl" + tempo "github.com/apache/incubator-devlake/plugins/tempo/impl" testmo "github.com/apache/incubator-devlake/plugins/testmo/impl" trello "github.com/apache/incubator-devlake/plugins/trello/impl" webhook "github.com/apache/incubator-devlake/plugins/webhook/impl" @@ -97,6 +98,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("taiga/models", taiga.Taiga{}.GetTablesInfo) checker.FeedIn("tapd/models", tapd.Tapd{}.GetTablesInfo) checker.FeedIn("teambition/models", teambition.Teambition{}.GetTablesInfo) + checker.FeedIn("tempo/models", tempo.Tempo{}.GetTablesInfo) checker.FeedIn("testmo/models", testmo.Testmo{}.GetTablesInfo) checker.FeedIn("trello/models", trello.Trello{}.GetTablesInfo) checker.FeedIn("webhook/models", webhook.Webhook{}.GetTablesInfo) diff --git a/backend/plugins/tempo/api/blueprint_v200.go b/backend/plugins/tempo/api/blueprint_v200.go new file mode 100644 index 00000000000..f786dd65453 --- /dev/null +++ b/backend/plugins/tempo/api/blueprint_v200.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + // Load connection, scope and scopeConfig from the db + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + // Needed for the connection to populate its access tokens + _, err = api.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + return plan, scopes, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.TempoTeam, models.TempoScopeConfig], + connection *models.TempoConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope := scopeDetail.Scope + // Construct task options for Tempo + task, err := api.MakePipelinePlanTask( + "tempo", + subtaskMetas, + nil, // No entities to select for Tempo + map[string]interface{}{ + "connectionId": scope.ConnectionId, + "teamId": scope.TeamId, + }, + ) + if err != nil { + return nil, err + } + + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.TempoTeam, models.TempoScopeConfig], + connection *models.TempoConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0) + for _, scopeDetail := range scopeDetails { + tempoTeam := scopeDetail.Scope + // Add team to scopes + domainTeam := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: didgen.NewDomainIdGenerator(&models.TempoTeam{}).Generate(tempoTeam.ConnectionId, tempoTeam.TeamId), + }, + Name: tempoTeam.Name, + Url: "", // Tempo doesn't provide a direct URL for teams + Type: "team", + } + scopes = append(scopes, domainTeam) + } + return scopes, nil +} diff --git a/backend/plugins/tempo/api/connection.go b/backend/plugins/tempo/api/connection.go new file mode 100644 index 00000000000..5ee6f99059e --- /dev/null +++ b/backend/plugins/tempo/api/connection.go @@ -0,0 +1,211 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type TempoTestConnResponse struct { + shared.ApiBody + Connection *models.TempoConnection +} + +func testConnection(ctx context.Context, connection models.TempoConnection) (*TempoTestConnResponse, errors.Error) { + // Create API client + apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + + // Test connection by fetching teams + res, err := apiClient.Get("teams", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to test connection to Tempo API") + } + + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("failed to connect to Tempo API") + } + + // Sanitize and return response + connection = connection.Sanitize() + body := TempoTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + + return &body, nil +} + +// TestConnection test tempo connection +// @Summary test tempo connection +// @Description Test Tempo Connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} TempoTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // Decode + var connection models.TempoConnection + if err := api.Decode(input.Body, &connection, nil); err != nil { + return nil, err + } + // Test connection + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test tempo connection +// @Summary test tempo connection +// @Description Test Tempo Connection +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Success 200 {object} TempoTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.Convert(err) + } + // Test connection + if result, err := testConnection(context.TODO(), *connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } else { + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil + } +} + +// PostConnections create tempo connection +// @Summary create tempo connection +// @Description Create Tempo connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch tempo connection +// @Summary patch tempo connection +// @Description Patch Tempo connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a tempo connection +// @Summary delete a tempo connection +// @Description Delete a Tempo connection +// @Tags plugins/tempo +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all tempo connections +// @Summary get all tempo connections +// @Description Get all Tempo connections +// @Tags plugins/tempo +// @Success 200 {object} []models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get tempo connection detail +// @Summary get tempo connection detail +// @Description Get Tempo connection detail +// @Tags plugins/tempo +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} + +// GetTeams get teams for a connection +// @Summary get teams +// @Description Get teams for a Tempo connection +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Success 200 {object} []models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/teams [GET] +func GetTeams(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.FindByPk(input) + if err != nil { + return nil, err + } + + // Create API client + apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, err + } + + // Get teams from Tempo API + res, err := apiClient.Get("teams", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var teams []models.TempoTeamResponse + err = api.UnmarshalResponse(res, &teams) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + // Convert to tool layer models + result := make([]models.TempoTeam, 0, len(teams)) + for _, t := range teams { + result = append(result, *t.ConvertToToolLayer(connection.ID)) + } + + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} diff --git a/backend/plugins/tempo/api/init.go b/backend/plugins/tempo/api/init.go new file mode 100644 index 00000000000..47ce280c2fe --- /dev/null +++ b/backend/plugins/tempo/api/init.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.TempoConnection, models.TempoTeam, models.TempoScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.TempoConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.TempoConnection, models.TempoTeam, TempoRemotePagination] +var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.TempoConnection, models.TempoTeam] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + dsHelper = api.NewDataSourceHelper[ + models.TempoConnection, + models.TempoTeam, + models.TempoScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.TempoConnection) models.TempoConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.TempoConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.TempoConnection, models.TempoTeam, TempoRemotePagination](raProxy, listTempoRemoteTeams) + raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.TempoConnection, models.TempoTeam](raProxy, searchTempoRemoteTeams) +} diff --git a/backend/plugins/tempo/api/remote.go b/backend/plugins/tempo/api/remote.go new file mode 100644 index 00000000000..8dd7199939c --- /dev/null +++ b/backend/plugins/tempo/api/remote.go @@ -0,0 +1,196 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/url" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type TempoRemotePagination struct { + Limit int `json:"limit" mapstructure:"limit"` + Offset int `json:"offset" mapstructure:"offset"` +} + +func listTempoRemoteTeams( + connection *models.TempoConnection, + apiClient plugin.ApiClient, + groupId string, + page TempoRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam], + nextPage *TempoRemotePagination, + err errors.Error, +) { + if page.Limit == 0 { + page.Limit = 50 + } + + queryParams := url.Values{ + "offset": {strconv.Itoa(page.Offset)}, + "limit": {strconv.Itoa(page.Limit)}, + } + + res, err := apiClient.Get("teams", queryParams, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Total int `json:"total"` + } `json:"metadata"` + Results []models.TempoTeamResponse `json:"results"` + } + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + for _, team := range response.Results { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: strconv.FormatInt(team.Id, 10), + ParentId: nil, + Name: team.Name, + FullName: team.Name, + Data: team.ConvertToToolLayer(connection.ID), + }) + } + + if page.Offset+page.Limit < response.Metadata.Total { + nextPage = &TempoRemotePagination{ + Limit: page.Limit, + Offset: page.Offset + page.Limit, + } + } + + return children, nextPage, nil +} + +// RemoteScopes list all available scopes on the remote server +// @Summary list all available scopes on the remote server +// @Description list all available scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TempoTeam] +// @Tags plugins/tempo +// @Router /plugins/tempo/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +func searchTempoRemoteTeams( + apiClient plugin.ApiClient, + params *dsmodels.DsRemoteApiScopeSearchParams, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam], + err errors.Error, +) { + var queryParams url.Values + if params.Search != "" { + queryParams = url.Values{ + "name": {params.Search}, + } + } + + res, err := apiClient.Get("teams", queryParams, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Total int `json:"total"` + } `json:"metadata"` + Results []models.TempoTeamResponse `json:"results"` + } + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + for _, team := range response.Results { + if params.Search == "" || strings.Contains(strings.ToLower(team.Name), strings.ToLower(params.Search)) { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: strconv.FormatInt(team.Id, 10), + ParentId: nil, + Name: team.Name, + FullName: team.Name, + Data: team.ConvertToToolLayer(0), // connectionId overridden by PutMultiple handler + }) + } + } + + start := (params.Page - 1) * params.PageSize + end := start + params.PageSize + if start >= len(children) { + return nil, nil + } + if end > len(children) { + end = len(children) + } + + return children[start:end], nil +} + +// SearchRemoteScopes searches scopes on the remote server +// @Summary searches scopes on the remote server +// @Description searches scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TempoTeam] "the parentIds are always null" +// @Tags plugins/tempo +// @Router /plugins/tempo/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeSearch.Get(input) +} + +// Proxy forward API requests to Tempo API +// @Summary Remote server API proxy +// @Description Forward API requests to the specified remote server +// @Param connectionId path int true "connection ID" +// @Param path path string true "path to a API endpoint" +// @Router /plugins/tempo/connections/{connectionId}/proxy/{path} [GET] +// @Tags plugins/tempo +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/tempo/api/scope.go b/backend/plugins/tempo/api/scope.go new file mode 100644 index 00000000000..4f546127117 --- /dev/null +++ b/backend/plugins/tempo/api/scope.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.TempoTeam] +type ScopeDetail api.ScopeDetail[models.TempoTeam, models.TempoScopeConfig] + +// PutScope create or update tempo team +// @Summary create or update tempo team +// @Description Create or update Tempo team +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes [PUT] +func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// UpdateScope patch to tempo team +// @Summary patch to tempo team +// @Description patch to tempo team +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path int false "team ID" +// @Param scope body models.TempoTeam true "json" +// @Success 200 {object} models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [PATCH] +func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get Tempo teams +// @Summary get Tempo teams +// @Description get Tempo teams +// @Tags plugins/tempo +// @Param connectionId path int false "connection ID" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one Tempo team +// @Summary get one Tempo team +// @Description get one Tempo team +// @Tags plugins/tempo +// @Param connectionId path int false "connection ID" +// @Param scopeId path int false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Param scopeId path int true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState get one Tempo team's latest sync state +// @Summary get one Tempo team's latest sync state +// @Description get one Tempo team's latest sync state +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Param scopeId path int true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/tempo/api/scope_config.go b/backend/plugins/tempo/api/scope_config.go new file mode 100644 index 00000000000..851aeac8bef --- /dev/null +++ b/backend/plugins/tempo/api/scope_config.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// CreateScopeConfig create scope config for Tempo +// @Summary create scope config for Tempo +// @Description create scope config for Tempo +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int true "connectionId" +// @Param scopeConfig body models.TempoScopeConfig true "scope config" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs [POST] +func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// UpdateScopeConfig update scope config for Tempo +// @Summary update scope config for Tempo +// @Description update scope config for Tempo +// @Tags plugins/tempo +// @Accept application/json +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Param scopeConfig body models.TempoScopeConfig true "scope config" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [PATCH] +func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/tempo +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/tempo +// @Param connectionId path int true "connectionId" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/tempo +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv b/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv new file mode 100644 index 00000000000..31b08444ff0 --- /dev/null +++ b/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv @@ -0,0 +1,3 @@ +"id","params","data","url","input","created_at" +1,"{""ConnectionId"":1}","[{""self"":""https://api.tempo.io/4/teams/17"",""id"":17,""name"":""Abstract BE Team"",""summary"":"""",""key"":""""}]","https://api.tempo.io/4/teams","{""self"":""https://api.tempo.io/4/teams/17"",""id"":17,""name"":""Abstract BE Team"",""summary"":"""",""key"":""""}","2024-01-15 10:00:00.000" +2,"{""ConnectionId"":1}","[{""self"":""https://api.tempo.io/4/teams/71"",""id"":71,""name"":""SmartShopper (RTL+WHS)"",""summary"":""Microservice Applications"",""key"":""""}]","https://api.tempo.io/4/teams","{""self"":""https://api.tempo.io/4/teams/71"",""id"":71,""name"":""SmartShopper (RTL+WHS)"",""summary"":""Microservice Applications"",""key"":""""}","2024-01-15 10:00:00.000" \ No newline at end of file diff --git a/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv new file mode 100644 index 00000000000..7b62282a2ee --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv @@ -0,0 +1,3 @@ +"connection_id","team_id","name","summary","key" +1,17,"Abstract BE Team","","" +1,71,"SmartShopper (RTL+WHS)","Microservice Applications","" \ No newline at end of file diff --git a/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv new file mode 100644 index 00000000000..282ba5f5cf6 --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv @@ -0,0 +1,3 @@ +"connection_id","tempo_worklog_id","issue_id","time_spent_seconds","billable_seconds","start_date","start_time","description","author_account_id","created_at","updated_at" +1,12345,10001,3600,3600,"2024-01-15","09:00:00","Working on task","abc123","2024-01-15T10:00:00Z","2024-01-15T10:00:00Z" +1,12346,10002,7200,0,"2024-01-16","14:00:00","Internal meeting","abc123","2024-01-16T16:00:00Z","2024-01-16T16:00:00Z" diff --git a/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv new file mode 100644 index 00000000000..a100e9640dd --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv @@ -0,0 +1,3 @@ +id,author_id,comment,time_spent_minutes,logged_date,started_date,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +tempo:TempoWorklog:12345,abc123,Working on task,60,2024-01-15T10:00:00.000+00:00,2024-01-15T09:00:00.000+00:00,jira:JiraIssues:1:10001,,,0, +tempo:TempoWorklog:12346,abc123,Internal meeting,120,2024-01-16T16:00:00.000+00:00,2024-01-16T14:00:00.000+00:00,jira:JiraIssues:1:10002,,,0, diff --git a/backend/plugins/tempo/e2e/worklog_test.go b/backend/plugins/tempo/e2e/worklog_test.go new file mode 100644 index 00000000000..bb4ea262bf5 --- /dev/null +++ b/backend/plugins/tempo/e2e/worklog_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/tempo/impl" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/tasks" +) + +func TestWorklogDataFlow(t *testing.T) { + var plugin impl.Tempo + dataflowTester := e2ehelper.NewDataFlowTester(t, "tempo", plugin) + + taskData := &tasks.TempoTaskData{ + Options: &tasks.TempoOptions{ + ConnectionId: 1, + }, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_tempo_worklogs.csv", &models.TempoWorklog{}) + dataflowTester.FlushTabler(&ticket.IssueWorklog{}) + dataflowTester.Subtask(tasks.ConvertWorklogsMeta, taskData) + dataflowTester.VerifyTable( + ticket.IssueWorklog{}, + "./snapshot_tables/issue_worklogs.csv", + []string{"id", "author_id", "time_spent_minutes", "issue_id", "started_date", "logged_date"}, + ) +} + +func TestTeamDataFlow(t *testing.T) { + var plugin impl.Tempo + dataflowTester := e2ehelper.NewDataFlowTester(t, "tempo", plugin) + + taskData := &tasks.TempoTaskData{ + Options: &tasks.TempoOptions{ + ConnectionId: 1, + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tempo_api_teams.csv", "_raw_tempo_api_teams") + dataflowTester.FlushTabler(&models.TempoTeam{}) + dataflowTester.Subtask(tasks.ExtractTeamsMeta, taskData) + dataflowTester.VerifyTable( + models.TempoTeam{}, + "./snapshot_tables/_tool_tempo_teams.csv", + e2ehelper.ColumnWithRawData( + "connection_id", + "team_id", + "name", + "summary", + "key", + ), + ) +} diff --git a/backend/plugins/tempo/impl/impl.go b/backend/plugins/tempo/impl/impl.go new file mode 100644 index 00000000000..0666033a7fb --- /dev/null +++ b/backend/plugins/tempo/impl/impl.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/tempo/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginModel + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask + plugin.PluginSource +} = (*Tempo)(nil) + +type Tempo struct { +} + +func (p Tempo) Connection() dal.Tabler { + return &models.TempoConnection{} +} + +func (p Tempo) Scope() plugin.ToolLayerScope { + return &models.TempoTeam{} +} + +func (p Tempo) ScopeConfig() dal.Tabler { + return &models.TempoScopeConfig{} +} + +func (p Tempo) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Tempo) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.TempoConnection{}, + &models.TempoTeam{}, + &models.TempoWorklog{}, + &models.TempoScopeConfig{}, + } +} + +func (p Tempo) Description() string { + return "Collect worklogs from Jira Tempo" +} + +func (p Tempo) Name() string { + return "tempo" +} + +func (p Tempo) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectTeamsMeta, + tasks.ExtractTeamsMeta, + tasks.CollectWorklogsMeta, + tasks.ExtractWorklogsMeta, + tasks.ConvertWorklogsMeta, + } +} + +func (p Tempo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.TempoOptions + var err errors.Error + + logger := taskCtx.GetLogger() + logger.Debug("%v", options) + + err = helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "could not decode Tempo options") + } + + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("tempo connectionId is invalid") + } + + connection := &models.TempoConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Tempo connection") + } + + tempoApiClient, err := tasks.NewTempoApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create Tempo api client") + } + + taskData := &tasks.TempoTaskData{ + Options: &op, + ApiClient: tempoApiClient, + Connection: connection, + } + + return taskData, nil +} + +func (p Tempo) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Tempo) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/tempo" +} + +func (p Tempo) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Tempo) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/proxy/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/teams": { + "GET": api.GetTeams, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.UpdateScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScope, + }, + "connections/:connectionId/scope-configs": { + "POST": api.CreateScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.UpdateScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + } +} + +func (p Tempo) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.TempoTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + data.ApiClient.Release() + return nil +} diff --git a/backend/plugins/tempo/models/connection.go b/backend/plugins/tempo/models/connection.go new file mode 100644 index 00000000000..2322dc991a3 --- /dev/null +++ b/backend/plugins/tempo/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/common" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// TempoConn holds the essential information to connect to the Tempo API +type TempoConn struct { + helper.RestConnection `mapstructure:",squash"` + helper.AccessToken `mapstructure:",squash"` +} + +// TempoConnection holds TempoConn plus ID/Name for database storage +type TempoConnection struct { + helper.BaseConnection `mapstructure:",squash"` + TempoConn `mapstructure:",squash"` +} + +func (TempoConnection) TableName() string { + return "_tool_tempo_connections" +} + +func (connection TempoConnection) Connection() dal.Tabler { + return &connection +} + +func (connection TempoConnection) Sanitize() TempoConnection { + connection.TempoConn.Token = "" + return connection +} + +// This object conforms to what the frontend currently expects. +type TempoResponse struct { + Name string `json:"name"` + ID uint64 `json:"id"` + TempoConnection +} + +// TempoScopeConfig holds the configuration for a Tempo scope +type TempoScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` +} + +func (TempoScopeConfig) TableName() string { + return "_tool_tempo_scope_configs" +} + +func (c TempoScopeConfig) ScopeConfigId() uint64 { + return c.ID +} + +func (c TempoScopeConfig) ScopeConfigConnectionId() uint64 { + return c.ConnectionId +} diff --git a/backend/plugins/tempo/models/migrationscripts/archived/models.go b/backend/plugins/tempo/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..76f0a9edd8d --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/archived/models.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type TempoConnection struct { + archived.BaseConnection `mapstructure:",squash"` + archived.RestConnection `mapstructure:",squash"` + archived.AccessToken `mapstructure:",squash"` +} + +func (TempoConnection) TableName() string { + return "_tool_tempo_connections" +} + +type TempoScopeConfig struct { + archived.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + Name string `gorm:"type:varchar(255)" mapstructure:"name" json:"name"` +} + +func (TempoScopeConfig) TableName() string { + return "_tool_tempo_scope_configs" +} + +type TempoTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + ScopeConfigId uint64 + Key string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Summary string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (TempoTeam) TableName() string { + return "_tool_tempo_teams" +} + +type TempoWorklog struct { + ConnectionId uint64 `gorm:"primaryKey"` + TempoWorklogId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"index"` + IssueId int64 `gorm:"index"` + IssueKey string `gorm:"type:varchar(255)"` + AuthorAccountId string `gorm:"type:varchar(255)"` + TimeSpentSeconds int + BillableSeconds int + StartDate string `gorm:"type:varchar(255)"` + StartTime string `gorm:"type:varchar(255)"` + Description string `gorm:"type:text"` + CreatedAt string `gorm:"type:varchar(255)"` + UpdatedAt string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (TempoWorklog) TableName() string { + return "_tool_tempo_worklogs" +} diff --git a/backend/plugins/tempo/models/migrationscripts/register.go b/backend/plugins/tempo/models/migrationscripts/register.go new file mode 100644 index 00000000000..42e280a58b9 --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All returns all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(tempoInitTables20240401), + } +} diff --git a/backend/plugins/tempo/models/migrationscripts/register_tables.go b/backend/plugins/tempo/models/migrationscripts/register_tables.go new file mode 100644 index 00000000000..ee4e7c8ddb3 --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/register_tables.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/tempo/models/migrationscripts/archived" +) + +type tempoInitTables20240401 struct{} + +func (script *tempoInitTables20240401) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.TempoConnection{}, + &archived.TempoScopeConfig{}, + &archived.TempoTeam{}, + &archived.TempoWorklog{}, + ) +} + +func (*tempoInitTables20240401) Version() uint64 { + return 20240401143000 +} + +func (*tempoInitTables20240401) Name() string { + return "Tempo init schemas" +} diff --git a/backend/plugins/tempo/models/team.go b/backend/plugins/tempo/models/team.go new file mode 100644 index 00000000000..d29f3fd72de --- /dev/null +++ b/backend/plugins/tempo/models/team.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +var _ plugin.ToolLayerScope = (*TempoTeam)(nil) + +// TempoTeam represents a team in Jira Tempo +type TempoTeam struct { + common.Scope `mapstructure:",squash" gorm:"embedded"` + TeamId int64 `json:"teamId" mapstructure:"teamId" validate:"required" gorm:"primaryKey"` + Id int64 `json:"id" gorm:"-" mapstructure:"-"` // JS scope selector compatibility (mirrors TeamId) + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Summary string `json:"summary" mapstructure:"summary" gorm:"type:varchar(255)"` +} + +// AfterFind populates the virtual Id field after reading from DB +func (t *TempoTeam) AfterFind(_ *gorm.DB) error { + t.Id = t.TeamId + return nil +} + +func (t TempoTeam) ScopeId() string { + return fmt.Sprintf("%d", t.TeamId) +} + +func (t TempoTeam) ScopeName() string { + return t.Name +} + +func (t TempoTeam) ScopeFullName() string { + return fmt.Sprintf("%s - %s", t.Key, t.Name) +} + +func (t TempoTeam) ScopeParams() interface{} { + return &TempoApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (TempoTeam) TableName() string { + return "_tool_tempo_teams" +} + +// TempoApiParams holds the API parameters for Tempo teams +type TempoApiParams struct { + ConnectionId uint64 + TeamId int64 +} + +// TempoTeamResponse represents the API response for a team from Tempo API +type TempoTeamResponse struct { + Id int64 `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Summary string `json:"summary"` +} + +// ConvertToToolLayer converts the API response to the tool layer model +func (r TempoTeamResponse) ConvertToToolLayer(connectionId uint64) *TempoTeam { + return &TempoTeam{ + Scope: common.Scope{ + ConnectionId: connectionId, + }, + TeamId: r.Id, + Id: r.Id, + Key: r.Key, + Name: r.Name, + Summary: r.Summary, + } +} diff --git a/backend/plugins/tempo/models/worklog.go b/backend/plugins/tempo/models/worklog.go new file mode 100644 index 00000000000..780c69ee597 --- /dev/null +++ b/backend/plugins/tempo/models/worklog.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type TempoWorklog struct { + common.NoPKModel `mapstructure:",squash" gorm:"embedded"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey"` + TempoWorklogId int64 `json:"tempoWorklogId" mapstructure:"tempoWorklogId" gorm:"primaryKey"` + TeamId int64 `json:"teamId" mapstructure:"teamId" gorm:"index"` + IssueId int64 `json:"issueId" mapstructure:"issueId" gorm:"index"` + IssueKey string `json:"issueKey" mapstructure:"issueKey" gorm:"type:varchar(255)"` + AuthorAccountId string `json:"authorAccountId" mapstructure:"authorAccountId" gorm:"type:varchar(255)"` + TimeSpentSeconds int `json:"timeSpentSeconds" mapstructure:"timeSpentSeconds"` + BillableSeconds int `json:"billableSeconds" mapstructure:"billableSeconds"` + StartDate string `json:"startDate" mapstructure:"startDate" gorm:"type:varchar(255)"` + StartTime string `json:"startTime" mapstructure:"startTime" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description" gorm:"type:text"` + CreatedAt string `json:"createdAt" mapstructure:"createdAt" gorm:"type:varchar(255)"` + UpdatedAt string `json:"updatedAt" mapstructure:"updatedAt" gorm:"type:varchar(255)"` +} + +func (TempoWorklog) TableName() string { + return "_tool_tempo_worklogs" +} diff --git a/backend/plugins/tempo/tasks/api_client.go b/backend/plugins/tempo/tasks/api_client.go new file mode 100644 index 00000000000..8dcefb13472 --- /dev/null +++ b/backend/plugins/tempo/tasks/api_client.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +const ( + BaseURL = "https://api.tempo.io/4" +) + +// NewTempoApiClient creates a new Tempo API client +func NewTempoApiClient( + taskCtx plugin.TaskContext, + connection *models.TempoConnection, +) (*helper.ApiAsyncClient, errors.Error) { + apiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + return helper.CreateAsyncApiClient(taskCtx, apiClient, nil) +} diff --git a/backend/plugins/tempo/tasks/tasks.go b/backend/plugins/tempo/tasks/tasks.go new file mode 100644 index 00000000000..547282d6a5a --- /dev/null +++ b/backend/plugins/tempo/tasks/tasks.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +const ( + RAW_WORKLOG_TABLE = "tempo_api_worklogs" + RAW_TEAM_TABLE = "tempo_api_teams" +) + +// TempoOptions holds the options for the Tempo plugin +type TempoOptions struct { + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + ScopeConfigId uint64 `mapstructure:"scopeConfigId" json:"scopeConfigId"` + ScopeConfig *models.TempoScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"` + TeamId int64 `mapstructure:"teamId" json:"teamId"` + FromDate string `mapstructure:"fromDate" json:"fromDate"` + ToDate string `mapstructure:"toDate" json:"toDate"` +} + +// TempoTaskData holds the data for a Tempo task +type TempoTaskData struct { + Options *TempoOptions + ApiClient *helper.ApiAsyncClient + Connection *models.TempoConnection +} diff --git a/backend/plugins/tempo/tasks/team_collector.go b/backend/plugins/tempo/tasks/team_collector.go new file mode 100644 index 00000000000..1ac9e2f5bf5 --- /dev/null +++ b/backend/plugins/tempo/tasks/team_collector.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var CollectTeamsMeta = plugin.SubTaskMeta{ + Name: "collect-teams", + Description: "Collect teams from Tempo API", + EntryPoint: CollectTeams, + EnabledByDefault: true, + Dependencies: nil, + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectTeams(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: "teams", + PageSize: 50, + GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Total int `json:"total"` + Offset int `json:"offset"` + } `json:"metadata"` + } + if err := api.UnmarshalResponse(res, &response); err != nil { + return 0, err + } + totalPages := (response.Metadata.Total + args.PageSize - 1) / args.PageSize + return totalPages, nil + }, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + pager := reqData.Pager + if pager == nil { + pager = &api.Pager{Page: 1, Skip: 0, Size: 50} + } + query.Set("offset", strconv.Itoa(pager.Skip)) + query.Set("limit", strconv.Itoa(pager.Size)) + + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + since := apiCollector.GetSince() + query.Set("updatedFrom", since.Format(time.RFC3339)) + } + + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var response struct { + Results []json.RawMessage `json:"results"` + } + err := api.UnmarshalResponse(res, &response) + if err != nil { + return nil, err + } + return response.Results, nil + }, + }) + + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/tempo/tasks/team_extractor.go b/backend/plugins/tempo/tasks/team_extractor.go new file mode 100644 index 00000000000..1ecbc345c4b --- /dev/null +++ b/backend/plugins/tempo/tasks/team_extractor.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ExtractTeamsMeta = plugin.SubTaskMeta{ + Name: "extract_teams", + EntryPoint: ExtractTeams, + EnabledByDefault: true, + Description: "Extract teams from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +// API response for Tempo team +type TempoTeamResponse struct { + Id int64 `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Summary string `json:"summary"` + Self string `json:"self"` +} + +func ExtractTeams(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiTeam TempoTeamResponse + err := errors.Convert(json.Unmarshal(row.Data, &apiTeam)) + if err != nil { + return nil, err + } + + // Transform to tool layer model + team := &models.TempoTeam{ + TeamId: apiTeam.Id, + Key: apiTeam.Key, + Name: apiTeam.Name, + Summary: apiTeam.Summary, + } + team.ConnectionId = data.Options.ConnectionId + + return []interface{}{team}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tempo/tasks/worklog_collector.go b/backend/plugins/tempo/tasks/worklog_collector.go new file mode 100644 index 00000000000..28738e35240 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_collector.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var CollectWorklogsMeta = plugin.SubTaskMeta{ + Name: "collect_worklogs", + EntryPoint: CollectWorklogs, + EnabledByDefault: true, + Description: "Collect worklogs from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }) + if err != nil { + return err + } + + urlTemplate := "worklogs" + if data.Options.TeamId != 0 { + urlTemplate = fmt.Sprintf("worklogs/team/%d", data.Options.TeamId) + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: urlTemplate, + PageSize: 1000, + GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Total int `json:"total"` + Offset int `json:"offset"` + } `json:"metadata"` + } + if err := api.UnmarshalResponse(res, &response); err != nil { + return 0, err + } + totalPages := (response.Metadata.Total + args.PageSize - 1) / args.PageSize + return totalPages, nil + }, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + pager := reqData.Pager + if pager == nil { + pager = &api.Pager{Page: 1, Skip: 0, Size: 1000} + } + query.Set("offset", strconv.Itoa(pager.Skip)) + query.Set("limit", strconv.Itoa(pager.Size)) + + if data.Options.TeamId != 0 { + fromDate := data.Options.FromDate + toDate := data.Options.ToDate + if fromDate == "" { + since := time.Now().AddDate(0, 0, -90) + fromDate = since.Format("2006-01-02") + } + if toDate == "" { + toDate = time.Now().Format("2006-01-02") + } + query.Set("from", fromDate) + query.Set("to", toDate) + } else { + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + since := apiCollector.GetSince() + query.Set("updatedFrom", since.Format(time.RFC3339)) + } + } + + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var response struct { + Results []json.RawMessage `json:"results"` + } + err := api.UnmarshalResponse(res, &response) + if err != nil { + return nil, err + } + return response.Results, nil + }, + }) + + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/tempo/tasks/worklog_convertor.go b/backend/plugins/tempo/tasks/worklog_convertor.go new file mode 100644 index 00000000000..60ee32a1416 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_convertor.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ConvertWorklogsMeta = plugin.SubTaskMeta{ + Name: "convert_worklogs", + EntryPoint: ConvertWorklogs, + EnabledByDefault: true, + Description: "Convert worklogs to domain layer", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + connectionId := data.Options.ConnectionId + + logger.Info("converting Tempo worklogs to domain layer") + + issueIdMapping, err := buildIssueIdMapping(db, connectionId) + if err != nil { + return errors.Default.Wrap(err, "failed to build issue ID mapping") + } + + clauses := []dal.Clause{ + dal.Select("*"), + dal.From("_tool_tempo_worklogs"), + dal.Where("connection_id = ?", connectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return errors.Default.Wrap(err, "failed to query Tempo worklogs") + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: connectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + InputRowType: reflect.TypeOf(models.TempoWorklog{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + tempoWorklog := inputRow.(*models.TempoWorklog) + + domainIssueId := "" + if domainId, ok := issueIdMapping[tempoWorklog.IssueId]; ok { + domainIssueId = domainId + } else { + domainIssueId = fmt.Sprintf("jira:JiraIssues:%d:%d", connectionId, tempoWorklog.IssueId) + } + + domainWorklogId := fmt.Sprintf("tempo:TempoWorklog:%d", tempoWorklog.TempoWorklogId) + timeSpentMinutes := tempoWorklog.TimeSpentSeconds / 60 + + var startedDate *time.Time + if tempoWorklog.StartDate != "" && tempoWorklog.StartTime != "" { + if t, err := time.Parse("2006-01-02 15:04:05", tempoWorklog.StartDate+" "+tempoWorklog.StartTime); err == nil { + startedDate = &t + } + } + + var loggedDate *time.Time + if tempoWorklog.CreatedAt != "" { + if t, err := time.Parse(time.RFC3339, tempoWorklog.CreatedAt); err == nil { + loggedDate = &t + } + } + + worklog := &ticket.IssueWorklog{ + DomainEntity: domainlayer.DomainEntity{ + Id: domainWorklogId, + }, + IssueId: domainIssueId, + AuthorId: tempoWorklog.AuthorAccountId, + TimeSpentMinutes: timeSpentMinutes, + StartedDate: startedDate, + LoggedDate: loggedDate, + Comment: tempoWorklog.Description, + } + + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return errors.Default.Wrap(err, "failed to create data converter") + } + + return converter.Execute() +} + +func buildIssueIdMapping(db dal.Dal, connectionId uint64) (map[int64]string, errors.Error) { + mapping := make(map[int64]string) + + if !db.HasTable("_tool_jira_issues") { + return mapping, nil + } + + clauses := []dal.Clause{ + dal.Select("issue_id"), + dal.From("_tool_jira_issues"), + dal.Where("connection_id = ?", connectionId), + } + + rows, err := db.Cursor(clauses...) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query jira issues") + } + defer rows.Close() + + for rows.Next() { + var issueId uint64 + if err := rows.Scan(&issueId); err != nil { + return nil, errors.Default.Wrap(err, "failed to scan issue") + } + domainId := fmt.Sprintf("jira:JiraIssues:%d:%d", connectionId, issueId) + mapping[int64(issueId)] = domainId + } + + return mapping, nil +} diff --git a/backend/plugins/tempo/tasks/worklog_extractor.go b/backend/plugins/tempo/tasks/worklog_extractor.go new file mode 100644 index 00000000000..31370868d38 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_extractor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ExtractWorklogsMeta = plugin.SubTaskMeta{ + Name: "extract_worklogs", + EntryPoint: ExtractWorklogs, + EnabledByDefault: true, + Description: "Extract worklogs from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type TempoWorklogAuthor struct { + AccountId string `json:"accountId"` +} + +type TempoWorklogResponse struct { + TempoWorklogId int64 `json:"tempoWorklogId"` + Issue struct { + Id int64 `json:"id"` + } `json:"issue"` + TimeSpentSeconds int `json:"timeSpentSeconds"` + BillableSeconds int `json:"billableSeconds"` + StartDate string `json:"startDate"` + StartTime string `json:"startTime"` + Description string `json:"description"` + Author TempoWorklogAuthor `json:"author"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +func ExtractWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiWorklog TempoWorklogResponse + err := errors.Convert(json.Unmarshal(row.Data, &apiWorklog)) + if err != nil { + return nil, err + } + + worklog := &models.TempoWorklog{ + ConnectionId: data.Options.ConnectionId, + TempoWorklogId: apiWorklog.TempoWorklogId, + IssueId: apiWorklog.Issue.Id, + TimeSpentSeconds: apiWorklog.TimeSpentSeconds, + BillableSeconds: apiWorklog.BillableSeconds, + StartDate: apiWorklog.StartDate, + StartTime: apiWorklog.StartTime, + Description: apiWorklog.Description, + AuthorAccountId: apiWorklog.Author.AccountId, + CreatedAt: apiWorklog.CreatedAt, + UpdatedAt: apiWorklog.UpdatedAt, + } + + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tempo/tempo.go b/backend/plugins/tempo/tempo.go new file mode 100644 index 00000000000..8b75aee5e4d --- /dev/null +++ b/backend/plugins/tempo/tempo.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/tempo/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Tempo //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "tempo"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "tempo connection id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index c885d5114b5..b8bd6672f41 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -42,6 +42,7 @@ import { QDevConfig } from './q-dev'; import { TeambitionConfig } from './teambition'; import { TestmoConfig } from './testmo'; import { SlackConfig } from './slack/config'; +import { TempoConfig } from './tempo'; export const pluginConfigs: IPluginConfig[] = [ ArgoCDConfig, @@ -61,6 +62,7 @@ export const pluginConfigs: IPluginConfig[] = [ PagerDutyConfig, RootlyConfig, SlackConfig, + TempoConfig, QDevConfig, SonarQubeConfig, TAPDConfig, diff --git a/config-ui/src/plugins/register/tempo/assets/icon.png b/config-ui/src/plugins/register/tempo/assets/icon.png new file mode 100644 index 00000000000..80f4e36f897 Binary files /dev/null and b/config-ui/src/plugins/register/tempo/assets/icon.png differ diff --git a/config-ui/src/plugins/register/tempo/assets/icon.svg b/config-ui/src/plugins/register/tempo/assets/icon.svg new file mode 100644 index 00000000000..6c7407b2b4f --- /dev/null +++ b/config-ui/src/plugins/register/tempo/assets/icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/config-ui/src/plugins/register/tempo/config.tsx b/config-ui/src/plugins/register/tempo/config.tsx new file mode 100644 index 00000000000..c4494d13908 --- /dev/null +++ b/config-ui/src/plugins/register/tempo/config.tsx @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { DOC_URL } from '@/release'; +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.png'; + +export const TempoConfig: IPluginConfig = { + plugin: 'tempo', + name: 'Tempo', + icon: () => , + sort: 20, + connection: { + docLink: 'https://devlake.apache.org/docs/Configuration/Tempo', + initialValues: { + endpoint: 'https://api.tempo.io/4', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'REST Endpoint', + subLabel: 'Tempo API v4 base URL', + placeholder: 'https://api.tempo.io/4', + }, + 'token', + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for default.', + defaultValue: 1000, + }, + ], + }, + dataScope: { + title: 'Teams', + millerColumn: { + columnCount: 2.5, + }, + }, + scopeConfig: { + entities: ['TICKET'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/tempo/index.ts b/config-ui/src/plugins/register/tempo/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/tempo/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config';