From 07a08631ae7f624d798b9ee297cee1294e639b4c Mon Sep 17 00:00:00 2001 From: Fix Bot Date: Sun, 24 May 2026 10:26:46 +0200 Subject: [PATCH 1/2] feat(tempo): add Jira Tempo Timesheets plugin Adds plugin for ingesting worklogs from Jira Tempo (Tempo Timesheets) API v4. Backend: - Plugin entry point (impl/impl.go) - Tempo API client with Bearer token auth (tasks/api_client.go) - Data collectors, extractors, and converters for worklogs and teams - Connection management API endpoints (api/) - Database migrations for _tool_tempo_worklogs, _tool_tempo_teams - E2E tests with CSV fixtures Config-UI: - Connection configuration UI (config.tsx) - Plugin registration in index.ts Dependency: requires Jira plugin for issue ID mapping. Closes #8883 --- backend/plugins/tempo/api/blueprint_v200.go | 120 ++++++++++ backend/plugins/tempo/api/connection.go | 211 ++++++++++++++++++ backend/plugins/tempo/api/init.go | 56 +++++ backend/plugins/tempo/api/remote.go | 196 ++++++++++++++++ backend/plugins/tempo/api/scope.go | 120 ++++++++++ backend/plugins/tempo/api/scope_config.go | 97 ++++++++ .../e2e/raw_tables/_raw_tempo_api_teams.csv | 3 + .../e2e/snapshot_tables/_tool_tempo_teams.csv | 3 + .../snapshot_tables/_tool_tempo_worklogs.csv | 3 + .../e2e/snapshot_tables/issue_worklogs.csv | 3 + backend/plugins/tempo/e2e/worklog_test.go | 74 ++++++ backend/plugins/tempo/impl/impl.go | 202 +++++++++++++++++ backend/plugins/tempo/models/connection.go | 74 ++++++ .../tempo/models/migrationscripts/register.go | 29 +++ .../migrationscripts/register_tables.go | 45 ++++ backend/plugins/tempo/models/team.go | 95 ++++++++ backend/plugins/tempo/models/worklog.go | 43 ++++ backend/plugins/tempo/tasks/api_client.go | 41 ++++ backend/plugins/tempo/tasks/tasks.go | 45 ++++ backend/plugins/tempo/tasks/team_collector.go | 115 ++++++++++ backend/plugins/tempo/tasks/team_extractor.go | 82 +++++++ .../plugins/tempo/tasks/worklog_collector.go | 134 +++++++++++ .../plugins/tempo/tasks/worklog_convertor.go | 155 +++++++++++++ .../plugins/tempo/tasks/worklog_extractor.go | 97 ++++++++ backend/plugins/tempo/tempo.go | 40 ++++ config-ui/src/plugins/register/index.ts | 2 + .../plugins/register/tempo/assets/icon.png | Bin 0 -> 4179 bytes .../plugins/register/tempo/assets/icon.svg | 11 + .../src/plugins/register/tempo/config.tsx | 61 +++++ config-ui/src/plugins/register/tempo/index.ts | 19 ++ 30 files changed, 2176 insertions(+) create mode 100644 backend/plugins/tempo/api/blueprint_v200.go create mode 100644 backend/plugins/tempo/api/connection.go create mode 100644 backend/plugins/tempo/api/init.go create mode 100644 backend/plugins/tempo/api/remote.go create mode 100644 backend/plugins/tempo/api/scope.go create mode 100644 backend/plugins/tempo/api/scope_config.go create mode 100644 backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv create mode 100644 backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv create mode 100644 backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv create mode 100644 backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv create mode 100644 backend/plugins/tempo/e2e/worklog_test.go create mode 100644 backend/plugins/tempo/impl/impl.go create mode 100644 backend/plugins/tempo/models/connection.go create mode 100644 backend/plugins/tempo/models/migrationscripts/register.go create mode 100644 backend/plugins/tempo/models/migrationscripts/register_tables.go create mode 100644 backend/plugins/tempo/models/team.go create mode 100644 backend/plugins/tempo/models/worklog.go create mode 100644 backend/plugins/tempo/tasks/api_client.go create mode 100644 backend/plugins/tempo/tasks/tasks.go create mode 100644 backend/plugins/tempo/tasks/team_collector.go create mode 100644 backend/plugins/tempo/tasks/team_extractor.go create mode 100644 backend/plugins/tempo/tasks/worklog_collector.go create mode 100644 backend/plugins/tempo/tasks/worklog_convertor.go create mode 100644 backend/plugins/tempo/tasks/worklog_extractor.go create mode 100644 backend/plugins/tempo/tempo.go create mode 100644 config-ui/src/plugins/register/tempo/assets/icon.png create mode 100644 config-ui/src/plugins/register/tempo/assets/icon.svg create mode 100644 config-ui/src/plugins/register/tempo/config.tsx create mode 100644 config-ui/src/plugins/register/tempo/index.ts diff --git a/backend/plugins/tempo/api/blueprint_v200.go b/backend/plugins/tempo/api/blueprint_v200.go new file mode 100644 index 00000000000..f786dd65453 --- /dev/null +++ b/backend/plugins/tempo/api/blueprint_v200.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + // Load connection, scope and scopeConfig from the db + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + // Needed for the connection to populate its access tokens + _, err = api.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + return plan, scopes, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.TempoTeam, models.TempoScopeConfig], + connection *models.TempoConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope := scopeDetail.Scope + // Construct task options for Tempo + task, err := api.MakePipelinePlanTask( + "tempo", + subtaskMetas, + nil, // No entities to select for Tempo + map[string]interface{}{ + "connectionId": scope.ConnectionId, + "teamId": scope.TeamId, + }, + ) + if err != nil { + return nil, err + } + + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.TempoTeam, models.TempoScopeConfig], + connection *models.TempoConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0) + for _, scopeDetail := range scopeDetails { + tempoTeam := scopeDetail.Scope + // Add team to scopes + domainTeam := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: didgen.NewDomainIdGenerator(&models.TempoTeam{}).Generate(tempoTeam.ConnectionId, tempoTeam.TeamId), + }, + Name: tempoTeam.Name, + Url: "", // Tempo doesn't provide a direct URL for teams + Type: "team", + } + scopes = append(scopes, domainTeam) + } + return scopes, nil +} diff --git a/backend/plugins/tempo/api/connection.go b/backend/plugins/tempo/api/connection.go new file mode 100644 index 00000000000..5ee6f99059e --- /dev/null +++ b/backend/plugins/tempo/api/connection.go @@ -0,0 +1,211 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type TempoTestConnResponse struct { + shared.ApiBody + Connection *models.TempoConnection +} + +func testConnection(ctx context.Context, connection models.TempoConnection) (*TempoTestConnResponse, errors.Error) { + // Create API client + apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + + // Test connection by fetching teams + res, err := apiClient.Get("teams", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to test connection to Tempo API") + } + + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("failed to connect to Tempo API") + } + + // Sanitize and return response + connection = connection.Sanitize() + body := TempoTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + + return &body, nil +} + +// TestConnection test tempo connection +// @Summary test tempo connection +// @Description Test Tempo Connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} TempoTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // Decode + var connection models.TempoConnection + if err := api.Decode(input.Body, &connection, nil); err != nil { + return nil, err + } + // Test connection + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test tempo connection +// @Summary test tempo connection +// @Description Test Tempo Connection +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Success 200 {object} TempoTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.Convert(err) + } + // Test connection + if result, err := testConnection(context.TODO(), *connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } else { + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil + } +} + +// PostConnections create tempo connection +// @Summary create tempo connection +// @Description Create Tempo connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch tempo connection +// @Summary patch tempo connection +// @Description Patch Tempo connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a tempo connection +// @Summary delete a tempo connection +// @Description Delete a Tempo connection +// @Tags plugins/tempo +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all tempo connections +// @Summary get all tempo connections +// @Description Get all Tempo connections +// @Tags plugins/tempo +// @Success 200 {object} []models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get tempo connection detail +// @Summary get tempo connection detail +// @Description Get Tempo connection detail +// @Tags plugins/tempo +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} + +// GetTeams get teams for a connection +// @Summary get teams +// @Description Get teams for a Tempo connection +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Success 200 {object} []models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/teams [GET] +func GetTeams(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.FindByPk(input) + if err != nil { + return nil, err + } + + // Create API client + apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, err + } + + // Get teams from Tempo API + res, err := apiClient.Get("teams", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var teams []models.TempoTeamResponse + err = api.UnmarshalResponse(res, &teams) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + // Convert to tool layer models + result := make([]models.TempoTeam, 0, len(teams)) + for _, t := range teams { + result = append(result, *t.ConvertToToolLayer(connection.ID)) + } + + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} diff --git a/backend/plugins/tempo/api/init.go b/backend/plugins/tempo/api/init.go new file mode 100644 index 00000000000..1313a8bb9f8 --- /dev/null +++ b/backend/plugins/tempo/api/init.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate + +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.TempoConnection, models.TempoTeam, models.TempoScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.TempoConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.TempoConnection, models.TempoTeam, TempoRemotePagination] +var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.TempoConnection, models.TempoTeam] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.TempoConnection, + models.TempoTeam, + models.TempoScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.TempoConnection) models.TempoConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.TempoConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.TempoConnection, models.TempoTeam, TempoRemotePagination](raProxy, listTempoRemoteTeams) + raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.TempoConnection, models.TempoTeam](raProxy, searchTempoRemoteTeams) +} diff --git a/backend/plugins/tempo/api/remote.go b/backend/plugins/tempo/api/remote.go new file mode 100644 index 00000000000..8dd7199939c --- /dev/null +++ b/backend/plugins/tempo/api/remote.go @@ -0,0 +1,196 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/url" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type TempoRemotePagination struct { + Limit int `json:"limit" mapstructure:"limit"` + Offset int `json:"offset" mapstructure:"offset"` +} + +func listTempoRemoteTeams( + connection *models.TempoConnection, + apiClient plugin.ApiClient, + groupId string, + page TempoRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam], + nextPage *TempoRemotePagination, + err errors.Error, +) { + if page.Limit == 0 { + page.Limit = 50 + } + + queryParams := url.Values{ + "offset": {strconv.Itoa(page.Offset)}, + "limit": {strconv.Itoa(page.Limit)}, + } + + res, err := apiClient.Get("teams", queryParams, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Total int `json:"total"` + } `json:"metadata"` + Results []models.TempoTeamResponse `json:"results"` + } + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + for _, team := range response.Results { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: strconv.FormatInt(team.Id, 10), + ParentId: nil, + Name: team.Name, + FullName: team.Name, + Data: team.ConvertToToolLayer(connection.ID), + }) + } + + if page.Offset+page.Limit < response.Metadata.Total { + nextPage = &TempoRemotePagination{ + Limit: page.Limit, + Offset: page.Offset + page.Limit, + } + } + + return children, nextPage, nil +} + +// RemoteScopes list all available scopes on the remote server +// @Summary list all available scopes on the remote server +// @Description list all available scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TempoTeam] +// @Tags plugins/tempo +// @Router /plugins/tempo/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +func searchTempoRemoteTeams( + apiClient plugin.ApiClient, + params *dsmodels.DsRemoteApiScopeSearchParams, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam], + err errors.Error, +) { + var queryParams url.Values + if params.Search != "" { + queryParams = url.Values{ + "name": {params.Search}, + } + } + + res, err := apiClient.Get("teams", queryParams, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Total int `json:"total"` + } `json:"metadata"` + Results []models.TempoTeamResponse `json:"results"` + } + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + for _, team := range response.Results { + if params.Search == "" || strings.Contains(strings.ToLower(team.Name), strings.ToLower(params.Search)) { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: strconv.FormatInt(team.Id, 10), + ParentId: nil, + Name: team.Name, + FullName: team.Name, + Data: team.ConvertToToolLayer(0), // connectionId overridden by PutMultiple handler + }) + } + } + + start := (params.Page - 1) * params.PageSize + end := start + params.PageSize + if start >= len(children) { + return nil, nil + } + if end > len(children) { + end = len(children) + } + + return children[start:end], nil +} + +// SearchRemoteScopes searches scopes on the remote server +// @Summary searches scopes on the remote server +// @Description searches scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TempoTeam] "the parentIds are always null" +// @Tags plugins/tempo +// @Router /plugins/tempo/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeSearch.Get(input) +} + +// Proxy forward API requests to Tempo API +// @Summary Remote server API proxy +// @Description Forward API requests to the specified remote server +// @Param connectionId path int true "connection ID" +// @Param path path string true "path to a API endpoint" +// @Router /plugins/tempo/connections/{connectionId}/proxy/{path} [GET] +// @Tags plugins/tempo +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/tempo/api/scope.go b/backend/plugins/tempo/api/scope.go new file mode 100644 index 00000000000..4f546127117 --- /dev/null +++ b/backend/plugins/tempo/api/scope.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.TempoTeam] +type ScopeDetail api.ScopeDetail[models.TempoTeam, models.TempoScopeConfig] + +// PutScope create or update tempo team +// @Summary create or update tempo team +// @Description Create or update Tempo team +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes [PUT] +func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// UpdateScope patch to tempo team +// @Summary patch to tempo team +// @Description patch to tempo team +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path int false "team ID" +// @Param scope body models.TempoTeam true "json" +// @Success 200 {object} models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [PATCH] +func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get Tempo teams +// @Summary get Tempo teams +// @Description get Tempo teams +// @Tags plugins/tempo +// @Param connectionId path int false "connection ID" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one Tempo team +// @Summary get one Tempo team +// @Description get one Tempo team +// @Tags plugins/tempo +// @Param connectionId path int false "connection ID" +// @Param scopeId path int false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Param scopeId path int true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState get one Tempo team's latest sync state +// @Summary get one Tempo team's latest sync state +// @Description get one Tempo team's latest sync state +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Param scopeId path int true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/tempo/api/scope_config.go b/backend/plugins/tempo/api/scope_config.go new file mode 100644 index 00000000000..851aeac8bef --- /dev/null +++ b/backend/plugins/tempo/api/scope_config.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// CreateScopeConfig create scope config for Tempo +// @Summary create scope config for Tempo +// @Description create scope config for Tempo +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int true "connectionId" +// @Param scopeConfig body models.TempoScopeConfig true "scope config" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs [POST] +func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// UpdateScopeConfig update scope config for Tempo +// @Summary update scope config for Tempo +// @Description update scope config for Tempo +// @Tags plugins/tempo +// @Accept application/json +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Param scopeConfig body models.TempoScopeConfig true "scope config" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [PATCH] +func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/tempo +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/tempo +// @Param connectionId path int true "connectionId" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/tempo +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv b/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv new file mode 100644 index 00000000000..31b08444ff0 --- /dev/null +++ b/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv @@ -0,0 +1,3 @@ +"id","params","data","url","input","created_at" +1,"{""ConnectionId"":1}","[{""self"":""https://api.tempo.io/4/teams/17"",""id"":17,""name"":""Abstract BE Team"",""summary"":"""",""key"":""""}]","https://api.tempo.io/4/teams","{""self"":""https://api.tempo.io/4/teams/17"",""id"":17,""name"":""Abstract BE Team"",""summary"":"""",""key"":""""}","2024-01-15 10:00:00.000" +2,"{""ConnectionId"":1}","[{""self"":""https://api.tempo.io/4/teams/71"",""id"":71,""name"":""SmartShopper (RTL+WHS)"",""summary"":""Microservice Applications"",""key"":""""}]","https://api.tempo.io/4/teams","{""self"":""https://api.tempo.io/4/teams/71"",""id"":71,""name"":""SmartShopper (RTL+WHS)"",""summary"":""Microservice Applications"",""key"":""""}","2024-01-15 10:00:00.000" \ No newline at end of file diff --git a/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv new file mode 100644 index 00000000000..7b62282a2ee --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv @@ -0,0 +1,3 @@ +"connection_id","team_id","name","summary","key" +1,17,"Abstract BE Team","","" +1,71,"SmartShopper (RTL+WHS)","Microservice Applications","" \ No newline at end of file diff --git a/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv new file mode 100644 index 00000000000..282ba5f5cf6 --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv @@ -0,0 +1,3 @@ +"connection_id","tempo_worklog_id","issue_id","time_spent_seconds","billable_seconds","start_date","start_time","description","author_account_id","created_at","updated_at" +1,12345,10001,3600,3600,"2024-01-15","09:00:00","Working on task","abc123","2024-01-15T10:00:00Z","2024-01-15T10:00:00Z" +1,12346,10002,7200,0,"2024-01-16","14:00:00","Internal meeting","abc123","2024-01-16T16:00:00Z","2024-01-16T16:00:00Z" diff --git a/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv new file mode 100644 index 00000000000..b93796906b3 --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv @@ -0,0 +1,3 @@ +"id","author_id","time_spent_minutes","issue_id","started_date","logged_date","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" +"tempo:TempoWorklog:12345","abc123",60,"jira:JiraIssues:1:10001","2024-01-15 09:00:00 +0000 UTC","2024-01-15 10:00:00 +0000 UTC","","0","" +"tempo:TempoWorklog:12346","abc123",120,"jira:JiraIssues:1:10002","2024-01-16 14:00:00 +0000 UTC","2024-01-16 16:00:00 +0000 UTC","","0","" \ No newline at end of file diff --git a/backend/plugins/tempo/e2e/worklog_test.go b/backend/plugins/tempo/e2e/worklog_test.go new file mode 100644 index 00000000000..5cd64556f2d --- /dev/null +++ b/backend/plugins/tempo/e2e/worklog_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/tempo/impl" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/tasks" +) + +func TestWorklogDataFlow(t *testing.T) { + var plugin impl.Tempo + dataflowTester := e2ehelper.NewDataFlowTester(t, "tempo", plugin) + + taskData := &tasks.TempoTaskData{ + Options: &tasks.TempoOptions{ + ConnectionId: 1, + }, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_tempo_worklogs.csv", &models.TempoWorklog{}) + dataflowTester.FlushTabler(&ticket.IssueWorklog{}) + dataflowTester.Subtask(tasks.ConvertWorklogsMeta, taskData) + dataflowTester.VerifyTable( + ticket.IssueWorklog{}, + "./snapshot_tables/issue_worklogs.csv", + []string{"id", "author_id", "time_spent_minutes", "issue_id", "started_date", "logged_date"}, + ) +} + +func TestTeamDataFlow(t *testing.T) { + var plugin impl.Tempo + dataflowTester := e2ehelper.NewDataFlowTester(t, "tempo", plugin) + + taskData := &tasks.TempoTaskData{ + Options: &tasks.TempoOptions{ + ConnectionId: 1, + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tempo_api_teams.csv", "_raw_tempo_api_teams") + dataflowTester.FlushTabler(&models.TempoTeam{}) + dataflowTester.Subtask(tasks.ExtractTeamsMeta, taskData) + dataflowTester.VerifyTable( + models.TempoTeam{}, + "./snapshot_tables/_tool_tempo_teams.csv", + e2ehelper.ColumnWithRawData( + "connection_id", + "team_id", + "name", + "summary", + "key", + ), + ) +} diff --git a/backend/plugins/tempo/impl/impl.go b/backend/plugins/tempo/impl/impl.go new file mode 100644 index 00000000000..cdc1bba0b49 --- /dev/null +++ b/backend/plugins/tempo/impl/impl.go @@ -0,0 +1,202 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/tempo/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginModel + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask + plugin.PluginSource +} = (*Tempo)(nil) + +type Tempo struct { +} + +func (p Tempo) Connection() dal.Tabler { + return &models.TempoConnection{} +} + +func (p Tempo) Scope() plugin.ToolLayerScope { + return &models.TempoTeam{} +} + +func (p Tempo) ScopeConfig() dal.Tabler { + return &models.TempoScopeConfig{} +} + +func (p Tempo) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Tempo) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.TempoConnection{}, + &models.TempoTeam{}, + &models.TempoWorklog{}, + &models.TempoScopeConfig{}, + } +} + +func (p Tempo) Description() string { + return "Collect worklogs from Jira Tempo" +} + +func (p Tempo) Name() string { + return "tempo" +} + +func (p Tempo) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectTeamsMeta, + tasks.ExtractTeamsMeta, + tasks.CollectWorklogsMeta, + tasks.ExtractWorklogsMeta, + tasks.ConvertWorklogsMeta, + } +} + +func (p Tempo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.TempoOptions + var err errors.Error + + logger := taskCtx.GetLogger() + logger.Debug("%v", options) + + err = helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "could not decode Tempo options") + } + + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("tempo connectionId is invalid") + } + + connection := &models.TempoConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Tempo connection") + } + + tempoApiClient, err := tasks.NewTempoApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create Tempo api client") + } + + taskData := &tasks.TempoTaskData{ + Options: &op, + ApiClient: tempoApiClient, + Connection: connection, + } + + return taskData, nil +} + +func (p Tempo) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Tempo) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/tempo" +} + +func (p Tempo) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Tempo) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/proxy/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/teams": { + "GET": api.GetTeams, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.UpdateScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScope, + }, + "connections/:connectionId/scope-configs": { + "POST": api.CreateScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.UpdateScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + } +} + +func (p Tempo) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.TempoTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + data.ApiClient.Release() + return nil +} diff --git a/backend/plugins/tempo/models/connection.go b/backend/plugins/tempo/models/connection.go new file mode 100644 index 00000000000..2322dc991a3 --- /dev/null +++ b/backend/plugins/tempo/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/common" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// TempoConn holds the essential information to connect to the Tempo API +type TempoConn struct { + helper.RestConnection `mapstructure:",squash"` + helper.AccessToken `mapstructure:",squash"` +} + +// TempoConnection holds TempoConn plus ID/Name for database storage +type TempoConnection struct { + helper.BaseConnection `mapstructure:",squash"` + TempoConn `mapstructure:",squash"` +} + +func (TempoConnection) TableName() string { + return "_tool_tempo_connections" +} + +func (connection TempoConnection) Connection() dal.Tabler { + return &connection +} + +func (connection TempoConnection) Sanitize() TempoConnection { + connection.TempoConn.Token = "" + return connection +} + +// This object conforms to what the frontend currently expects. +type TempoResponse struct { + Name string `json:"name"` + ID uint64 `json:"id"` + TempoConnection +} + +// TempoScopeConfig holds the configuration for a Tempo scope +type TempoScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` +} + +func (TempoScopeConfig) TableName() string { + return "_tool_tempo_scope_configs" +} + +func (c TempoScopeConfig) ScopeConfigId() uint64 { + return c.ID +} + +func (c TempoScopeConfig) ScopeConfigConnectionId() uint64 { + return c.ConnectionId +} diff --git a/backend/plugins/tempo/models/migrationscripts/register.go b/backend/plugins/tempo/models/migrationscripts/register.go new file mode 100644 index 00000000000..42e280a58b9 --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All returns all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(tempoInitTables20240401), + } +} diff --git a/backend/plugins/tempo/models/migrationscripts/register_tables.go b/backend/plugins/tempo/models/migrationscripts/register_tables.go new file mode 100644 index 00000000000..98bfdc47766 --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/register_tables.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type tempoInitTables20240401 struct{} + +func (script *tempoInitTables20240401) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.TempoConnection{}, + &models.TempoScopeConfig{}, + &models.TempoTeam{}, + &models.TempoWorklog{}, + ) +} + +func (*tempoInitTables20240401) Version() uint64 { + return 20240401143000 +} + +func (*tempoInitTables20240401) Name() string { + return "Tempo init schemas" +} diff --git a/backend/plugins/tempo/models/team.go b/backend/plugins/tempo/models/team.go new file mode 100644 index 00000000000..d29f3fd72de --- /dev/null +++ b/backend/plugins/tempo/models/team.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +var _ plugin.ToolLayerScope = (*TempoTeam)(nil) + +// TempoTeam represents a team in Jira Tempo +type TempoTeam struct { + common.Scope `mapstructure:",squash" gorm:"embedded"` + TeamId int64 `json:"teamId" mapstructure:"teamId" validate:"required" gorm:"primaryKey"` + Id int64 `json:"id" gorm:"-" mapstructure:"-"` // JS scope selector compatibility (mirrors TeamId) + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Summary string `json:"summary" mapstructure:"summary" gorm:"type:varchar(255)"` +} + +// AfterFind populates the virtual Id field after reading from DB +func (t *TempoTeam) AfterFind(_ *gorm.DB) error { + t.Id = t.TeamId + return nil +} + +func (t TempoTeam) ScopeId() string { + return fmt.Sprintf("%d", t.TeamId) +} + +func (t TempoTeam) ScopeName() string { + return t.Name +} + +func (t TempoTeam) ScopeFullName() string { + return fmt.Sprintf("%s - %s", t.Key, t.Name) +} + +func (t TempoTeam) ScopeParams() interface{} { + return &TempoApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (TempoTeam) TableName() string { + return "_tool_tempo_teams" +} + +// TempoApiParams holds the API parameters for Tempo teams +type TempoApiParams struct { + ConnectionId uint64 + TeamId int64 +} + +// TempoTeamResponse represents the API response for a team from Tempo API +type TempoTeamResponse struct { + Id int64 `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Summary string `json:"summary"` +} + +// ConvertToToolLayer converts the API response to the tool layer model +func (r TempoTeamResponse) ConvertToToolLayer(connectionId uint64) *TempoTeam { + return &TempoTeam{ + Scope: common.Scope{ + ConnectionId: connectionId, + }, + TeamId: r.Id, + Id: r.Id, + Key: r.Key, + Name: r.Name, + Summary: r.Summary, + } +} diff --git a/backend/plugins/tempo/models/worklog.go b/backend/plugins/tempo/models/worklog.go new file mode 100644 index 00000000000..780c69ee597 --- /dev/null +++ b/backend/plugins/tempo/models/worklog.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type TempoWorklog struct { + common.NoPKModel `mapstructure:",squash" gorm:"embedded"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey"` + TempoWorklogId int64 `json:"tempoWorklogId" mapstructure:"tempoWorklogId" gorm:"primaryKey"` + TeamId int64 `json:"teamId" mapstructure:"teamId" gorm:"index"` + IssueId int64 `json:"issueId" mapstructure:"issueId" gorm:"index"` + IssueKey string `json:"issueKey" mapstructure:"issueKey" gorm:"type:varchar(255)"` + AuthorAccountId string `json:"authorAccountId" mapstructure:"authorAccountId" gorm:"type:varchar(255)"` + TimeSpentSeconds int `json:"timeSpentSeconds" mapstructure:"timeSpentSeconds"` + BillableSeconds int `json:"billableSeconds" mapstructure:"billableSeconds"` + StartDate string `json:"startDate" mapstructure:"startDate" gorm:"type:varchar(255)"` + StartTime string `json:"startTime" mapstructure:"startTime" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description" gorm:"type:text"` + CreatedAt string `json:"createdAt" mapstructure:"createdAt" gorm:"type:varchar(255)"` + UpdatedAt string `json:"updatedAt" mapstructure:"updatedAt" gorm:"type:varchar(255)"` +} + +func (TempoWorklog) TableName() string { + return "_tool_tempo_worklogs" +} diff --git a/backend/plugins/tempo/tasks/api_client.go b/backend/plugins/tempo/tasks/api_client.go new file mode 100644 index 00000000000..8dcefb13472 --- /dev/null +++ b/backend/plugins/tempo/tasks/api_client.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +const ( + BaseURL = "https://api.tempo.io/4" +) + +// NewTempoApiClient creates a new Tempo API client +func NewTempoApiClient( + taskCtx plugin.TaskContext, + connection *models.TempoConnection, +) (*helper.ApiAsyncClient, errors.Error) { + apiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + return helper.CreateAsyncApiClient(taskCtx, apiClient, nil) +} diff --git a/backend/plugins/tempo/tasks/tasks.go b/backend/plugins/tempo/tasks/tasks.go new file mode 100644 index 00000000000..547282d6a5a --- /dev/null +++ b/backend/plugins/tempo/tasks/tasks.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +const ( + RAW_WORKLOG_TABLE = "tempo_api_worklogs" + RAW_TEAM_TABLE = "tempo_api_teams" +) + +// TempoOptions holds the options for the Tempo plugin +type TempoOptions struct { + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + ScopeConfigId uint64 `mapstructure:"scopeConfigId" json:"scopeConfigId"` + ScopeConfig *models.TempoScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"` + TeamId int64 `mapstructure:"teamId" json:"teamId"` + FromDate string `mapstructure:"fromDate" json:"fromDate"` + ToDate string `mapstructure:"toDate" json:"toDate"` +} + +// TempoTaskData holds the data for a Tempo task +type TempoTaskData struct { + Options *TempoOptions + ApiClient *helper.ApiAsyncClient + Connection *models.TempoConnection +} diff --git a/backend/plugins/tempo/tasks/team_collector.go b/backend/plugins/tempo/tasks/team_collector.go new file mode 100644 index 00000000000..1ac9e2f5bf5 --- /dev/null +++ b/backend/plugins/tempo/tasks/team_collector.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var CollectTeamsMeta = plugin.SubTaskMeta{ + Name: "collect-teams", + Description: "Collect teams from Tempo API", + EntryPoint: CollectTeams, + EnabledByDefault: true, + Dependencies: nil, + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectTeams(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: "teams", + PageSize: 50, + GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Total int `json:"total"` + Offset int `json:"offset"` + } `json:"metadata"` + } + if err := api.UnmarshalResponse(res, &response); err != nil { + return 0, err + } + totalPages := (response.Metadata.Total + args.PageSize - 1) / args.PageSize + return totalPages, nil + }, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + pager := reqData.Pager + if pager == nil { + pager = &api.Pager{Page: 1, Skip: 0, Size: 50} + } + query.Set("offset", strconv.Itoa(pager.Skip)) + query.Set("limit", strconv.Itoa(pager.Size)) + + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + since := apiCollector.GetSince() + query.Set("updatedFrom", since.Format(time.RFC3339)) + } + + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var response struct { + Results []json.RawMessage `json:"results"` + } + err := api.UnmarshalResponse(res, &response) + if err != nil { + return nil, err + } + return response.Results, nil + }, + }) + + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/tempo/tasks/team_extractor.go b/backend/plugins/tempo/tasks/team_extractor.go new file mode 100644 index 00000000000..1ecbc345c4b --- /dev/null +++ b/backend/plugins/tempo/tasks/team_extractor.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ExtractTeamsMeta = plugin.SubTaskMeta{ + Name: "extract_teams", + EntryPoint: ExtractTeams, + EnabledByDefault: true, + Description: "Extract teams from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +// API response for Tempo team +type TempoTeamResponse struct { + Id int64 `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Summary string `json:"summary"` + Self string `json:"self"` +} + +func ExtractTeams(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiTeam TempoTeamResponse + err := errors.Convert(json.Unmarshal(row.Data, &apiTeam)) + if err != nil { + return nil, err + } + + // Transform to tool layer model + team := &models.TempoTeam{ + TeamId: apiTeam.Id, + Key: apiTeam.Key, + Name: apiTeam.Name, + Summary: apiTeam.Summary, + } + team.ConnectionId = data.Options.ConnectionId + + return []interface{}{team}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tempo/tasks/worklog_collector.go b/backend/plugins/tempo/tasks/worklog_collector.go new file mode 100644 index 00000000000..28738e35240 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_collector.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var CollectWorklogsMeta = plugin.SubTaskMeta{ + Name: "collect_worklogs", + EntryPoint: CollectWorklogs, + EnabledByDefault: true, + Description: "Collect worklogs from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }) + if err != nil { + return err + } + + urlTemplate := "worklogs" + if data.Options.TeamId != 0 { + urlTemplate = fmt.Sprintf("worklogs/team/%d", data.Options.TeamId) + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: urlTemplate, + PageSize: 1000, + GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Total int `json:"total"` + Offset int `json:"offset"` + } `json:"metadata"` + } + if err := api.UnmarshalResponse(res, &response); err != nil { + return 0, err + } + totalPages := (response.Metadata.Total + args.PageSize - 1) / args.PageSize + return totalPages, nil + }, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + pager := reqData.Pager + if pager == nil { + pager = &api.Pager{Page: 1, Skip: 0, Size: 1000} + } + query.Set("offset", strconv.Itoa(pager.Skip)) + query.Set("limit", strconv.Itoa(pager.Size)) + + if data.Options.TeamId != 0 { + fromDate := data.Options.FromDate + toDate := data.Options.ToDate + if fromDate == "" { + since := time.Now().AddDate(0, 0, -90) + fromDate = since.Format("2006-01-02") + } + if toDate == "" { + toDate = time.Now().Format("2006-01-02") + } + query.Set("from", fromDate) + query.Set("to", toDate) + } else { + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + since := apiCollector.GetSince() + query.Set("updatedFrom", since.Format(time.RFC3339)) + } + } + + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var response struct { + Results []json.RawMessage `json:"results"` + } + err := api.UnmarshalResponse(res, &response) + if err != nil { + return nil, err + } + return response.Results, nil + }, + }) + + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/tempo/tasks/worklog_convertor.go b/backend/plugins/tempo/tasks/worklog_convertor.go new file mode 100644 index 00000000000..c83dabc567a --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_convertor.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ConvertWorklogsMeta = plugin.SubTaskMeta{ + Name: "convert_worklogs", + EntryPoint: ConvertWorklogs, + EnabledByDefault: true, + Description: "Convert worklogs to domain layer", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + connectionId := data.Options.ConnectionId + + logger.Info("converting Tempo worklogs to domain layer") + + issueIdMapping, err := buildIssueIdMapping(db, connectionId) + if err != nil { + return errors.Default.Wrap(err, "failed to build issue ID mapping") + } + + clauses := []dal.Clause{ + dal.Select("*"), + dal.From("_tool_tempo_worklogs"), + dal.Where("connection_id = ?", connectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return errors.Default.Wrap(err, "failed to query Tempo worklogs") + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: connectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + InputRowType: reflect.TypeOf(models.TempoWorklog{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + tempoWorklog := inputRow.(*models.TempoWorklog) + + domainIssueId := "" + if domainId, ok := issueIdMapping[tempoWorklog.IssueId]; ok { + domainIssueId = domainId + } else { + domainIssueId = fmt.Sprintf("jira:JiraIssues:%d:%d", connectionId, tempoWorklog.IssueId) + } + + domainWorklogId := fmt.Sprintf("tempo:TempoWorklog:%d", tempoWorklog.TempoWorklogId) + timeSpentMinutes := tempoWorklog.TimeSpentSeconds / 60 + + var startedDate *time.Time + if tempoWorklog.StartDate != "" && tempoWorklog.StartTime != "" { + if t, err := time.Parse("2006-01-02 15:04:05", tempoWorklog.StartDate+" "+tempoWorklog.StartTime); err == nil { + startedDate = &t + } + } + + var loggedDate *time.Time + if tempoWorklog.CreatedAt != "" { + if t, err := time.Parse(time.RFC3339, tempoWorklog.CreatedAt); err == nil { + loggedDate = &t + } + } + + worklog := &ticket.IssueWorklog{ + DomainEntity: domainlayer.DomainEntity{ + Id: domainWorklogId, + }, + IssueId: domainIssueId, + AuthorId: tempoWorklog.AuthorAccountId, + TimeSpentMinutes: timeSpentMinutes, + StartedDate: startedDate, + LoggedDate: loggedDate, + Comment: tempoWorklog.Description, + } + + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return errors.Default.Wrap(err, "failed to create data converter") + } + + return converter.Execute() +} + +func buildIssueIdMapping(db dal.Dal, connectionId uint64) (map[int64]string, errors.Error) { + mapping := make(map[int64]string) + + if !db.HasTable("_tool_jira_issues") { + return mapping, nil + } + + clauses := []dal.Clause{ + dal.Select("issue_id"), + dal.From("_tool_jira_issues"), + dal.Where("connection_id = ?", connectionId), + } + + rows, err := db.Cursor(clauses...) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query jira issues") + } + defer rows.Close() + + for rows.Next() { + var issueId uint64 + if err := rows.Scan(&issueId); err != nil { + return nil, errors.Default.Wrap(err, "failed to scan issue") + } + domainId := fmt.Sprintf("jira:JiraIssues:%d:%d", connectionId, issueId) + mapping[int64(issueId)] = domainId + } + + return mapping, nil +} diff --git a/backend/plugins/tempo/tasks/worklog_extractor.go b/backend/plugins/tempo/tasks/worklog_extractor.go new file mode 100644 index 00000000000..5e5055aca04 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_extractor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ExtractWorklogsMeta = plugin.SubTaskMeta{ + Name: "extract_worklogs", + EntryPoint: ExtractWorklogs, + EnabledByDefault: true, + Description: "Extract worklogs from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type TempoWorklogAuthor struct { + AccountId string `json:"accountId"` +} + +type TempoWorklogResponse struct { + TempoWorklogId int64 `json:"tempoWorklogId"` + Issue struct { + Id int64 `json:"id"` + } `json:"issue"` + TimeSpentSeconds int `json:"timeSpentSeconds"` + BillableSeconds int `json:"billableSeconds"` + StartDate string `json:"startDate"` + StartTime string `json:"startTime"` + Description string `json:"description"` + Author TempoWorklogAuthor `json:"author"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +func ExtractWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiWorklog TempoWorklogResponse + err := errors.Convert(json.Unmarshal(row.Data, &apiWorklog)) + if err != nil { + return nil, err + } + + worklog := &models.TempoWorklog{ + ConnectionId: data.Options.ConnectionId, + TempoWorklogId: apiWorklog.TempoWorklogId, + IssueId: apiWorklog.Issue.Id, + TimeSpentSeconds: apiWorklog.TimeSpentSeconds, + BillableSeconds: apiWorklog.BillableSeconds, + StartDate: apiWorklog.StartDate, + StartTime: apiWorklog.StartTime, + Description: apiWorklog.Description, + AuthorAccountId: apiWorklog.Author.AccountId, + CreatedAt: apiWorklog.CreatedAt, + UpdatedAt: apiWorklog.UpdatedAt, + } + + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tempo/tempo.go b/backend/plugins/tempo/tempo.go new file mode 100644 index 00000000000..8b75aee5e4d --- /dev/null +++ b/backend/plugins/tempo/tempo.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/tempo/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Tempo //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "tempo"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "tempo connection id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index fdc82a8778c..3153d306eaf 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -41,6 +41,7 @@ import { QDevConfig } from './q-dev'; import { TeambitionConfig } from './teambition'; import { TestmoConfig } from './testmo'; import { SlackConfig } from './slack/config'; +import { TempoConfig } from './tempo'; export const pluginConfigs: IPluginConfig[] = [ ArgoCDConfig, @@ -59,6 +60,7 @@ export const pluginConfigs: IPluginConfig[] = [ PagerDutyConfig, RootlyConfig, SlackConfig, + TempoConfig, QDevConfig, SonarQubeConfig, TAPDConfig, diff --git a/config-ui/src/plugins/register/tempo/assets/icon.png b/config-ui/src/plugins/register/tempo/assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..80f4e36f89794e5660d0152ae810bd7d7f0b6b99 GIT binary patch literal 4179 zcmV-Z5UlTsP) zeej({b;my&g9tGMBuZ=v(Fz8LBtgjwfrOYqz@ie~rH;yrBeR{Uwz{1<(`lz2XRLHq zXZWMDtrUnzDOHh*%wRPjA@3m}L;@kA1S2m3LEcOxh;08jd&$GieSdbJ=RWt|&&+Rz z+~2+DS$@Co@9f#LXU{4|bd0wC)4=(_XPi+llsug1j468nRY6;SF7R)_$AN3K^=~hE zNYNQv_5iAYw*EZeTR=?!7a)H~=a)991gNqH(C-8xECa?j`1bdaDdTsA4k{6&X`%%g zkI+a60OtYA24%_^d4?v07~$K%xDMX_ec;p1sNHEtm6lP=(146b=tKx09s(*JLySB_ z9l;~Cni^VB=@?=ph74U!4b8X(xU`Cb7>OYeNeDG=K|DmIVu(@9KuDiuQvb)F#z;RezO0c4eNeO2-hL+cqL#|Z7HL<}*Mo*_Yu(2nzg8@2Tl z3VL2o44oK)M_5L>si7HX0MiP7W>*Z=YNFt&Aup_wtgvTQ_RiyJ(6YYTOnxcCj)bVCmJnt2P)4Hgm5F}hLU51!aYKm_fG~+K}0Ye zIH*fA2Oz?^Kl@^(#V`ypg0}u0$S-^e68udE4v%1Z5pZ?(h04YZC3=L~)bO2@57E}2 z51b1uK<53oM1I6?fa9G}&(~h)*)wSCj{xR4qi%~avM!&!l=4!{NrezjdVdj^k?I+N zcLFB@chz3#Dd`l7G`dJI=YC}@uw)FpDUa~y2_INnc77@QVrFV6BSuJ) zM`*w~z`+GPCl8Cpg#GefR}I}#3o5?^psoKn<-`cdcOUDRQ_yp|#RIiY-B67NNPeT!Xwm*QnypQ=WE9Ez$Rcl@KfNvT1~@P;G0MV&6ENM^fqL3 zrf7aofkFsOrTERYEW3dRfqQ_vfd`yXI~%=mtTXB^ZT&?^cab9t8)EU)VuYfmh8jl` zze=eUuL8dYRv-e{n}v+j>)M;St3Qp2w93qGCc)s0G1=}f4ej4 zrEZP9QQYE;S{FRT_kcf*HOfAqkmX|w4ohTrJ#Ysy??3E}dM2;IH&r?XL0kZQm%*4a z%7_t);-8#RKZN6QR02ipHFUxYiR{jgZLr}+l{0ERa3K`GvF9Fa4dZOh(*hq#db7etYJ4ISf*dJf6>rQr0K#yTp9t$-lPUOu*C zLn6CT58HHHDxVQVzr_$WA@tMKP|w|o>`v3xe;}5zt`5=k;j*(IV}wRL0Q@|G-El;$ zlj!=aWgkr$eHkM(B3zaHKqB)?Vj1e`;B64ZMU*Fqevc6vaaa4d^u&qU`Y5}iJ-7Y~ z%g&1^K8Dx@Tu_0jp`MjUGAuSzkOm&P$<+rk2;yVF>KKFRfX9GOIHT?;=y}m_Mx}}y z#sVLXVJLS~P-oO;q;~Q@f!A`6s5$=&%yLHEneZ8@N%cn)<%Cw$Ha^<=`M^IQN{!Vo zY76jnXVkYN*((ig{Uu1@acmw({!o6C$4Jx$U(TqNz!Gi!0^n~DQS@>Z&mQ1D;2LMt z4H1V}Hfw<20`G`y_ekVKcdqF(X@jrOgWyc0^7=SnB5){D2Bwhe+&?0}{O5p&fu8_B zc1Asza%^Rxt-l*-yBC|UJEL+Pc23&*FBD#06TZ;azZWj|a{psa z){{mIxsUi)w)J0C(bk_uPIumfmw*$UQIGa&U?8s;=yNubrfBF8q-$)?jKokd&Zu38 zAX1R4Kh#JJ<*+`H-I;klJQ71WEGJEZZmPDv=cZvJF_gm&B()x!gUGQ`8Hu3;oKe38 z?oDKVcCQ~CiJ?4FZE2qzI`DMONDSq%8p#dBW(v?#Dcnd5CE<+PhQtoBQNS6!d~hU& za#@+k?y0?esH1eWwtgmXJ@624J96efMWMz}5jx!6n!xnaz)8-iuF4WS-KhF7lF*n5 z`~~omw*C>||A2MC>JTB1=#G1lGQNqC?H&Pqh%ReXI~7BH&ijz?un!^I&`)dYw*hw| zfzg)i-A5xDzZs3F6 z#CDxb6Q`4Jpl=0^4POAyYwNco`T14A<`^w#D$GMjz&9zf-El}mkEmjp*Yj{1a~RTU za4ynh{5fs?Dx^MeEwF8vf>CGGGurxlks}AOnFU6=@ML0kWB(%zUd25AaR%5Je1& za}Osj6Og?78Ax%+)0rjg8fR3h4k<}-M!g*3hS*|&E?WIGohG#P??;w`hejAs5xfp; zMY4J;fsIv)Hxiz<+KHs5+j`Jep5YX#BLt9Obb9!@0@$Oi--;x?Rv`I}T*uE6vjy4G zj@^C!D8%Em6+E2l(TzmeA2-d_28XVh2Qy{VyR z2+benMjujnWK-r#kSS$H_~EZYrj=hfqf+!w=!O-f5yKMX=De0P^bFzt#Z?sB8ox~T zA{T^gLK<_gXm%$204r>vp4 zE&2_}FYfTjb`M9=1TDqTtkS8cvZ(=LB9eLiGIEpsMs5AIp~xp0VUK2WBJ(G;d&7QW z(AK|`eVa-ng!clM1FN+4Kg@Gbr2|rRrdinT4UIg*EZ{w{M=~Ic13n6@*4F=vwm!ur zWGj)?Pi#)m)}PSo4UNQbENQx+4-y9Mc)ccUm1iA2J? zL>#k`DP@?%ag#IZlX;D3>;D7zT!it|bGtLDVO3S?)_)U_X=FAMyv#%ncntcnzRR6a z*XJ>ixl%GVPa+LO_q#}~&>%2e5`H|=UU&|Y zZ`5|V)D?lY{ut!=_(za{^eAKvKL}jmjJht50d4(S((E38#ToUF^)DuE3fc~5)FVjH zbt{0jej3t-W`6iOf|AFWgTDlFi2>O`O(BMcUFiEZR}i|7oPda95i-3@q&o4&Uf@(` zRL^bxwDso!x5OLL8@vomcSb#yeX(!<>cYX@d&1XOwDm_KQ_2~L*Ek;8E9h4aM3QuQ zdOofNo&nw!WkmZJkE8=0%f8sRo}nGu`aeT<;!Z{4jmgA6caWF0&Zt}`(zNxr17}AX zPnPSPQCDPN=v&Xwj?g3dQQ$gl{dk}zjx!OjaTMi?{r%edNjVE~e@Z+t`~fM9{TcA1 z`fpZN3|X8}FC)vjEx^?wzcCvLzRp7G5#C+G&?X>vLF5rb-qym;B3C!8A`}BR>zy$W zVyMR%^&)a_#ZQoUV-iyOzX++GQqa~Pi-=<}QV#YW(xuF=bVhwMj{$A{4Zx?GeET)vDdY&Y-kqGmxI4 z9nPqS5!ZPwyipB)3Gn#@kBjj$Ue7);ypCMhavyT6@Gf9$m@0EIlwzpI8MRJZKdbC< zwBT-Js#%2ueh)jNeii#6BQfkFl^AyAH4x5>CKqc}vUy1f+0NJt_chFJ{rNmJ1BxHBr%#pg*GW-+8%QdUw3Fr;G8)*niaJEPlBxg}+dp%eqsLt=6Q zyRVY&!t>+|s~AorP1DmY;acNLGOS|QfgDcRuSY>nD(e+sNK55wga)YdkzNtC^P7jz z`_JVZXVh(xhF3U4Erxo+`OQ<2V|sIeqv?G4`7CnDM`g;~>KTcl1>rnJ&1=j=j_Ea< zTJLd2omSA$k}wiOhlJ(m3`7_Ukm{(z+4nR2vNP(caz+=2kr;X;OewRF{!}LcUvNe( dFLzY=_rMaw002ovPDHLkV1g^y*1rG% literal 0 HcmV?d00001 diff --git a/config-ui/src/plugins/register/tempo/assets/icon.svg b/config-ui/src/plugins/register/tempo/assets/icon.svg new file mode 100644 index 00000000000..6c7407b2b4f --- /dev/null +++ b/config-ui/src/plugins/register/tempo/assets/icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/config-ui/src/plugins/register/tempo/config.tsx b/config-ui/src/plugins/register/tempo/config.tsx new file mode 100644 index 00000000000..c4494d13908 --- /dev/null +++ b/config-ui/src/plugins/register/tempo/config.tsx @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { DOC_URL } from '@/release'; +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.png'; + +export const TempoConfig: IPluginConfig = { + plugin: 'tempo', + name: 'Tempo', + icon: () => , + sort: 20, + connection: { + docLink: 'https://devlake.apache.org/docs/Configuration/Tempo', + initialValues: { + endpoint: 'https://api.tempo.io/4', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'REST Endpoint', + subLabel: 'Tempo API v4 base URL', + placeholder: 'https://api.tempo.io/4', + }, + 'token', + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for default.', + defaultValue: 1000, + }, + ], + }, + dataScope: { + title: 'Teams', + millerColumn: { + columnCount: 2.5, + }, + }, + scopeConfig: { + entities: ['TICKET'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/tempo/index.ts b/config-ui/src/plugins/register/tempo/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/tempo/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; From 0a4c26d7556199835901099086cf120684f1588f Mon Sep 17 00:00:00 2001 From: acarmisc Date: Sat, 30 May 2026 12:23:30 +0200 Subject: [PATCH 2/2] fix(tempo): resolve CI failures on tempo plugin PR - fix Apache license headers in 5 files (missing/typo content) - move migration script to archived models pattern (forbidden import on plugins/tempo/models) - register tempo in plugins/table_info_test.go - drop unused validator var/import in api/init.go - fix malformed issue_worklogs.csv snapshot (header/body column mismatch) --- backend/plugins/table_info_test.go | 2 + backend/plugins/tempo/api/init.go | 4 - .../e2e/snapshot_tables/issue_worklogs.csv | 6 +- backend/plugins/tempo/e2e/worklog_test.go | 2 +- backend/plugins/tempo/impl/impl.go | 8 +- .../migrationscripts/archived/models.go | 77 +++++++++++++++++++ .../migrationscripts/register_tables.go | 14 ++-- .../plugins/tempo/tasks/worklog_convertor.go | 2 +- .../plugins/tempo/tasks/worklog_extractor.go | 2 +- 9 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 backend/plugins/tempo/models/migrationscripts/archived/models.go diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index bf187ee1df1..d273d25e065 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -56,6 +56,7 @@ import ( taiga "github.com/apache/incubator-devlake/plugins/taiga/impl" tapd "github.com/apache/incubator-devlake/plugins/tapd/impl" teambition "github.com/apache/incubator-devlake/plugins/teambition/impl" + tempo "github.com/apache/incubator-devlake/plugins/tempo/impl" testmo "github.com/apache/incubator-devlake/plugins/testmo/impl" trello "github.com/apache/incubator-devlake/plugins/trello/impl" webhook "github.com/apache/incubator-devlake/plugins/webhook/impl" @@ -96,6 +97,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("taiga/models", taiga.Taiga{}.GetTablesInfo) checker.FeedIn("tapd/models", tapd.Tapd{}.GetTablesInfo) checker.FeedIn("teambition/models", teambition.Teambition{}.GetTablesInfo) + checker.FeedIn("tempo/models", tempo.Tempo{}.GetTablesInfo) checker.FeedIn("testmo/models", testmo.Testmo{}.GetTablesInfo) checker.FeedIn("trello/models", trello.Trello{}.GetTablesInfo) checker.FeedIn("webhook/models", webhook.Webhook{}.GetTablesInfo) diff --git a/backend/plugins/tempo/api/init.go b/backend/plugins/tempo/api/init.go index 1313a8bb9f8..47ce280c2fe 100644 --- a/backend/plugins/tempo/api/init.go +++ b/backend/plugins/tempo/api/init.go @@ -22,11 +22,8 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/tempo/models" - "github.com/go-playground/validator/v10" ) -var vld *validator.Validate - var basicRes context.BasicRes var dsHelper *api.DsHelper[models.TempoConnection, models.TempoTeam, models.TempoScopeConfig] var raProxy *api.DsRemoteApiProxyHelper[models.TempoConnection] @@ -35,7 +32,6 @@ var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.TempoConnection, mode func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br - vld = validator.New() dsHelper = api.NewDataSourceHelper[ models.TempoConnection, models.TempoTeam, diff --git a/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv index b93796906b3..a100e9640dd 100644 --- a/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv +++ b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv @@ -1,3 +1,3 @@ -"id","author_id","time_spent_minutes","issue_id","started_date","logged_date","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" -"tempo:TempoWorklog:12345","abc123",60,"jira:JiraIssues:1:10001","2024-01-15 09:00:00 +0000 UTC","2024-01-15 10:00:00 +0000 UTC","","0","" -"tempo:TempoWorklog:12346","abc123",120,"jira:JiraIssues:1:10002","2024-01-16 14:00:00 +0000 UTC","2024-01-16 16:00:00 +0000 UTC","","0","" \ No newline at end of file +id,author_id,comment,time_spent_minutes,logged_date,started_date,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +tempo:TempoWorklog:12345,abc123,Working on task,60,2024-01-15T10:00:00.000+00:00,2024-01-15T09:00:00.000+00:00,jira:JiraIssues:1:10001,,,0, +tempo:TempoWorklog:12346,abc123,Internal meeting,120,2024-01-16T16:00:00.000+00:00,2024-01-16T14:00:00.000+00:00,jira:JiraIssues:1:10002,,,0, diff --git a/backend/plugins/tempo/e2e/worklog_test.go b/backend/plugins/tempo/e2e/worklog_test.go index 5cd64556f2d..bb4ea262bf5 100644 --- a/backend/plugins/tempo/e2e/worklog_test.go +++ b/backend/plugins/tempo/e2e/worklog_test.go @@ -4,7 +4,7 @@ contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -the License. You may obtain the License at +the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 diff --git a/backend/plugins/tempo/impl/impl.go b/backend/plugins/tempo/impl/impl.go index cdc1bba0b49..0666033a7fb 100644 --- a/backend/plugins/tempo/impl/impl.go +++ b/backend/plugins/tempo/impl/impl.go @@ -1,11 +1,13 @@ /* Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with +contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/backend/plugins/tempo/models/migrationscripts/archived/models.go b/backend/plugins/tempo/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..76f0a9edd8d --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/archived/models.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type TempoConnection struct { + archived.BaseConnection `mapstructure:",squash"` + archived.RestConnection `mapstructure:",squash"` + archived.AccessToken `mapstructure:",squash"` +} + +func (TempoConnection) TableName() string { + return "_tool_tempo_connections" +} + +type TempoScopeConfig struct { + archived.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + Name string `gorm:"type:varchar(255)" mapstructure:"name" json:"name"` +} + +func (TempoScopeConfig) TableName() string { + return "_tool_tempo_scope_configs" +} + +type TempoTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + ScopeConfigId uint64 + Key string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Summary string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (TempoTeam) TableName() string { + return "_tool_tempo_teams" +} + +type TempoWorklog struct { + ConnectionId uint64 `gorm:"primaryKey"` + TempoWorklogId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"index"` + IssueId int64 `gorm:"index"` + IssueKey string `gorm:"type:varchar(255)"` + AuthorAccountId string `gorm:"type:varchar(255)"` + TimeSpentSeconds int + BillableSeconds int + StartDate string `gorm:"type:varchar(255)"` + StartTime string `gorm:"type:varchar(255)"` + Description string `gorm:"type:text"` + CreatedAt string `gorm:"type:varchar(255)"` + UpdatedAt string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (TempoWorklog) TableName() string { + return "_tool_tempo_worklogs" +} diff --git a/backend/plugins/tempo/models/migrationscripts/register_tables.go b/backend/plugins/tempo/models/migrationscripts/register_tables.go index 98bfdc47766..ee4e7c8ddb3 100644 --- a/backend/plugins/tempo/models/migrationscripts/register_tables.go +++ b/backend/plugins/tempo/models/migrationscripts/register_tables.go @@ -4,12 +4,12 @@ contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -the License. You may obtain the License at +the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS IS" BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -21,7 +21,7 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/helpers/migrationhelper" - "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/models/migrationscripts/archived" ) type tempoInitTables20240401 struct{} @@ -29,10 +29,10 @@ type tempoInitTables20240401 struct{} func (script *tempoInitTables20240401) Up(basicRes context.BasicRes) errors.Error { return migrationhelper.AutoMigrateTables( basicRes, - &models.TempoConnection{}, - &models.TempoScopeConfig{}, - &models.TempoTeam{}, - &models.TempoWorklog{}, + &archived.TempoConnection{}, + &archived.TempoScopeConfig{}, + &archived.TempoTeam{}, + &archived.TempoWorklog{}, ) } diff --git a/backend/plugins/tempo/tasks/worklog_convertor.go b/backend/plugins/tempo/tasks/worklog_convertor.go index c83dabc567a..60ee32a1416 100644 --- a/backend/plugins/tempo/tasks/worklog_convertor.go +++ b/backend/plugins/tempo/tasks/worklog_convertor.go @@ -4,7 +4,7 @@ contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at +the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 diff --git a/backend/plugins/tempo/tasks/worklog_extractor.go b/backend/plugins/tempo/tasks/worklog_extractor.go index 5e5055aca04..31370868d38 100644 --- a/backend/plugins/tempo/tasks/worklog_extractor.go +++ b/backend/plugins/tempo/tasks/worklog_extractor.go @@ -4,7 +4,7 @@ contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at +the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0