From e1f7f937e0a9eadc9939a78c7a2f1f4ecee523e9 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 2 Feb 2024 15:58:50 +0000 Subject: [PATCH 01/17] feat(hooks): trigger on pull-request Signed-off-by: Yvonnick Esnault --- engine/hooks/scheduler_repository_event.go | 4 +- engine/hooks/workflow_hook.go | 47 ++++++++++++++++++++++ sdk/interpolate/interpolate.go | 3 ++ sdk/interpolate/interpolate_helper.go | 5 ++- 4 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 engine/hooks/workflow_hook.go diff --git a/engine/hooks/scheduler_repository_event.go b/engine/hooks/scheduler_repository_event.go index 55d669a91c..393b4b87a9 100644 --- a/engine/hooks/scheduler_repository_event.go +++ b/engine/hooks/scheduler_repository_event.go @@ -170,7 +170,7 @@ func (s *Service) executeEvent(ctx context.Context, hre *sdk.HookRepositoryEvent // analyze have to be trigger only on push event hre.Status = sdk.HookEventStatusAnalysis if err := s.triggerAnalyses(ctx, hre); err != nil { - return sdk.WrapError(err, "unable to trigger analyses") + return sdk.WrapError(err, "unable to trigger analysis") } } else { hre.Status = sdk.HookEventStatusWorkflowHooks @@ -182,7 +182,7 @@ func (s *Service) executeEvent(ctx context.Context, hre *sdk.HookRepositoryEvent // Check if all analysis are ended case sdk.HookEventStatusAnalysis: if err := s.triggerAnalyses(ctx, hre); err != nil { - return sdk.WrapError(err, "unable to trigger analyses") + return sdk.WrapError(err, "unable to trigger analysis") } // Check if all workflow triggered has been sent case sdk.HookEventStatusWorkflowHooks: diff --git a/engine/hooks/workflow_hook.go b/engine/hooks/workflow_hook.go new file mode 100644 index 0000000000..8b9d3f58da --- /dev/null +++ b/engine/hooks/workflow_hook.go @@ -0,0 +1,47 @@ +package hooks + +import "github.com/ovh/cds/sdk" + +func GetWorkflowHookEventFromBitbucketEvent(event string) string { + switch event { + case "repo:refs_changed": + return sdk.WorkflowHookEventPush + case "pr:opened", "pr:from_ref_updated": + return sdk.WorkflowHookEventPullRequest + default: + return "" + } +} + +func GetWorkflowHookEventFromGithubEvent(event string) string { + switch event { + case "push": + return sdk.WorkflowHookEventPush + case "pull_request": + return sdk.WorkflowHookEventPullRequest + default: + return "" + } +} + +func GetWorkflowHookEventFromGiteaEvent(event string) string { + switch event { + case "push": + return sdk.WorkflowHookEventPush + case "pull_request": + return sdk.WorkflowHookEventPullRequest + default: + return "" + } +} + +func GetWorkflowHookEventFromGitlabEvent(event string) string { + switch event { + case "Push Hook": + return sdk.WorkflowHookEventPush + case "Merge Request Hook": + return sdk.WorkflowHookEventPullRequest + default: + return "" + } +} diff --git a/sdk/interpolate/interpolate.go b/sdk/interpolate/interpolate.go index 5c34c4776c..bd2126ecbb 100644 --- a/sdk/interpolate/interpolate.go +++ b/sdk/interpolate/interpolate.go @@ -94,6 +94,9 @@ func Do(input string, vars map[string]string) (string, error) { } processedExpression[expression] = void{} + if strings.HasPrefix(expression, "${{") { + continue + } var quotedStuff = []string{} var trimmedExpression = strings.TrimPrefix(expression, "{{") trimmedExpression = strings.TrimSuffix(trimmedExpression, "}}") diff --git a/sdk/interpolate/interpolate_helper.go b/sdk/interpolate/interpolate_helper.go index dde2940608..793fe70e78 100644 --- a/sdk/interpolate/interpolate_helper.go +++ b/sdk/interpolate/interpolate_helper.go @@ -211,8 +211,9 @@ func coalesce(v ...interface{}) interface{} { // toJSON encodes an item into a JSON string func toJSON(v interface{}) string { - output, _ := json.Marshal(v) - return string(output) + // output, _ := json.Marshal(v) + // return string(output) + return "" } // toPrettyJSON encodes an item into a pretty (indented) JSON string From f84130f31700bfa50e8e7f8de7d32421d3977bf8 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 2 Feb 2024 16:07:57 +0000 Subject: [PATCH 02/17] fix: cr Signed-off-by: Yvonnick Esnault --- engine/hooks/scheduler_repository_event.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/hooks/scheduler_repository_event.go b/engine/hooks/scheduler_repository_event.go index 393b4b87a9..55d669a91c 100644 --- a/engine/hooks/scheduler_repository_event.go +++ b/engine/hooks/scheduler_repository_event.go @@ -170,7 +170,7 @@ func (s *Service) executeEvent(ctx context.Context, hre *sdk.HookRepositoryEvent // analyze have to be trigger only on push event hre.Status = sdk.HookEventStatusAnalysis if err := s.triggerAnalyses(ctx, hre); err != nil { - return sdk.WrapError(err, "unable to trigger analysis") + return sdk.WrapError(err, "unable to trigger analyses") } } else { hre.Status = sdk.HookEventStatusWorkflowHooks @@ -182,7 +182,7 @@ func (s *Service) executeEvent(ctx context.Context, hre *sdk.HookRepositoryEvent // Check if all analysis are ended case sdk.HookEventStatusAnalysis: if err := s.triggerAnalyses(ctx, hre); err != nil { - return sdk.WrapError(err, "unable to trigger analysis") + return sdk.WrapError(err, "unable to trigger analyses") } // Check if all workflow triggered has been sent case sdk.HookEventStatusWorkflowHooks: From 855dc0848f92b1aeade209004f745a7d5a0b8069 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 2 Feb 2024 16:10:45 +0000 Subject: [PATCH 03/17] fix: cr Signed-off-by: Yvonnick Esnault --- sdk/interpolate/interpolate_helper.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/interpolate/interpolate_helper.go b/sdk/interpolate/interpolate_helper.go index 793fe70e78..dde2940608 100644 --- a/sdk/interpolate/interpolate_helper.go +++ b/sdk/interpolate/interpolate_helper.go @@ -211,9 +211,8 @@ func coalesce(v ...interface{}) interface{} { // toJSON encodes an item into a JSON string func toJSON(v interface{}) string { - // output, _ := json.Marshal(v) - // return string(output) - return "" + output, _ := json.Marshal(v) + return string(output) } // toPrettyJSON encodes an item into a pretty (indented) JSON string From bd4e119eb5327b03f8a417790a5f141a54ae2c3c Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 2 Feb 2024 16:13:42 +0000 Subject: [PATCH 04/17] fix: cr Signed-off-by: Yvonnick Esnault --- sdk/interpolate/interpolate.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdk/interpolate/interpolate.go b/sdk/interpolate/interpolate.go index bd2126ecbb..5c34c4776c 100644 --- a/sdk/interpolate/interpolate.go +++ b/sdk/interpolate/interpolate.go @@ -94,9 +94,6 @@ func Do(input string, vars map[string]string) (string, error) { } processedExpression[expression] = void{} - if strings.HasPrefix(expression, "${{") { - continue - } var quotedStuff = []string{} var trimmedExpression = strings.TrimPrefix(expression, "{{") trimmedExpression = strings.TrimSuffix(trimmedExpression, "}}") From 1b50d888a2a7da0f147dd21a9f3648efa84c523a Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Feb 2024 15:55:50 +0000 Subject: [PATCH 05/17] fix: cr Signed-off-by: Yvonnick Esnault --- engine/hooks/workflow_hook.go | 47 ----------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 engine/hooks/workflow_hook.go diff --git a/engine/hooks/workflow_hook.go b/engine/hooks/workflow_hook.go deleted file mode 100644 index 8b9d3f58da..0000000000 --- a/engine/hooks/workflow_hook.go +++ /dev/null @@ -1,47 +0,0 @@ -package hooks - -import "github.com/ovh/cds/sdk" - -func GetWorkflowHookEventFromBitbucketEvent(event string) string { - switch event { - case "repo:refs_changed": - return sdk.WorkflowHookEventPush - case "pr:opened", "pr:from_ref_updated": - return sdk.WorkflowHookEventPullRequest - default: - return "" - } -} - -func GetWorkflowHookEventFromGithubEvent(event string) string { - switch event { - case "push": - return sdk.WorkflowHookEventPush - case "pull_request": - return sdk.WorkflowHookEventPullRequest - default: - return "" - } -} - -func GetWorkflowHookEventFromGiteaEvent(event string) string { - switch event { - case "push": - return sdk.WorkflowHookEventPush - case "pull_request": - return sdk.WorkflowHookEventPullRequest - default: - return "" - } -} - -func GetWorkflowHookEventFromGitlabEvent(event string) string { - switch event { - case "Push Hook": - return sdk.WorkflowHookEventPush - case "Merge Request Hook": - return sdk.WorkflowHookEventPullRequest - default: - return "" - } -} From b51e85cc64132d2bd8d8e4078076b9a00294c0d9 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Wed, 7 Feb 2024 15:04:20 +0000 Subject: [PATCH 06/17] feat: send build status / comment v2 and refact old way Signed-off-by: Yvonnick Esnault --- engine/api/admin.go | 2 +- engine/api/api.go | 2 +- engine/api/application_test.go | 3 +- engine/api/ascode.go | 2 +- engine/api/ascode/pull_request.go | 5 +- engine/api/ascode/sync.go | 2 +- engine/api/environment_ascode_test.go | 3 +- engine/api/event/elasticsearch.go | 4 +- engine/api/event_v2/event.go | 167 ++++++++++++------ engine/api/metrics/elasticsearch.go | 4 +- engine/api/operation/operation.go | 8 +- engine/api/purge/purge.go | 2 +- engine/api/purge/purge_run.go | 11 +- engine/api/purge/purge_test.go | 5 +- engine/api/repositoriesmanager/events.go | 28 +-- .../repositories_manager.go | 57 +----- engine/api/services/http.go | 7 +- engine/api/v2_hooks.go | 4 +- engine/api/v2_hooks_test.go | 3 +- engine/api/v2_project_clean_ascode_test.go | 3 +- engine/api/v2_project_repository.go | 2 +- engine/api/v2_project_repository_test.go | 10 +- engine/api/v2_project_vcs_test.go | 7 +- engine/api/v2_repository_analyze.go | 2 +- engine/api/v2_repository_analyze_test.go | 19 +- engine/api/v2_workflow_run.go | 2 +- engine/api/v2_workflow_run_craft_test.go | 3 +- engine/api/v2_workflow_run_test.go | 9 +- engine/api/workflow.go | 2 +- engine/api/workflow/execute_node_run.go | 2 +- engine/api/workflow/hook.go | 4 +- engine/api/workflow/process_outgoinghook.go | 2 +- engine/api/workflow/workflow_run_event.go | 24 ++- .../api/workflow/workflow_run_event_test.go | 21 ++- engine/api/workflow_ascode_rename_test.go | 3 +- engine/api/workflow_ascode_test.go | 6 +- engine/api/workflow_ascode_with_hooks_test.go | 9 +- .../api/workflow_ascode_with_secrets_test.go | 3 +- engine/api/workflow_event.go | 2 +- engine/api/workflow_export_test.go | 12 +- engine/api/workflow_run.go | 6 +- engine/api/workflow_test.go | 5 +- engine/vcs/bitbucketcloud/bitbucketcloud.go | 13 +- engine/vcs/bitbucketcloud/client_status.go | 120 ++++--------- engine/vcs/bitbucketserver/bitbucketserver.go | 9 +- engine/vcs/bitbucketserver/client_status.go | 69 +------- engine/vcs/gerrit/client_status.go | 39 ++-- engine/vcs/gerrit/gerrit.go | 13 +- engine/vcs/gitea/client_status.go | 6 +- engine/vcs/github/client_status.go | 39 +--- engine/vcs/github/github.go | 21 ++- engine/vcs/gitlab/client_status.go | 67 ++----- engine/vcs/gitlab/gitlab.go | 9 +- engine/vcs/vcs_handlers.go | 14 +- engine/vcs/vcs_router.go | 2 + sdk/vcs.go | 29 ++- 56 files changed, 376 insertions(+), 551 deletions(-) diff --git a/engine/api/admin.go b/engine/api/admin.go index 29f83a2b63..3712685b77 100644 --- a/engine/api/admin.go +++ b/engine/api/admin.go @@ -124,7 +124,7 @@ func (api *API) postMaintenanceHandler() service.Handler { return err } url := fmt.Sprintf("/admin/maintenance?enable=%v", enable) - _, code, errHooks := services.NewClient(api.mustDB(), srvs).DoJSONRequest(ctx, http.MethodPost, url, nil, nil) + _, code, errHooks := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodPost, url, nil, nil) if errHooks != nil || code >= 400 { return fmt.Errorf("unable to change hook maintenant state to %v. Code result %d: %v", enable, code, errHooks) } diff --git a/engine/api/api.go b/engine/api/api.go index 66b68935a2..c9f4e4a958 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -851,7 +851,7 @@ func (a *API) Serve(ctx context.Context) error { auditCleanerRoutine(ctx, a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper)) }) a.GoRoutines.RunWithRestart(ctx, "repositoriesmanager.ReceiveEvents", func(ctx context.Context) { - repositoriesmanager.ReceiveEvents(ctx, a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper), a.Cache) + repositoriesmanager.ReceiveEvents(ctx, a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper), a.Cache, a.Config.URL.UI) }) a.GoRoutines.RunWithRestart(ctx, "services.KillDeadServices", func(ctx context.Context) { services.KillDeadServices(ctx, a.mustDB) diff --git a/engine/api/application_test.go b/engine/api/application_test.go index 45d712e9e0..f887a07406 100644 --- a/engine/api/application_test.go +++ b/engine/api/application_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -115,7 +114,7 @@ func TestUpdateAsCodeApplicationHandler(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/ascode.go b/engine/api/ascode.go index 3ee62210f3..9988c52412 100644 --- a/engine/api/ascode.go +++ b/engine/api/ascode.go @@ -59,7 +59,7 @@ func (api *API) postImportAsCodeHandler() service.Handler { return sdk.WrapError(err, "cannot load project") } - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, p.Key, ope.VCSServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, p.Key, ope.VCSServer) if err != nil { return sdk.NewErrorWithStack(err, sdk.NewErrorFrom(sdk.ErrNoReposManagerClientAuth, "cannot get client for %s %s", key, ope.VCSServer)) diff --git a/engine/api/ascode/pull_request.go b/engine/api/ascode/pull_request.go index 896cbf2b41..652d2cca8d 100644 --- a/engine/api/ascode/pull_request.go +++ b/engine/api/ascode/pull_request.go @@ -12,7 +12,6 @@ import ( "github.com/ovh/cds/engine/api/operation" "github.com/ovh/cds/engine/api/repositoriesmanager" "github.com/ovh/cds/engine/cache" - "github.com/ovh/cds/engine/gorpmapper" "github.com/ovh/cds/sdk" ) @@ -56,7 +55,7 @@ func UpdateAsCodeResult(ctx context.Context, db *gorp.DbMap, store cache.Store, } defer tx.Rollback() // nolint - asCodeEvent, err = createPullRequest(ctx, tx, store, proj, workflowHolder.ID, rootApp, ed, u, ope.Setup) + asCodeEvent, err = createPullRequest(ctx, db, store, proj, workflowHolder.ID, rootApp, ed, u, ope.Setup) if err != nil { return err } @@ -104,7 +103,7 @@ func UpdateAsCodeResult(ctx context.Context, db *gorp.DbMap, store cache.Store, }) } -func createPullRequest(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store cache.Store, proj sdk.Project, workflowHolderID int64, rootApp sdk.Application, ed EntityData, u sdk.Identifiable, opeSetup sdk.OperationSetup) (*sdk.AsCodeEvent, error) { +func createPullRequest(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk.Project, workflowHolderID int64, rootApp sdk.Application, ed EntityData, u sdk.Identifiable, opeSetup sdk.OperationSetup) (*sdk.AsCodeEvent, error) { client, err := repositoriesmanager.AuthorizedClient(ctx, db, store, proj.Key, rootApp.VCSServer) if err != nil { return nil, sdk.NewErrorFrom(err, "unable to create repositories manager client") diff --git a/engine/api/ascode/sync.go b/engine/api/ascode/sync.go index ec49ab7573..7194957d9b 100644 --- a/engine/api/ascode/sync.go +++ b/engine/api/ascode/sync.go @@ -33,7 +33,7 @@ func SyncEvents(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk } defer tx.Rollback() //nolint - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, store, proj.Key, rootApp.VCSServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, db, store, proj.Key, rootApp.VCSServer) if err != nil { return res, err } diff --git a/engine/api/environment_ascode_test.go b/engine/api/environment_ascode_test.go index 63f5385150..9cd8c63879 100644 --- a/engine/api/environment_ascode_test.go +++ b/engine/api/environment_ascode_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -62,7 +61,7 @@ func TestUpdateAsCodeEnvironmentHandler(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/event/elasticsearch.go b/engine/api/event/elasticsearch.go index 009dd22fec..a6c65a4065 100644 --- a/engine/api/event/elasticsearch.go +++ b/engine/api/event/elasticsearch.go @@ -42,7 +42,7 @@ func PushInElasticSearch(ctx context.Context, db gorp.SqlExecutor, store cache.S } e.Payload = nil log.Info(ctx, "sending event %q to %s services", e.EventType, sdk.TypeElasticsearch) - _, code, errD := services.NewClient(db, esServices).DoJSONRequest(context.Background(), "POST", "/events", e, nil) + _, code, errD := services.NewClient(esServices).DoJSONRequest(context.Background(), "POST", "/events", e, nil) if code >= 400 || errD != nil { log.Error(ctx, "PushInElasticSearch> Unable to send event %s to elasticsearch [%d]: %v", e.EventType, code, errD) continue @@ -59,7 +59,7 @@ func GetEvents(ctx context.Context, db gorp.SqlExecutor, store cache.Store, filt } var esEvents []elastic.SearchHit - if _, _, err := services.NewClient(db, srvs).DoJSONRequest(context.Background(), "GET", "/events", filters, &esEvents); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(context.Background(), "GET", "/events", filters, &esEvents); err != nil { return nil, sdk.WrapError(err, "Unable to get events") } diff --git a/engine/api/event_v2/event.go b/engine/api/event_v2/event.go index 37c7c8cc7d..f436b05b94 100644 --- a/engine/api/event_v2/event.go +++ b/engine/api/event_v2/event.go @@ -16,6 +16,7 @@ import ( "github.com/rockbears/log" "github.com/ovh/cds/engine/api/notification_v2" + "github.com/ovh/cds/engine/api/repositoriesmanager" "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/cache" "github.com/ovh/cds/engine/gorpmapper" @@ -51,7 +52,6 @@ func Dequeue(ctx context.Context, db *gorp.DbMap, store cache.Store, goroutines log.Error(ctx, "EventV2.DequeueEvent> store.DequeueWithContext err: %v", err) continue } - log.Debug(ctx, "event received: %v", e.Type) wg := sync.WaitGroup{} @@ -87,12 +87,22 @@ func Dequeue(ctx context.Context, db *gorp.DbMap, store cache.Store, goroutines } }) + // Workflow notifications + wg.Add(1) + goroutines.Exec(ctx, "event.workflow.notifications", func(ctx context.Context) { + defer wg.Done() + if err := workflowNotifications(ctx, db, store, e); err != nil { + ctx := log.ContextWithStackTrace(ctx, err) + log.Error(ctx, "EventV2.workflowNotifications: %v", err) + } + }) + wg.Wait() } } -func pushToWebsockets(ctx context.Context, store cache.Store, e sdk.FullEventV2) { - msg, err := json.Marshal(e) +func pushToWebsockets(ctx context.Context, store cache.Store, event sdk.FullEventV2) { + msg, err := json.Marshal(event) if err != nil { log.Error(ctx, "EventV2.pushToWebsockets: unable to marshal event: %v", err) return @@ -101,68 +111,121 @@ func pushToWebsockets(ctx context.Context, store cache.Store, e sdk.FullEventV2) log.Error(ctx, "EventV2.pushToWebsockets: ui: %v", err) } - if e.Type == sdk.EventRunJobEnqueued { + if event.Type == sdk.EventRunJobEnqueued { if err := store.Publish(ctx, EventHatcheryWS, string(msg)); err != nil { log.Error(ctx, "EventV2.pushToWebsockets: hatchery: %v", err) } } } -func pushNotifications(ctx context.Context, db *gorp.DbMap, e sdk.FullEventV2) error { - if e.ProjectKey != "" { - notifications, err := notification_v2.LoadAll(ctx, db, e.ProjectKey, gorpmapper.GetOptions.WithDecryption) - if err != nil { - return sdk.WrapError(err, "unable to load project %s notifications", e.ProjectKey) +func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Store, event sdk.FullEventV2) error { + // EventRunEnded + if event.Type != sdk.EventRunEnded || event.ProjectKey == "" { + return nil + } + + //event.Payload contains a V2WorkflowRun + var run sdk.V2WorkflowRun + if err := json.Unmarshal(event.Payload, &run); err != nil { + return sdk.WrapError(err, "cannot read payload for type %q", event.Type) + } + + // always send build status on workflow End + + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, store, event.ProjectKey, event.VCSName) + if err != nil { + return sdk.WrapError(err, "can't get AuthorizedClient for %v/%v", event.ProjectKey, event.VCSName) + } + + // TODO yesnault + vcsClient.SetStatus(ctx, sdk.VCSBuildStatus{}) + + // TODO send build status + + if run.WorkflowData.Workflow.On == nil { + return nil + } + + if run.WorkflowData.Workflow.On.PullRequest != nil { + if run.WorkflowData.Workflow.On.PullRequest.Comment != "" { + // TODO yesnault send comment + + // if err := e.vcsClient.PullRequestComment(ctx, app.RepositoryFullname, reqComment); err != nil { + // log.ErrorWithStackTrace(ctx, err) + // return err + // } + } - for _, n := range notifications { - canSend := false - if len(n.Filters) == 0 { - canSend = true - } else { - filterLoop: - for _, f := range n.Filters { - for _, evt := range f.Events { - reg, err := regexp.Compile(evt) - if err != nil { - log.ErrorWithStackTrace(ctx, err) - continue - } - if reg.MatchString(e.Type) { - canSend = true - break filterLoop - } - } - } - } - if canSend { - bts, _ := json.Marshal(e) - req, err := http.NewRequest("POST", n.WebHookURL, bytes.NewBuffer(bts)) - if err != nil { - log.Error(ctx, "unable to create request for notification %s for project %s: %v", n.Name, n.ProjectKey, err) - continue - } - for k, v := range n.Auth.Headers { - req.Header.Set(k, v) + } - } + if run.WorkflowData.Workflow.On.PullRequestComment != nil { + if run.WorkflowData.Workflow.On.PullRequestComment.Comment != "" { + // TODO yesnault send comment + } + } - resp, err := httpClient.Do(req) - if err != nil { - log.Error(ctx, "unable to send notification %s for project %s: %v", n.Name, n.ProjectKey, err) - continue - } - if resp.StatusCode >= 400 { - body, err := io.ReadAll(resp.Body) + return nil +} + +func pushNotifications(ctx context.Context, db *gorp.DbMap, event sdk.FullEventV2) error { + if event.ProjectKey == "" { + return nil + } + notifications, err := notification_v2.LoadAll(ctx, db, event.ProjectKey, gorpmapper.GetOptions.WithDecryption) + if err != nil { + return sdk.WrapError(err, "unable to load project %s notifications", event.ProjectKey) + } + for _, n := range notifications { + canSend := false + if len(n.Filters) == 0 { + canSend = true + } else { + filterLoop: + for _, f := range n.Filters { + for _, evt := range f.Events { + reg, err := regexp.Compile(evt) if err != nil { - log.Error(ctx, "unable to read body %s: %v", string(body), err) + log.ErrorWithStackTrace(ctx, err) + continue + } + if reg.MatchString(event.Type) { + canSend = true + break filterLoop } - log.Error(ctx, "unable to send notification %s for project %s. Http code: %d Body: %s", n.Name, n.ProjectKey, resp.StatusCode, string(body)) - _ = resp.Body.Close() - continue } - log.Debug(ctx, "notification %s - %s send on event %s", n.ProjectKey, n.Name, e.Type) } } + if !canSend { + continue + } + + bts, _ := json.Marshal(event) + req, err := http.NewRequest("POST", n.WebHookURL, bytes.NewBuffer(bts)) + if err != nil { + log.Error(ctx, "unable to create request for notification %s for project %s: %v", n.Name, n.ProjectKey, err) + continue + } + for k, v := range n.Auth.Headers { + req.Header.Set(k, v) + + } + + resp, err := httpClient.Do(req) + if err != nil { + log.Error(ctx, "unable to send notification %s for project %s: %v", n.Name, n.ProjectKey, err) + continue + } + if resp.StatusCode >= 400 { + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Error(ctx, "unable to read body %s: %v", string(body), err) + } + log.Error(ctx, "unable to send notification %s for project %s. Http code: %d Body: %s", n.Name, n.ProjectKey, resp.StatusCode, string(body)) + _ = resp.Body.Close() + continue + } + log.Debug(ctx, "notification %s - %s send on event %s", n.ProjectKey, n.Name, event.Type) + } return nil } @@ -179,7 +242,7 @@ func pushToElasticSearch(ctx context.Context, db *gorp.DbMap, e sdk.FullEventV2) e.Payload = nil log.Info(ctx, "sending event %q to %s services", e.Type, sdk.TypeElasticsearch) - _, code, err := services.NewClient(db, esServices).DoJSONRequest(context.Background(), "POST", "/v2/events", e, nil) + _, code, err := services.NewClient(esServices).DoJSONRequest(context.Background(), "POST", "/v2/events", e, nil) if code >= 400 || err != nil { return sdk.WrapError(err, "unable to send event %s to elasticsearch [%d]: %v", e.Type, code, err) } diff --git a/engine/api/metrics/elasticsearch.go b/engine/api/metrics/elasticsearch.go index 500d1396d0..e6eec4f4a5 100644 --- a/engine/api/metrics/elasticsearch.go +++ b/engine/api/metrics/elasticsearch.go @@ -38,7 +38,7 @@ func Init(ctx context.Context, DBFunc func() *gorp.DbMap) { continue } - _, code, errD := services.NewClient(DBFunc(), esServices).DoJSONRequest(context.Background(), "POST", "/metrics", e, nil) + _, code, errD := services.NewClient(esServices).DoJSONRequest(context.Background(), "POST", "/metrics", e, nil) if code >= 400 || errD != nil { log.Error(ctx, "metrics.pushInElasticSearch> Unable to send metrics to elasticsearch [%d]: %v", code, errD) continue @@ -61,7 +61,7 @@ func GetMetrics(ctx context.Context, db gorp.SqlExecutor, key string, appID int6 } var esMetrics []elastic.SearchHit - if _, _, err := services.NewClient(db, srvs).DoJSONRequest(context.Background(), "GET", "/metrics", metricsRequest, &esMetrics); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(context.Background(), "GET", "/metrics", metricsRequest, &esMetrics); err != nil { return nil, sdk.WrapError(err, "Unable to get metrics") } diff --git a/engine/api/operation/operation.go b/engine/api/operation/operation.go index 15c921e23e..63ead232ac 100644 --- a/engine/api/operation/operation.go +++ b/engine/api/operation/operation.go @@ -20,7 +20,7 @@ import ( "github.com/ovh/cds/sdk/exportentities" ) -func pushOperation(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store cache.Store, proj sdk.Project, data exportentities.WorkflowComponents, ope sdk.Operation) (*sdk.Operation, error) { +func pushOperation(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj sdk.Project, data exportentities.WorkflowComponents, ope sdk.Operation) (*sdk.Operation, error) { if ope.RepositoryStrategy.SSHKey != "" { key := proj.GetSSHKey(ope.RepositoryStrategy.SSHKey) if key == nil { @@ -130,12 +130,12 @@ func PostRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, prj sdk.P } if multipartData == nil { - if _, _, err := services.NewClient(db, srvs).DoJSONRequest(ctx, http.MethodPost, "/operations", ope, ope); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodPost, "/operations", ope, ope); err != nil { return sdk.WrapError(err, "Unable to perform operation") } return nil } - if _, err := services.NewClient(db, srvs).DoMultiPartRequest(ctx, http.MethodPost, "/operations", multipartData, ope, ope); err != nil { + if _, err := services.NewClient(srvs).DoMultiPartRequest(ctx, http.MethodPost, "/operations", multipartData, ope, ope); err != nil { return sdk.WrapError(err, "unable to perform multipart operation") } return nil @@ -148,7 +148,7 @@ func GetRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, uuid strin return nil, sdk.WrapError(err, "unable to found repositories service") } var ope sdk.Operation - if _, _, err := services.NewClient(db, srvs).DoJSONRequest(ctx, http.MethodGet, "/operations/"+uuid, nil, &ope); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodGet, "/operations/"+uuid, nil, &ope); err != nil { return nil, sdk.WrapError(err, "unable to get operation") } return &ope, nil diff --git a/engine/api/purge/purge.go b/engine/api/purge/purge.go index b47d2d99b2..ed115dcd28 100644 --- a/engine/api/purge/purge.go +++ b/engine/api/purge/purge.go @@ -245,7 +245,7 @@ func deleteWorkflowRunsHistory(ctx context.Context, db *gorp.DbMap, workflowRuns if err != nil { return err } - cdnClient := services.NewClient(db, srvs) + cdnClient := services.NewClient(srvs) limit := int64(2000) offset := int64(0) diff --git a/engine/api/purge/purge_run.go b/engine/api/purge/purge_run.go index 9820c33b20..c0b4af595e 100644 --- a/engine/api/purge/purge_run.go +++ b/engine/api/purge/purge_run.go @@ -84,20 +84,11 @@ func ApplyRetentionPolicyOnWorkflow(ctx context.Context, store cache.Store, db * } app = *appDB if app.RepositoryFullname != "" { - tx, err := db.Begin() - if err != nil { - return sdk.WithStack(err) - } //Get the RepositoriesManager Client - vcsClient, err = repositoriesmanager.AuthorizedClient(ctx, tx, store, wf.ProjectKey, app.VCSServer) + vcsClient, err = repositoriesmanager.AuthorizedClient(ctx, db, store, wf.ProjectKey, app.VCSServer) if err != nil { - _ = tx.Rollback() return sdk.WithStack(err) } - if err := tx.Commit(); err != nil { - _ = tx.Rollback() - return err - } } } } diff --git a/engine/api/purge/purge_test.go b/engine/api/purge/purge_test.go index 3738cfd8eb..60d857dad5 100644 --- a/engine/api/purge/purge_test.go +++ b/engine/api/purge/purge_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/services/mock_services" @@ -25,7 +24,7 @@ func Test_deleteWorkflowRunsHistory(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -48,7 +47,7 @@ func Test_deleteWorkflowRunsHistory(t *testing.T) { srvs, err := services.LoadAllByType(context.TODO(), db, sdk.TypeCDN) require.NoError(t, err) - cdnClient := services.NewClient(db, srvs) + cdnClient := services.NewClient(srvs) err = deleteRunHistory(context.Background(), db.DbMap, wr.ID, cdnClient, nil) require.NoError(t, err) diff --git a/engine/api/repositoriesmanager/events.go b/engine/api/repositoriesmanager/events.go index 225c3c3484..bbb3be654b 100644 --- a/engine/api/repositoriesmanager/events.go +++ b/engine/api/repositoriesmanager/events.go @@ -9,12 +9,11 @@ import ( "github.com/rockbears/log" "github.com/ovh/cds/engine/cache" - "github.com/ovh/cds/engine/gorpmapper" "github.com/ovh/cds/sdk" ) // ReceiveEvents has to be launched as a goroutine. -func ReceiveEvents(ctx context.Context, DBFunc func() *gorp.DbMap, store cache.Store) { +func ReceiveEvents(ctx context.Context, DBFunc func() *gorp.DbMap, store cache.Store, cdsUIURL string) { for { if err := ctx.Err(); err != nil { log.Error(ctx, "repositoriesmanager.ReceiveEvents> exiting: %v", err) @@ -29,20 +28,12 @@ func ReceiveEvents(ctx context.Context, DBFunc func() *gorp.DbMap, store cache.S db := DBFunc() if db != nil { - tx, err := db.Begin() - if err != nil { - log.Error(ctx, "ReceiveEvents> err opening tx: %v", err) - } - if err := processEvent(ctx, tx, e, store); err != nil { + if err := processEvent(ctx, db, e, store, cdsUIURL); err != nil { log.Error(ctx, "ReceiveEvents> err while processing error: %v", err) if err2 := RetryEvent(&e, err, store); err2 != nil { log.Error(ctx, "ReceiveEvents> err while processing error on retry: %v", err2) } } - if err := tx.Commit(); err != nil { - tx.Rollback() // nolint - log.Error(ctx, "ReceiveEvents> err commit tx: %v", err) - } continue } if err := RetryEvent(&e, nil, store); err != nil { @@ -60,13 +51,12 @@ func RetryEvent(e *sdk.Event, err error, store cache.Store) error { return store.Enqueue("events_repositoriesmanager", e) } -func processEvent(ctx context.Context, db gorpmapper.SqlExecutorWithTx, event sdk.Event, store cache.Store) error { +func processEvent(ctx context.Context, db *gorp.DbMap, event sdk.Event, store cache.Store, cdsUIURL string) error { if event.EventType != fmt.Sprintf("%T", sdk.EventRunWorkflowNode{}) { return nil } var eventWNR sdk.EventRunWorkflowNode - if err := sdk.JSONUnmarshal(event.Payload, &eventWNR); err != nil { return sdk.WrapError(err, "cannot read payload") } @@ -79,7 +69,17 @@ func processEvent(ctx context.Context, db gorpmapper.SqlExecutorWithTx, event sd return sdk.WrapError(err, "AuthorizedClient (%s, %s)", event.ProjectKey, eventWNR.RepositoryManagerName) } - if err := c.SetStatus(ctx, event); err != nil { + buildStatus := sdk.VCSBuildStatus{ + Description: eventWNR.NodeName + ":" + eventWNR.Status, + URLCDS: fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", cdsUIURL, event.ProjectKey, event.WorkflowName, eventWNR.Number), + Context: fmt.Sprintf("%s-%s-%s", event.ProjectKey, event.WorkflowName, eventWNR.NodeName), + Status: eventWNR.Status, + RepositoryFullname: eventWNR.RepositoryFullName, + GitHash: eventWNR.Hash, + GerritChange: eventWNR.GerritChange, + } + + if err := c.SetStatus(ctx, buildStatus); err != nil { if err := RetryEvent(&event, err, store); err != nil { log.Error(ctx, "repositoriesmanager>processEvent> err while retry event: %v", err) } diff --git a/engine/api/repositoriesmanager/repositories_manager.go b/engine/api/repositoriesmanager/repositories_manager.go index 7e119a70c6..0a31d2fec7 100644 --- a/engine/api/repositoriesmanager/repositories_manager.go +++ b/engine/api/repositoriesmanager/repositories_manager.go @@ -20,7 +20,6 @@ import ( "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/vcs" "github.com/ovh/cds/engine/cache" - "github.com/ovh/cds/engine/gorpmapper" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/telemetry" ) @@ -39,21 +38,11 @@ func (c *vcsClient) IsGerrit(ctx context.Context, db gorp.SqlExecutor) (bool, er return false, nil } -type vcsConsumer struct { - name string - proj *sdk.Project - db gorpmapper.SqlExecutorWithTx -} - type vcsClient struct { name string - token string - secret string projectKey string - created int64 //Timestamp .Unix() of creation srvs []sdk.Service cache *gocache.Cache - db gorpmapper.SqlExecutorWithTx vcsProject *sdk.VCSProject } @@ -68,7 +57,7 @@ type Options struct { Sync bool } -func GetReposForProjectVCSServer(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store cache.Store, proj sdk.Project, vcsServerName string, opts Options) ([]sdk.VCSRepo, error) { +func GetReposForProjectVCSServer(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj sdk.Project, vcsServerName string, opts Options) ([]sdk.VCSRepo, error) { log.Debug(ctx, "GetReposForProjectVCSServer> Loading repo for %s", vcsServerName) client, err := AuthorizedClient(ctx, db, store, proj.Key, vcsServerName) @@ -102,26 +91,8 @@ func GetReposForProjectVCSServer(ctx context.Context, db gorpmapper.SqlExecutorW return repos, nil } -func (c *vcsConsumer) GetAuthorizedClient(ctx context.Context, token, secret string, created int64) (sdk.VCSAuthorizedClientService, error) { - srvs, err := services.LoadAllByType(ctx, c.db, sdk.TypeVCS) - if err != nil { - return nil, sdk.WithStack(err) - } - - return &vcsClient{ - name: c.name, - token: token, - projectKey: c.proj.Key, - created: created, - secret: secret, - srvs: srvs, - cache: gocache.New(5*time.Second, 60*time.Second), - db: c.db, - }, nil -} - // AuthorizedClient returns an implementation of AuthorizedClient wrapping calls to vcs uService -func AuthorizedClient(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store cache.Store, projectKey string, vcsName string) (sdk.VCSAuthorizedClientService, error) { +func AuthorizedClient(ctx context.Context, db gorp.SqlExecutor, store cache.Store, projectKey string, vcsName string) (sdk.VCSAuthorizedClientService, error) { vcsProject, err := vcs.LoadVCSByProject(ctx, db, projectKey, vcsName, gorpmapping.GetOptions.WithDecryption) if err != nil { return nil, sdk.WithStack(err) @@ -135,7 +106,6 @@ func AuthorizedClient(ctx context.Context, db gorpmapper.SqlExecutorWithTx, stor vcs := &vcsClient{ name: vcsProject.Name, srvs: srvs, - db: db, projectKey: projectKey, vcsProject: vcsProject, } @@ -157,7 +127,7 @@ func (c *vcsClient) setAuthHeader(ctx context.Context, req *http.Request) { } func (c *vcsClient) doStreamRequest(ctx context.Context, method, path string, in interface{}) (io.Reader, http.Header, error) { - reader, headers, code, err := services.NewClient(c.db, c.srvs).StreamRequest(ctx, method, path, in, func(req *http.Request) { + reader, headers, code, err := services.NewClient(c.srvs).StreamRequest(ctx, method, path, in, func(req *http.Request) { c.setAuthHeader(ctx, req) }) if code >= 400 { @@ -180,7 +150,7 @@ func (c *vcsClient) doStreamRequest(ctx context.Context, method, path string, in } func (c *vcsClient) doJSONRequest(ctx context.Context, method, path string, in interface{}, out interface{}) (int, error) { - _, code, err := services.NewClient(c.db, c.srvs).DoJSONRequest(ctx, method, path, in, out, func(req *http.Request) { + _, code, err := services.NewClient(c.srvs).DoJSONRequest(ctx, method, path, in, out, func(req *http.Request) { c.setAuthHeader(ctx, req) }) @@ -485,29 +455,18 @@ func (c *vcsClient) PullRequestEvents(ctx context.Context, fullname string, evts return events, nil } -func (c *vcsClient) IsDisableStatusDetails(ctx context.Context) bool { - if c.vcsProject != nil { - return c.vcsProject.Options.DisableStatusDetails - } - return true -} - -func (c *vcsClient) SetDisableStatusDetails(disableStatusDetails bool) { - // nothing here for the implementation of this func into api -} - -func (c *vcsClient) SetStatus(ctx context.Context, event sdk.Event) error { +func (c *vcsClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { if c.vcsProject != nil && c.vcsProject.Options.DisableStatus { return nil } path := fmt.Sprintf("/vcs/%s/status", c.name) if c.vcsProject != nil && c.vcsProject.Options.DisableStatusDetails { - path += "?disableStatusDetails=true" + buildStatus.URLCDS = "" } - _, err := c.doJSONRequest(ctx, "POST", path, event, nil) - return sdk.NewErrorFrom(err, "unable to set status on %s (workflow: %s, application: %s)", event.WorkflowName, event.ApplicationName, c.name) + _, err := c.doJSONRequest(ctx, "POST", path, buildStatus, nil) + return sdk.NewErrorFrom(err, "unable to set build-status on Context: %s", buildStatus.Context) } func (c *vcsClient) Release(ctx context.Context, fullname, tagName, releaseTitle, releaseDescription string) (*sdk.VCSRelease, error) { diff --git a/engine/api/services/http.go b/engine/api/services/http.go index 48159b6891..2b9deec17f 100644 --- a/engine/api/services/http.go +++ b/engine/api/services/http.go @@ -15,7 +15,6 @@ import ( "strconv" "time" - "github.com/go-gorp/gorp" "github.com/rockbears/log" "gopkg.in/spacemonkeygo/httpsig.v0" @@ -48,15 +47,13 @@ type Client interface { } type defaultServiceClient struct { - db gorp.SqlExecutor srvs []sdk.Service } -var NewClient func(gorp.SqlExecutor, []sdk.Service) Client = NewDefaultClient +var NewClient func([]sdk.Service) Client = NewDefaultClient -func NewDefaultClient(db gorp.SqlExecutor, srvs []sdk.Service) Client { +func NewDefaultClient(srvs []sdk.Service) Client { return &defaultServiceClient{ - db: db, srvs: srvs, } } diff --git a/engine/api/v2_hooks.go b/engine/api/v2_hooks.go index d1cc4c0c03..1436555890 100644 --- a/engine/api/v2_hooks.go +++ b/engine/api/v2_hooks.go @@ -153,7 +153,7 @@ func (api *API) postHookEventRetrieveSignKeyHandler() ([]service.RbacChecker, se callback.SigningKeyCallback.Error = ope.Error.Message + fmt.Sprintf("(Operation ID: %s)", ope.UUID) } - if _, code, err := services.NewClient(api.mustDB(), srvs).DoJSONRequest(ctx, http.MethodPost, "/v2/repository/event/callback", callback, nil); err != nil { + if _, code, err := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodPost, "/v2/repository/event/callback", callback, nil); err != nil { log.ErrorWithStackTrace(ctx, sdk.WrapError(err, "unable to send analysis call to hook [HTTP: %d]", code)) return } @@ -402,7 +402,7 @@ func (api *API) getRepositoryWebHookSecretHandler() ([]service.RbacChecker, serv path := fmt.Sprintf("/v2/repository/key/%s/%s", vcsName, url.PathEscape(repositoryName)) var keyResp sdk.GenerateRepositoryWebhook - _, code, err := services.NewClient(tx, srvs).DoJSONRequest(ctx, http.MethodGet, path, nil, &keyResp) + _, code, err := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodGet, path, nil, &keyResp) if err != nil { return sdk.WrapError(err, "unable to delete hook [HTTP: %d]", code) } diff --git a/engine/api/v2_hooks_test.go b/engine/api/v2_hooks_test.go index 4463e71786..57699bff2c 100644 --- a/engine/api/v2_hooks_test.go +++ b/engine/api/v2_hooks_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/entity" "github.com/ovh/cds/engine/api/services" @@ -67,7 +66,7 @@ func TestPostHookEventRetrieveSignKeyHandler(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/v2_project_clean_ascode_test.go b/engine/api/v2_project_clean_ascode_test.go index a9b4a361dd..0c49925ca8 100644 --- a/engine/api/v2_project_clean_ascode_test.go +++ b/engine/api/v2_project_clean_ascode_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/entity" "github.com/ovh/cds/engine/api/repository" @@ -83,7 +82,7 @@ spec: ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/v2_project_repository.go b/engine/api/v2_project_repository.go index a8eebb3249..f51ca3ca70 100644 --- a/engine/api/v2_project_repository.go +++ b/engine/api/v2_project_repository.go @@ -71,7 +71,7 @@ func (api *API) getProjectRepositoryEventsHandler() ([]service.RbacChecker, serv } path := fmt.Sprintf("/v2/repository/event/%s/%s", vcsProject.Name, url.PathEscape(repo.Name)) var repositoryEvents []sdk.HookRepositoryEvent - _, code, errHooks := services.NewClient(api.mustDB(), srvs).DoJSONRequest(ctx, http.MethodGet, path, nil, &repositoryEvents) + _, code, errHooks := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodGet, path, nil, &repositoryEvents) if (errHooks != nil || code >= 400) && code != 404 { return sdk.WrapError(errHooks, "unable to delete hook [HTTP: %d]", code) } diff --git a/engine/api/v2_project_repository_test.go b/engine/api/v2_project_repository_test.go index 7c8d40beb2..94b52cd7cf 100644 --- a/engine/api/v2_project_repository_test.go +++ b/engine/api/v2_project_repository_test.go @@ -4,16 +4,16 @@ import ( "bytes" "context" "encoding/json" - "github.com/go-gorp/gorp" - "github.com/golang/mock/gomock" - "github.com/ovh/cds/engine/api/services" - "github.com/ovh/cds/engine/api/services/mock_services" "io" "net/http" "net/http/httptest" "net/url" "testing" + "github.com/golang/mock/gomock" + "github.com/ovh/cds/engine/api/services" + "github.com/ovh/cds/engine/api/services/mock_services" + "github.com/ovh/cds/engine/api/test" "github.com/ovh/cds/engine/api/test/assets" "github.com/ovh/cds/sdk" @@ -39,7 +39,7 @@ func Test_crudRepositoryOnProjectLambdaUserOK(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/v2_project_vcs_test.go b/engine/api/v2_project_vcs_test.go index 59643829c4..a700d04e64 100644 --- a/engine/api/v2_project_vcs_test.go +++ b/engine/api/v2_project_vcs_test.go @@ -12,7 +12,6 @@ import ( "github.com/ovh/cds/engine/api/keys" "github.com/ovh/cds/engine/api/project" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/services/mock_services" @@ -76,7 +75,7 @@ func Test_crudVCSOnProjectLambdaUserOK(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -158,7 +157,7 @@ func Test_crudVCSOnProjectAdminOk(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -277,7 +276,7 @@ func Test_crudVCSOnPublicProject(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/v2_repository_analyze.go b/engine/api/v2_repository_analyze.go index e8e764a151..bc45e86c59 100644 --- a/engine/api/v2_repository_analyze.go +++ b/engine/api/v2_repository_analyze.go @@ -842,7 +842,7 @@ func sendAnalysisHookCallback(ctx context.Context, db *gorp.DbMap, analysis sdk. } } - if _, code, err := services.NewClient(db, srvs).DoJSONRequest(ctx, http.MethodPost, "/v2/repository/event/callback", callback, nil); err != nil { + if _, code, err := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodPost, "/v2/repository/event/callback", callback, nil); err != nil { return sdk.WrapError(err, "unable to send analysis call to hook [HTTP: %d]", code) } return nil diff --git a/engine/api/v2_repository_analyze_test.go b/engine/api/v2_repository_analyze_test.go index 451fd38c24..f55587d59a 100644 --- a/engine/api/v2_repository_analyze_test.go +++ b/engine/api/v2_repository_analyze_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -133,7 +132,7 @@ func TestAnalyzeGithubWithoutHash(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -201,7 +200,7 @@ func TestAnalyzeGithubWrongSignature(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -278,7 +277,7 @@ func TestAnalyzeGithubGPGKeyNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -413,7 +412,7 @@ GDFkaTe3nUJdYV4= ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -539,7 +538,7 @@ func TestAnalyzeGithubServerCommitNotSigned(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -674,7 +673,7 @@ GDFkaTe3nUJdYV4= ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer t.Cleanup(func() { @@ -879,7 +878,7 @@ GDFkaTe3nUJdYV4= ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -1041,7 +1040,7 @@ GDFkaTe3nUJdYV4= ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -1250,7 +1249,7 @@ GDFkaTe3nUJdYV4= ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/v2_workflow_run.go b/engine/api/v2_workflow_run.go index 34ecadad43..df9195d440 100644 --- a/engine/api/v2_workflow_run.go +++ b/engine/api/v2_workflow_run.go @@ -993,7 +993,7 @@ func restartWorkflowRun(ctx context.Context, tx gorpmapper.SqlExecutorWithTx, wr } } req := sdk.CDNDuplicateItemRequest{FromJob: rj.ID, ToJob: duplicatedRJ.ID} - _, code, err := services.NewClient(tx, srvs).DoJSONRequest(ctx, http.MethodPost, "/item/duplicate", req, nil) + _, code, err := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodPost, "/item/duplicate", req, nil) if err != nil || code >= 400 { return fmt.Errorf("unable to duplicate cdn item for runjob %s. Code result %d: %v", rj.ID, code, err) } diff --git a/engine/api/v2_workflow_run_craft_test.go b/engine/api/v2_workflow_run_craft_test.go index 4a94fcb3cc..00b61c692d 100644 --- a/engine/api/v2_workflow_run_craft_test.go +++ b/engine/api/v2_workflow_run_craft_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/entity" "github.com/ovh/cds/engine/api/hatchery" @@ -400,7 +399,7 @@ func TestCraftWorkflowRunDepsDifferentRepo(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/v2_workflow_run_test.go b/engine/api/v2_workflow_run_test.go index ae86a9b0a6..ffcec8039e 100644 --- a/engine/api/v2_workflow_run_test.go +++ b/engine/api/v2_workflow_run_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/authentication" @@ -161,7 +160,7 @@ func TestRunManualJob_WrongGateReviewer(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -322,7 +321,7 @@ func TestRunManualJob_WrongGateCondition(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -490,7 +489,7 @@ func TestRunManualJob(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -678,7 +677,7 @@ func TestPutWorkflowRun(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/workflow.go b/engine/api/workflow.go index 8716135174..5c85b16468 100644 --- a/engine/api/workflow.go +++ b/engine/api/workflow.go @@ -855,7 +855,7 @@ func (api *API) getWorkflowHookHandler() service.Handler { path := fmt.Sprintf("/task/%s/execution", uuid) task := sdk.Task{} - if _, _, err := services.NewClient(api.mustDB(), srvs).DoJSONRequest(ctx, "GET", path, nil, &task); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(ctx, "GET", path, nil, &task); err != nil { return sdk.WrapError(err, "unable to get hook %s task and executions", uuid) } diff --git a/engine/api/workflow/execute_node_run.go b/engine/api/workflow/execute_node_run.go index 479366b29a..2bda49578d 100644 --- a/engine/api/workflow/execute_node_run.go +++ b/engine/api/workflow/execute_node_run.go @@ -929,7 +929,7 @@ func stopWorkflowNodeOutGoingHook(ctx context.Context, dbFunc func() *gorp.DbMap if nodeRun.HookExecutionID != "" { path := fmt.Sprintf("/task/%s/execution/%d/stop", nodeRun.HookExecutionID, nodeRun.HookExecutionTimeStamp) - if _, _, err := services.NewClient(db, srvs).DoJSONRequest(ctx, "POST", path, nil, nil); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(ctx, "POST", path, nil, nil); err != nil { return sdk.WrapError(err, "unable to stop task execution") } } diff --git a/engine/api/workflow/hook.go b/engine/api/workflow/hook.go index 5789ee8131..84864ec2eb 100644 --- a/engine/api/workflow/hook.go +++ b/engine/api/workflow/hook.go @@ -61,7 +61,7 @@ func hookUnregistration(ctx context.Context, db gorpmapper.SqlExecutorWithTx, st if err != nil { return err } - _, code, errHooks := services.NewClient(db, srvs).DoJSONRequest(ctx, http.MethodDelete, "/task/bulk", hookToDelete, nil) + _, code, errHooks := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodDelete, "/task/bulk", hookToDelete, nil) if errHooks != nil || code >= 400 { // if we return an error, transaction will be rollbacked => hook will in database be not anymore on gitlab/bitbucket/github. // so, it's just a warn log @@ -171,7 +171,7 @@ func hookRegistration(ctx context.Context, db gorpmapper.SqlExecutorWithTx, stor if len(hookToUpdate) > 0 { // Create hook on µservice - _, code, errHooks := services.NewClient(db, srvs).DoJSONRequest(ctx, http.MethodPost, "/task/bulk", hookToUpdate, &hookToUpdate) + _, code, errHooks := services.NewClient(srvs).DoJSONRequest(ctx, http.MethodPost, "/task/bulk", hookToUpdate, &hookToUpdate) if errHooks != nil || code >= 400 { return sdk.WrapError(errHooks, "unable to create hooks [%d]", code) } diff --git a/engine/api/workflow/process_outgoinghook.go b/engine/api/workflow/process_outgoinghook.go index ea62dc6ee0..790ccf19d1 100644 --- a/engine/api/workflow/process_outgoinghook.go +++ b/engine/api/workflow/process_outgoinghook.go @@ -135,7 +135,7 @@ func processNodeOutGoingHook(ctx context.Context, db gorpmapper.SqlExecutorWithT } var task sdk.Task - if _, _, err := services.NewClient(db, srvs).DoJSONRequest(ctx, "POST", "/task/execute", hookRun, &task); err != nil { + if _, _, err := services.NewClient(srvs).DoJSONRequest(ctx, "POST", "/task/execute", hookRun, &task); err != nil { log.Warn(ctx, "outgoing hook execution failed: %v", err) hookRun.Status = sdk.StatusFail } diff --git a/engine/api/workflow/workflow_run_event.go b/engine/api/workflow/workflow_run_event.go index 9dbdb70781..5512e327e0 100644 --- a/engine/api/workflow/workflow_run_event.go +++ b/engine/api/workflow/workflow_run_event.go @@ -23,7 +23,7 @@ type VCSEventMessenger struct { } // ResyncCommitStatus resync commit status for a workflow run -func ResyncCommitStatus(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk.Project, wr *sdk.WorkflowRun) error { +func ResyncCommitStatus(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk.Project, wr *sdk.WorkflowRun, cdsUIURL string) error { _, end := telemetry.Span(ctx, "workflow.resyncCommitStatus", telemetry.Tag(telemetry.TagWorkflow, wr.Workflow.Name), telemetry.Tag(telemetry.TagWorkflowRun, wr.Number), @@ -37,7 +37,7 @@ func ResyncCommitStatus(ctx context.Context, db *gorp.DbMap, store cache.Store, }) nodeRun := nodeRuns[0] - if err := eventMessenger.SendVCSEvent(ctx, db, store, proj, *wr, nodeRun); err != nil { + if err := eventMessenger.SendVCSEvent(ctx, db, store, proj, *wr, nodeRun, cdsUIURL); err != nil { log.Error(ctx, "resyncCommitStatus > unable to send vcs event: %v", err) } } @@ -45,7 +45,7 @@ func ResyncCommitStatus(ctx context.Context, db *gorp.DbMap, store cache.Store, return nil } -func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk.Project, wr sdk.WorkflowRun, nodeRun sdk.WorkflowNodeRun) error { +func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk.Project, wr sdk.WorkflowRun, nodeRun sdk.WorkflowNodeRun, cdsUIURL string) error { ctx = context.WithValue(ctx, cdslog.NodeRunID, nodeRun.ID) tx, err := db.Begin() @@ -144,7 +144,7 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st if statusFound == nil || statusFound.State == "" { for i := range notifs { log.Info(ctx, "status %q %s not found, sending a new one %+v", expected, nodeRun.Status, notifs[i]) - if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName); err != nil { + if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName, cdsUIURL); err != nil { return sdk.WrapError(err, "can't sendVCSEventStatus vcs %v/%v", proj.Key, vcsServerName) } } @@ -175,7 +175,7 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st if !skipStatus { for i := range notifs { log.Info(ctx, "status %q %s not found, sending a new one %+v", expected, nodeRun.Status, notifs[i]) - if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName); err != nil { + if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName, cdsUIURL); err != nil { return sdk.WrapError(err, "can't sendVCSEventStatus vcs %v/%v", proj.Key, vcsServerName) } } @@ -199,7 +199,7 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st } // sendVCSEventStatus send status -func (e *VCSEventMessenger) sendVCSEventStatus(ctx context.Context, db gorp.SqlExecutor, store cache.Store, projectKey string, wr sdk.WorkflowRun, nodeRun *sdk.WorkflowNodeRun, notif sdk.WorkflowNotification, vcsServerName string) error { +func (e *VCSEventMessenger) sendVCSEventStatus(ctx context.Context, db gorp.SqlExecutor, store cache.Store, projectKey string, wr sdk.WorkflowRun, nodeRun *sdk.WorkflowNodeRun, notif sdk.WorkflowNotification, vcsServerName string, cdsUIURL string) error { if notif.Settings.Template == nil || (notif.Settings.Template.DisableStatus != nil && *notif.Settings.Template.DisableStatus) { return nil } @@ -314,7 +314,17 @@ func (e *VCSEventMessenger) sendVCSEventStatus(ctx context.Context, db gorp.SqlE EnvironmentName: envName, } - if err := e.vcsClient.SetStatus(ctx, evt); err != nil { + buildStatus := sdk.VCSBuildStatus{ + Description: eventWNR.NodeName + ": " + eventWNR.Status, + URLCDS: fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", cdsUIURL, evt.ProjectKey, evt.WorkflowName, eventWNR.Number), + Context: fmt.Sprintf("%s-%s-%s", evt.ProjectKey, evt.WorkflowName, eventWNR.NodeName), + Status: eventWNR.Status, + RepositoryFullname: eventWNR.RepositoryFullName, + GitHash: eventWNR.Hash, + GerritChange: eventWNR.GerritChange, + } + + if err := e.vcsClient.SetStatus(ctx, buildStatus); err != nil { if err2 := repositoriesmanager.RetryEvent(&evt, err, store); err2 != nil { return err2 } diff --git a/engine/api/workflow/workflow_run_event_test.go b/engine/api/workflow/workflow_run_event_test.go index 14938a1ffd..222e280240 100644 --- a/engine/api/workflow/workflow_run_event_test.go +++ b/engine/api/workflow/workflow_run_event_test.go @@ -5,7 +5,6 @@ import ( "net/http" "testing" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/repositoriesmanager" @@ -99,7 +98,7 @@ func TestResyncCommitStatusNotifDisabled(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -111,7 +110,7 @@ func TestResyncCommitStatusNotifDisabled(t *testing.T) { gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, 201, nil) - err = workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr) + err = workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr, "") assert.NoError(t, err) } @@ -187,7 +186,7 @@ func TestResyncCommitStatusSetStatus(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -210,7 +209,7 @@ func TestResyncCommitStatusSetStatus(t *testing.T) { DoJSONRequest(gomock.Any(), "POST", "/vcs/gerrit/status", gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, 201, nil) - err := workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr) + err := workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr, "") assert.NoError(t, err) } @@ -296,7 +295,7 @@ func TestResyncCommitStatusCommentPR(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -324,7 +323,7 @@ func TestResyncCommitStatusCommentPR(t *testing.T) { return nil, 200, nil }).MaxTimes(1) - err := workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr) + err := workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr, "") assert.NoError(t, err) } @@ -410,7 +409,7 @@ func TestResyncCommitStatusCommentPRNotTerminated(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -431,7 +430,7 @@ func TestResyncCommitStatusCommentPRNotTerminated(t *testing.T) { return nil, 200, nil }).MaxTimes(1) - err := workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr) + err := workflow.ResyncCommitStatus(ctx, db.DbMap, cache, *proj, wr, "") assert.NoError(t, err) } @@ -517,7 +516,7 @@ func TestResyncCommitStatusCommitCache(t *testing.T) { defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -555,7 +554,7 @@ func TestResyncCommitStatusCommitCache(t *testing.T) { return nil, 200, nil }).MaxTimes(1) e := workflow.VCSEventMessenger{} - err := e.SendVCSEvent(ctx, db.DbMap, cache, *proj, *wr, wr.WorkflowNodeRuns[1][0]) + err := e.SendVCSEvent(ctx, db.DbMap, cache, *proj, *wr, wr.WorkflowNodeRuns[1][0], "") assert.NoError(t, err) } diff --git a/engine/api/workflow_ascode_rename_test.go b/engine/api/workflow_ascode_rename_test.go index cd9f593195..c55362f40c 100644 --- a/engine/api/workflow_ascode_rename_test.go +++ b/engine/api/workflow_ascode_rename_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/rockbears/log" @@ -37,7 +36,7 @@ func Test_WorkflowAsCodeRename(t *testing.T) { // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/workflow_ascode_test.go b/engine/api/workflow_ascode_test.go index 5e786ea7d6..9003b10f1d 100644 --- a/engine/api/workflow_ascode_test.go +++ b/engine/api/workflow_ascode_test.go @@ -527,7 +527,7 @@ func Test_WorkflowAsCodeWithNotifications(t *testing.T) { // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -926,7 +926,7 @@ hooks: // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -1185,7 +1185,7 @@ func Test_WorkflowAsCodeWithPermissions(t *testing.T) { // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/workflow_ascode_with_hooks_test.go b/engine/api/workflow_ascode_with_hooks_test.go index a053d833ef..3c91046d75 100644 --- a/engine/api/workflow_ascode_with_hooks_test.go +++ b/engine/api/workflow_ascode_with_hooks_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/rockbears/log" "github.com/stretchr/testify/assert" @@ -37,7 +36,7 @@ func Test_WorkflowAsCodeWithNoHook_ShouldGive_AnAutomaticRepoWebHook(t *testing. // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { return servicesClients } + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } t.Cleanup(func() { services.NewClient = services.NewDefaultClient }) // Create a project with a repository manager @@ -227,7 +226,7 @@ func Test_WorkflowAsCodeWithDefaultHook_ShouldGive_TheSameRepoWebHook(t *testing // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { return servicesClients } + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } t.Cleanup(func() { services.NewClient = services.NewDefaultClient }) // Create a project with a repository manager @@ -476,7 +475,7 @@ func Test_WorkflowAsCodeWithDefaultHookAndAScheduler_ShouldGive_TheSameRepoWebHo // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { return servicesClients } + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } t.Cleanup(func() { services.NewClient = services.NewDefaultClient }) // Create a project with a repository manager @@ -808,7 +807,7 @@ func Test_WorkflowAsCodeWithJustAcheduler_ShouldGive_ARepoWebHookAndTheScheduler // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { return servicesClients } + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } t.Cleanup(func() { services.NewClient = services.NewDefaultClient }) // Create a project with a repository manager diff --git a/engine/api/workflow_ascode_with_secrets_test.go b/engine/api/workflow_ascode_with_secrets_test.go index 9ac2659c35..3084e06623 100644 --- a/engine/api/workflow_ascode_with_secrets_test.go +++ b/engine/api/workflow_ascode_with_secrets_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "testing" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -43,7 +42,7 @@ func Test_RunNonDefaultBranchWithSecrets(t *testing.T) { // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/workflow_event.go b/engine/api/workflow_event.go index ae504cae61..ff5481f4af 100644 --- a/engine/api/workflow_event.go +++ b/engine/api/workflow_event.go @@ -66,7 +66,7 @@ func (api *API) WorkflowSendEvent(ctx context.Context, proj sdk.Project, report eventsNotif := notification.GetUserWorkflowEvents(ctx, db, api.Cache, wr.Workflow.ProjectID, wr.Workflow.ProjectKey, workDB.Name, wr.Workflow.Notifications, previousNodeRun, *nr) event.PublishWorkflowNodeRun(ctx, *nr, wr.Workflow, eventsNotif) e := &workflow.VCSEventMessenger{} - if err := e.SendVCSEvent(ctx, db, api.Cache, proj, *wr, wnr); err != nil { + if err := e.SendVCSEvent(ctx, db, api.Cache, proj, *wr, wnr, api.Config.URL.UI); err != nil { ctx := sdk.ContextWithStacktrace(ctx, err) log.Warn(ctx, "WorkflowSendEvent> Cannot send vcs notification err:%v", err) } diff --git a/engine/api/workflow_export_test.go b/engine/api/workflow_export_test.go index 1152d3b280..2621063070 100644 --- a/engine/api/workflow_export_test.go +++ b/engine/api/workflow_export_test.go @@ -4,16 +4,16 @@ import ( "archive/tar" "bytes" "context" - "github.com/go-gorp/gorp" + "io" + "net/http" + "net/http/httptest" + "testing" + "github.com/golang/mock/gomock" "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/repositoriesmanager" "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/services/mock_services" - "io" - "net/http" - "net/http/httptest" - "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -177,7 +177,7 @@ func Test_getWorkflowExportHandler(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/api/workflow_run.go b/engine/api/workflow_run.go index ea95da0727..c050db6b17 100644 --- a/engine/api/workflow_run.go +++ b/engine/api/workflow_run.go @@ -392,7 +392,7 @@ func (api *API) stopWorkflowRunHandler() service.Handler { // The function could be called with nil project so we need to test if project is not nil if sdk.StatusIsTerminated(wRun.Status) && proj != nil { wRun.LastExecution = time.Now() - if err := workflow.ResyncCommitStatus(context.Background(), api.mustDB(), api.Cache, *proj, wRun); err != nil { + if err := workflow.ResyncCommitStatus(context.Background(), api.mustDB(), api.Cache, *proj, wRun, api.Config.URL.UI); err != nil { log.Error(ctx, "workflow.UpdateNodeJobRunStatus> %v", err) } } @@ -777,7 +777,7 @@ func (api *API) stopWorkflowNodeRunHandler() service.Handler { //The function could be called with nil project so we need to test if project is not nil if sdk.StatusIsTerminated(wRun.Status) && p != nil { wRun.LastExecution = time.Now() - if err := workflow.ResyncCommitStatus(context.Background(), api.mustDB(), api.Cache, *p, wRun); err != nil { + if err := workflow.ResyncCommitStatus(context.Background(), api.mustDB(), api.Cache, *p, wRun, api.Config.URL.UI); err != nil { log.Error(ctx, "workflow.stopWorkflowNodeRun> %v", err) } } @@ -1471,7 +1471,7 @@ func (api *API) postResyncVCSWorkflowRunHandler() service.Handler { return sdk.WrapError(err, "cannot load workflow run") } - if err := workflow.ResyncCommitStatus(ctx, api.mustDB(), api.Cache, *proj, wfr); err != nil { + if err := workflow.ResyncCommitStatus(ctx, api.mustDB(), api.Cache, *proj, wfr, api.Config.URL.UI); err != nil { return sdk.WrapError(err, "cannot resync workflow run commit status") } diff --git a/engine/api/workflow_test.go b/engine/api/workflow_test.go index 4fc4517398..964a1ecf6b 100644 --- a/engine/api/workflow_test.go +++ b/engine/api/workflow_test.go @@ -13,7 +13,6 @@ import ( "testing" "time" - "github.com/go-gorp/gorp" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1573,7 +1572,7 @@ func Test_putWorkflowShouldNotCallHOOKSIfHookDoesNotChange(t *testing.T) { // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { @@ -1681,7 +1680,7 @@ func Test_putWorkflowWithDuplicateHooksShouldRaiseAnError(t *testing.T) { // If you have to regenerate thi mock you just have to run, from directory $GOPATH/src/github.com/ovh/cds/engine/api/services: // mockgen -source=http.go -destination=mock_services/services_mock.go Client servicesClients := mock_services.NewMockClient(ctrl) - services.NewClient = func(_ gorp.SqlExecutor, _ []sdk.Service) services.Client { + services.NewClient = func(_ []sdk.Service) services.Client { return servicesClients } defer func() { diff --git a/engine/vcs/bitbucketcloud/bitbucketcloud.go b/engine/vcs/bitbucketcloud/bitbucketcloud.go index ac80f3e31b..532276dd09 100644 --- a/engine/vcs/bitbucketcloud/bitbucketcloud.go +++ b/engine/vcs/bitbucketcloud/bitbucketcloud.go @@ -11,13 +11,12 @@ const rootURL = "https://api.bitbucket.org/2.0" // bitbucketcloudClient is a https://bitbucket.org wrapper for CDS vcs. interface type bitbucketcloudClient struct { - appPassword string - username string - disableStatusDetails bool - Cache cache.Store - apiURL string - uiURL string - proxyURL string + appPassword string + username string + Cache cache.Store + apiURL string + uiURL string + proxyURL string } // bitbucketcloudConsumer implements vcs.Server and it's used to instantiate a githubClient diff --git a/engine/vcs/bitbucketcloud/client_status.go b/engine/vcs/bitbucketcloud/client_status.go index d04c5df79f..a92bc48aed 100644 --- a/engine/vcs/bitbucketcloud/client_status.go +++ b/engine/vcs/bitbucketcloud/client_status.go @@ -13,52 +13,50 @@ import ( "github.com/ovh/cds/sdk" ) -type statusData struct { - pipName string - desc string - status string - repoFullName string - hash string - urlPipeline string - context string -} - -func (client *bitbucketcloudClient) SetDisableStatusDetails(disableStatusDetails bool) { - client.disableStatusDetails = disableStatusDetails -} - // SetStatus Users with push access can create commit statuses for a given ref: -func (client *bitbucketcloudClient) SetStatus(ctx context.Context, event sdk.Event) error { - var data statusData - var err error - switch event.EventType { - case fmt.Sprintf("%T", sdk.EventRunWorkflowNode{}): - data, err = client.processEventWorkflowNodeRun(event, client.uiURL) - default: - log.Error(ctx, "bitbucketcloud.SetStatus> Unknown event %v", event) +func (client *bitbucketcloudClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { + if buildStatus.Status == "" { + log.Debug(ctx, "bitbucketcloud.SetStatus> Do not process event for empty status") return nil } - if err != nil { - return sdk.WrapError(err, "Cannot process Event") - } - if data.status == "" { - log.Debug(ctx, "bitbucketcloud.SetStatus> Do not process event for current status: %v", event) + if buildStatus.Status == sdk.StatusChecking || + buildStatus.Status == sdk.StatusDisabled || + buildStatus.Status == sdk.StatusNeverBuilt || + buildStatus.Status == sdk.StatusSkipped || + buildStatus.Status == sdk.StatusUnknown || + buildStatus.Status == sdk.StatusWaiting { return nil } + var state string + switch buildStatus.Status { + case sdk.StatusFail: + state = "FAILED" + case sdk.StatusSuccess, sdk.StatusSkipped: + state = "SUCCESSFUL" + case sdk.StatusStopped: + state = "STOPPED" + default: + state = "INPROGRESS" + } + bbStatus := Status{ - Description: data.desc, - URL: data.urlPipeline, - State: data.status, - Name: data.context, - Key: data.context, + Description: buildStatus.Description, + URL: buildStatus.URLCDS, + State: state, + Name: buildStatus.Context, + Key: buildStatus.Context, + } + + if len(buildStatus.Context) > 36 { // 40 maxlength on bitbucket cloud + buildStatus.Context = buildStatus.Context[:36] } - path := fmt.Sprintf("/repositories/%s/commit/%s/statuses/build", data.repoFullName, data.hash) + path := fmt.Sprintf("/repositories/%s/commit/%s/statuses/build", buildStatus.RepositoryFullname, buildStatus.GitHash) b, err := json.Marshal(bbStatus) if err != nil { - return sdk.WrapError(err, "Unable to marshal github status") + return sdk.WrapError(err, "Unable to marshal bitbucketcloud status") } buf := bytes.NewBuffer(b) @@ -73,7 +71,7 @@ func (client *bitbucketcloudClient) SetStatus(ctx context.Context, event sdk.Eve return sdk.WrapError(err, "Unable to read body") } if res.StatusCode != 201 && res.StatusCode != 200 { - return fmt.Errorf("Unable to create status on bitbucket cloud. Status code : %d - Body: %s - target:%s", res.StatusCode, body, data.urlPipeline) + return fmt.Errorf("unable to create status on bitbucket cloud. Status code : %d - Body: %s - context:%s", res.StatusCode, body, buildStatus.Context) } var resp Status @@ -128,55 +126,3 @@ func processBbitbucketState(s Status) string { return sdk.StatusBuilding } } - -func (client *bitbucketcloudClient) processEventWorkflowNodeRun(event sdk.Event, cdsUIURL string) (statusData, error) { - data := statusData{} - var eventNR sdk.EventRunWorkflowNode - if err := sdk.JSONUnmarshal(event.Payload, &eventNR); err != nil { - return data, sdk.WrapError(err, "cannot unmarshal payload") - } - //We only manage status Success, Failure and Stopped - if eventNR.Status == sdk.StatusChecking || - eventNR.Status == sdk.StatusDisabled || - eventNR.Status == sdk.StatusNeverBuilt || - eventNR.Status == sdk.StatusSkipped || - eventNR.Status == sdk.StatusUnknown || - eventNR.Status == sdk.StatusWaiting { - return data, nil - } - - switch eventNR.Status { - case sdk.StatusFail: - data.status = "FAILED" - case sdk.StatusSuccess, sdk.StatusSkipped: - data.status = "SUCCESSFUL" - case sdk.StatusStopped: - data.status = "STOPPED" - default: - data.status = "INPROGRESS" - } - data.hash = eventNR.Hash - data.repoFullName = eventNR.RepositoryFullName - data.pipName = eventNR.NodeName - - if !client.disableStatusDetails { - data.urlPipeline = fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", - cdsUIURL, - event.ProjectKey, - event.WorkflowName, - eventNR.Number, - ) - } else { - //CDS can avoid sending bitbucket target url in status, if it's disabled - if client.disableStatusDetails { - data.urlPipeline = "https://ovh.github.io/cds/" // because it's mandatory - } - } - - data.context = sdk.VCSCommitStatusDescription(event.ProjectKey, event.WorkflowName, eventNR) - if len(data.context) > 36 { // 40 maxlength on bitbucket cloud - data.context = data.context[:36] - } - data.desc = eventNR.NodeName + ": " + eventNR.Status - return data, nil -} diff --git a/engine/vcs/bitbucketserver/bitbucketserver.go b/engine/vcs/bitbucketserver/bitbucketserver.go index 142ef7cc74..b9d53e8591 100644 --- a/engine/vcs/bitbucketserver/bitbucketserver.go +++ b/engine/vcs/bitbucketserver/bitbucketserver.go @@ -11,11 +11,10 @@ import ( // bitbucketClient is a bitbucket wrapper for CDS vcs. interface type bitbucketClient struct { - username string - token string - proxyURL string - disableStatusDetails bool - consumer bitbucketConsumer + username string + token string + proxyURL string + consumer bitbucketConsumer } // bitbucketConsumer implements vcs.Server and it's used to instantiate a bitbucketClient diff --git a/engine/vcs/bitbucketserver/client_status.go b/engine/vcs/bitbucketserver/client_status.go index f21b0a808f..59f3c06e14 100644 --- a/engine/vcs/bitbucketserver/client_status.go +++ b/engine/vcs/bitbucketserver/client_status.go @@ -14,43 +14,17 @@ import ( "github.com/ovh/cds/sdk/telemetry" ) -type statusData struct { - key string - buildNumber int64 - status string - url string - hash string - description string -} - -func (client *bitbucketClient) SetDisableStatusDetails(disableStatusDetails bool) { - client.disableStatusDetails = disableStatusDetails -} - -func (client *bitbucketClient) SetStatus(ctx context.Context, event sdk.Event) error { +func (client *bitbucketClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { ctx, end := telemetry.Span(ctx, "bitbucketserver.SetStatus") defer end() - var statusData statusData - var err error - switch event.EventType { - case fmt.Sprintf("%T", sdk.EventRunWorkflowNode{}): - statusData, err = client.processWorkflowNodeRunEvent(event, client.consumer.uiURL) - default: - return nil - } - - if err != nil { - return sdk.WrapError(err, "bitbucketClient.SetStatus: Cannot process Event") - } - - state := getBitbucketStateFromStatus(statusData.status) + state := getBitbucketStateFromStatus(buildStatus.Status) status := Status{ - Key: statusData.key, - Name: fmt.Sprintf("%s%d", statusData.key, statusData.buildNumber), + Key: buildStatus.Context, + Name: buildStatus.Context, State: state, - URL: statusData.url, - Description: statusData.description, + URL: buildStatus.URLCDS, + Description: buildStatus.Description, } values, err := json.Marshal(status) @@ -58,9 +32,9 @@ func (client *bitbucketClient) SetStatus(ctx context.Context, event sdk.Event) e return sdk.WrapError(err, "Unable to marshall status") } - log.Info(ctx, "sending build status for %s : %s %s - %s", statusData.hash, status.Key, status.Name, state) + log.Info(ctx, "sending build status for %s : %s %s - %s", buildStatus.GitHash, status.Key, status.Name, state) - if err := client.do(ctx, "POST", "build-status", fmt.Sprintf("/commits/%s", statusData.hash), nil, values, nil); err != nil { + if err := client.do(ctx, "POST", "build-status", fmt.Sprintf("/commits/%s", buildStatus.GitHash), nil, values, nil); err != nil { return sdk.WrapError(err, "Unable to post build-status name:%s status:%s", status.Name, state) } return nil @@ -130,33 +104,6 @@ const ( failed = "FAILED" ) -func (client *bitbucketClient) processWorkflowNodeRunEvent(event sdk.Event, uiURL string) (statusData, error) { - data := statusData{} - var eventNR sdk.EventRunWorkflowNode - if err := sdk.JSONUnmarshal(event.Payload, &eventNR); err != nil { - return data, sdk.WrapError(err, "cannot unmarshal payload") - } - data.key = fmt.Sprintf("%s-%s-%s", - event.ProjectKey, - event.WorkflowName, - eventNR.NodeName, - ) - if !client.disableStatusDetails { - data.url = fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", - uiURL, - event.ProjectKey, - event.WorkflowName, - eventNR.Number, - ) - } - data.buildNumber = eventNR.Number - data.status = eventNR.Status - data.hash = eventNR.Hash - data.description = sdk.VCSCommitStatusDescription(event.ProjectKey, event.WorkflowName, eventNR) - - return data, nil -} - func getBitbucketStateFromStatus(status string) string { switch status { case sdk.StatusSuccess, sdk.StatusSkipped, sdk.StatusDisabled: diff --git a/engine/vcs/gerrit/client_status.go b/engine/vcs/gerrit/client_status.go index 441769a710..9f7fb2ef7e 100644 --- a/engine/vcs/gerrit/client_status.go +++ b/engine/vcs/gerrit/client_status.go @@ -11,19 +11,10 @@ import ( "github.com/ovh/cds/sdk" ) -func (client *gerritClient) SetDisableStatusDetails(disableStatusDetails bool) { - client.disableStatusDetails = disableStatusDetails -} - // SetStatus set build status on Gerrit -func (client *gerritClient) SetStatus(ctx context.Context, event sdk.Event) error { - var eventNR sdk.EventRunWorkflowNode - if err := sdk.JSONUnmarshal(event.Payload, &eventNR); err != nil { - return sdk.WrapError(err, "cannot unmarshal payload") - } - - if eventNR.GerritChange == nil { - log.Debug(ctx, "gerrit.setStatus> no gerrit change provided: %s/%s", eventNR.Status, eventNR.NodeName) +func (client *gerritClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { + if buildStatus.GerritChange == nil { + log.Debug(ctx, "gerrit.setStatus> no gerrit change provided - context %s", buildStatus.Context) return nil } @@ -32,14 +23,14 @@ func (client *gerritClient) SetStatus(ctx context.Context, event sdk.Event) erro // https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input ri := gerrit.ReviewInput{ - Message: client.buildMessage(eventNR), + Message: client.buildMessage(buildStatus), Tag: "CDS", - Labels: client.buildLabel(eventNR), + Labels: client.buildLabel(buildStatus), Notify: "OWNER", // Send notification to the owner } // Check if we already send the message - changeDetail, _, err := client.client.Changes.GetChangeDetail(eventNR.GerritChange.ID, nil) + changeDetail, _, err := client.client.Changes.GetChangeDetail(buildStatus.GerritChange.ID, nil) if err != nil { return sdk.WrapError(err, "error while getting change detail") } @@ -55,7 +46,7 @@ func (client *gerritClient) SetStatus(ctx context.Context, event sdk.Event) erro } if !found { - if _, _, err := client.client.Changes.SetReview(eventNR.GerritChange.ID, eventNR.GerritChange.Revision, &ri); err != nil { + if _, _, err := client.client.Changes.SetReview(buildStatus.GerritChange.ID, buildStatus.GerritChange.Revision, &ri); err != nil { return sdk.WrapError(err, "unable to set gerrit review") } } @@ -67,24 +58,24 @@ func (client *gerritClient) ListStatuses(ctx context.Context, repo string, ref s return nil, nil } -func (client *gerritClient) buildMessage(eventNR sdk.EventRunWorkflowNode) string { +func (client *gerritClient) buildMessage(buildStatus sdk.VCSBuildStatus) string { var message string - switch eventNR.Status { + switch buildStatus.Status { case sdk.StatusSuccess: - message += fmt.Sprintf("Build Success on %s\n%s", eventNR.NodeName, eventNR.GerritChange.URL) + message += fmt.Sprintf("Build Success on %s\n%s", buildStatus.Context, buildStatus.GerritChange.URL) case sdk.StatusSkipped: - message += fmt.Sprintf("Build Skipped on %s\n%s", eventNR.NodeName, eventNR.GerritChange.URL) + message += fmt.Sprintf("Build Skipped on %s\n%s", buildStatus.Context, buildStatus.GerritChange.URL) case sdk.StatusFail, sdk.StatusStopped: - message += fmt.Sprintf("Build Failed on %s\n%s \n%s", eventNR.NodeName, eventNR.GerritChange.URL, eventNR.GerritChange.Report) + message += fmt.Sprintf("Build Failed on %s\n%s \n%s", buildStatus.Context, buildStatus.GerritChange.URL, buildStatus.GerritChange.Report) case sdk.StatusBuilding: - message += fmt.Sprintf("CDS starts working on %s\n%s", eventNR.NodeName, eventNR.GerritChange.URL) + message += fmt.Sprintf("CDS starts working on %s\n%s", buildStatus.Context, buildStatus.GerritChange.URL) } return message } -func (client *gerritClient) buildLabel(eventNR sdk.EventRunWorkflowNode) map[string]string { +func (client *gerritClient) buildLabel(buildStatus sdk.VCSBuildStatus) map[string]string { labels := make(map[string]string) - switch eventNR.Status { + switch buildStatus.Status { case sdk.StatusSuccess: labels["Verified"] = "1" case sdk.StatusFail, sdk.StatusStopped: diff --git a/engine/vcs/gerrit/gerrit.go b/engine/vcs/gerrit/gerrit.go index 99c966e7d5..a7ef1bfdd4 100644 --- a/engine/vcs/gerrit/gerrit.go +++ b/engine/vcs/gerrit/gerrit.go @@ -10,13 +10,12 @@ import ( // gerritClient implements VCSAuthorizedClient interface type gerritClient struct { - client *ger.Client - url string - disableStatusDetails bool - sshUsername string - sshPort int - reviewerName string - reviewerToken string + client *ger.Client + url string + sshUsername string + sshPort int + reviewerName string + reviewerToken string } // gerritConsumer implements vcs.Server and it's used to instantiate a gerritClient diff --git a/engine/vcs/gitea/client_status.go b/engine/vcs/gitea/client_status.go index 539f4a0e3d..1c83770cdd 100644 --- a/engine/vcs/gitea/client_status.go +++ b/engine/vcs/gitea/client_status.go @@ -6,11 +6,7 @@ import ( "github.com/ovh/cds/sdk" ) -func (client *giteaClient) SetDisableStatusDetails(disableStatusDetails bool) { - // no implementation for gitea -} - -func (client *giteaClient) SetStatus(ctx context.Context, event sdk.Event) error { +func (client *giteaClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { return sdk.WithStack(sdk.ErrNotImplemented) } diff --git a/engine/vcs/github/client_status.go b/engine/vcs/github/client_status.go index 0c3492efcb..d17cd093f8 100644 --- a/engine/vcs/github/client_status.go +++ b/engine/vcs/github/client_status.go @@ -16,7 +16,6 @@ import ( ) type statusData struct { - pipName string desc string status string repoFullName string @@ -25,41 +24,22 @@ type statusData struct { context string } -func (g *githubClient) SetDisableStatusDetails(disableStatusDetails bool) { - g.disableStatusDetails = disableStatusDetails -} - // SetStatus Users with push access can create commit statuses for a given ref: // https://developer.github.com/v3/repos/statuses/#create-a-status -func (g *githubClient) SetStatus(ctx context.Context, event sdk.Event) error { - var data statusData - var err error - switch event.EventType { - case fmt.Sprintf("%T", sdk.EventRunWorkflowNode{}): - data, err = processEventWorkflowNodeRun(event, g.uiURL, g.disableStatusDetails) - default: - log.Error(ctx, "github.SetStatus> Unknown event %v", event) - return nil - } - if err != nil { - return sdk.WrapError(err, "Cannot process Event") - } - - if data.status == "" { - log.Debug(ctx, "github.SetStatus> Do not process event for current status: %v", event) +func (g *githubClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { + if buildStatus.Status == "" { + log.Debug(ctx, "github.SetStatus> Do not process event for empty status") return nil } ghStatus := CreateStatus{ - Description: data.desc, - State: data.status, - Context: data.context, + Description: buildStatus.Description, + State: buildStatus.Status, + Context: buildStatus.Context, + TargetURL: buildStatus.URLCDS, } - if !g.disableStatusDetails { - ghStatus.TargetURL = data.urlPipeline - } - path := fmt.Sprintf("/repos/%s/statuses/%s", data.repoFullName, data.hash) + path := fmt.Sprintf("/repos/%s/statuses/%s", buildStatus.RepositoryFullname, buildStatus.GitHash) b, err := json.Marshal(ghStatus) if err != nil { @@ -84,7 +64,7 @@ func (g *githubClient) SetStatus(ctx context.Context, event sdk.Event) error { log.Debug(ctx, "SetStatus> github response for %v body:%v", path, string(body)) if res.StatusCode != 201 { - return sdk.WrapError(err, "Unable to create status on github. Status code : %d - Body: %s - target:%s", res.StatusCode, body, data.urlPipeline) + return sdk.WrapError(err, "Unable to create status on github. Status code : %d - Body: %s - context:%s", res.StatusCode, body, buildStatus.Context) } s := &Status{} @@ -179,7 +159,6 @@ func processEventWorkflowNodeRun(event sdk.Event, cdsUIURL string, disabledStatu } data.hash = eventNR.Hash data.repoFullName = eventNR.RepositoryFullName - data.pipName = eventNR.NodeName if !disabledStatusDetail { data.urlPipeline = fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", diff --git a/engine/vcs/github/github.go b/engine/vcs/github/github.go index b3d742de4d..f10b44b31d 100644 --- a/engine/vcs/github/github.go +++ b/engine/vcs/github/github.go @@ -9,17 +9,16 @@ import ( // githubClient is a github.com wrapper for CDS vcs. interface type githubClient struct { - GitHubURL string - GitHubAPIURL string - ClientID string - OAuthToken string - disableStatusDetails bool - Cache cache.Store - apiURL string - uiURL string - proxyURL string - username string - token string + GitHubURL string + GitHubAPIURL string + ClientID string + OAuthToken string + Cache cache.Store + apiURL string + uiURL string + proxyURL string + username string + token string } // GithubConsumer implements vcs.Server and it's used to instantiate a githubClient diff --git a/engine/vcs/gitlab/client_status.go b/engine/vcs/gitlab/client_status.go index eb0b4c59c1..ffc7d9d3e4 100644 --- a/engine/vcs/gitlab/client_status.go +++ b/engine/vcs/gitlab/client_status.go @@ -2,7 +2,6 @@ package gitlab import ( "context" - "fmt" "strings" "github.com/rockbears/log" @@ -43,48 +42,26 @@ func getGitlabStateFromStatus(s string) gitlab.BuildStateValue { return gitlab.Failed } -func (g *gitlabClient) SetDisableStatusDetails(disableStatusDetails bool) { - g.disableStatusDetails = disableStatusDetails -} - // SetStatus set build status on Gitlab -func (c *gitlabClient) SetStatus(ctx context.Context, event sdk.Event) error { +func (c *gitlabClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { if c.disableStatus { log.Warn(ctx, "disableStatus.SetStatus> ⚠ Gitlab statuses are disabled") return nil } - var data statusData - var err error - switch event.EventType { - case fmt.Sprintf("%T", sdk.EventRunWorkflowNode{}): - data, err = processWorkflowNodeRunEvent(event, c.uiURL) - default: - log.Debug(ctx, "gitlabClient.SetStatus> Unknown event %v", event) - return nil - } - - if err != nil { - return sdk.WrapError(err, "cannot process event %v", event) - } - - if c.disableStatusDetails { - data.url = "" - } - cds := "CDS" opt := &gitlab.SetCommitStatusOptions{ Name: &cds, Context: &cds, - State: getGitlabStateFromStatus(data.status), - Ref: &data.branchName, - TargetURL: &data.url, - Description: &data.desc, + State: getGitlabStateFromStatus(buildStatus.Status), + Ref: &buildStatus.GitHash, + TargetURL: &buildStatus.URLCDS, + Description: &buildStatus.Description, } - val, _, err := c.client.Commits.GetCommitStatuses(data.repoFullName, data.hash, nil) + val, _, err := c.client.Commits.GetCommitStatuses(buildStatus.RepositoryFullname, buildStatus.GitHash, nil) if err != nil { - return sdk.WrapError(err, "unable to get commit statuses - repo:%s hash:%s", data.repoFullName, data.hash) + return sdk.WrapError(err, "unable to get commit statuses - repo:%s hash:%s", buildStatus.RepositoryFullname, buildStatus.GitHash) } found := false @@ -92,19 +69,19 @@ func (c *gitlabClient) SetStatus(ctx context.Context, event sdk.Event) error { sameRequest := s.TargetURL == *opt.TargetURL && // Comparing TargetURL as there is the workflow run number inside s.Status == string(opt.State) && // Comparing Status to avoid duplicate entries s.Ref == *opt.Ref && // Comparing branches name - s.SHA == data.hash && // Comparing commit SHA to match the right commit + s.SHA == buildStatus.GitHash && // Comparing commit SHA to match the right commit s.Name == *opt.Name && // Comparing app name (CDS) s.Description == *opt.Description // Comparing Description as there are the pipelines names inside if sameRequest { - log.Debug(ctx, "gitlabClient.SetStatus> Duplicate commit status, ignoring request - repo:%s hash:%s", data.repoFullName, data.hash) + log.Debug(ctx, "gitlabClient.SetStatus> Duplicate commit status, ignoring request - repo:%s hash:%s", buildStatus.RepositoryFullname, buildStatus.GitHash) found = true break } } if !found { - if _, _, err := c.client.Commits.SetCommitStatus(data.repoFullName, data.hash, opt); err != nil { - return sdk.WrapError(err, "cannot process event %v - repo:%s hash:%s", event, data.repoFullName, data.hash) + if _, _, err := c.client.Commits.SetCommitStatus(buildStatus.RepositoryFullname, buildStatus.GitHash, opt); err != nil { + return sdk.WrapError(err, "cannot process event repo:%s hash:%s", buildStatus.RepositoryFullname, buildStatus.GitHash) } } return nil @@ -144,25 +121,3 @@ func processGitlabState(s gitlab.CommitStatus) string { return sdk.StatusDisabled } } - -func processWorkflowNodeRunEvent(event sdk.Event, uiURL string) (statusData, error) { - data := statusData{} - var eventNR sdk.EventRunWorkflowNode - if err := sdk.JSONUnmarshal(event.Payload, &eventNR); err != nil { - return data, sdk.WrapError(err, "cannot read payload") - } - - data.url = fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", - uiURL, - event.ProjectKey, - event.WorkflowName, - eventNR.Number, - ) - - data.desc = sdk.VCSCommitStatusDescription(event.ProjectKey, event.WorkflowName, eventNR) - data.hash = eventNR.Hash - data.repoFullName = eventNR.RepositoryFullName - data.status = eventNR.Status - data.branchName = eventNR.BranchName - return data, nil -} diff --git a/engine/vcs/gitlab/gitlab.go b/engine/vcs/gitlab/gitlab.go index d351cdbf81..62e04b93ad 100644 --- a/engine/vcs/gitlab/gitlab.go +++ b/engine/vcs/gitlab/gitlab.go @@ -18,11 +18,10 @@ var ( // gitlabClient implements VCSAuthorizedClient interface type gitlabClient struct { - client *gitlab.Client - uiURL string - proxyURL string - disableStatus bool - disableStatusDetails bool + client *gitlab.Client + uiURL string + proxyURL string + disableStatus bool } // gitlabConsumer implements vcs.Server and it's used to instantiate a gitlabClient diff --git a/engine/vcs/vcs_handlers.go b/engine/vcs/vcs_handlers.go index e1b0186193..bf16ec50b1 100644 --- a/engine/vcs/vcs_handlers.go +++ b/engine/vcs/vcs_handlers.go @@ -592,8 +592,8 @@ func (s *Service) postStatusHandler() service.Handler { return func(ctx context.Context, _ http.ResponseWriter, r *http.Request) error { name := muxVar(r, "name") - evt := sdk.Event{} - if err := service.UnmarshalBody(r, &evt); err != nil { + buildStatus := sdk.VCSBuildStatus{} + if err := service.UnmarshalBody(r, &buildStatus); err != nil { return sdk.WrapError(err, "Unable to read body") } @@ -612,15 +612,7 @@ func (s *Service) postStatusHandler() service.Handler { return sdk.WrapError(err, "Unable to get authorized client") } - var disableStatusDetails bool - d := r.URL.Query().Get("disableStatusDetails") - if d != "" { - disableStatusDetails, _ = strconv.ParseBool(d) - } - - client.SetDisableStatusDetails(disableStatusDetails) - - if err := client.SetStatus(ctx, evt); err != nil { + if err := client.SetStatus(ctx, buildStatus); err != nil { return sdk.WrapError(err, "Unable to set status on %s", name) } diff --git a/engine/vcs/vcs_router.go b/engine/vcs/vcs_router.go index bede98183f..e9684e7c04 100644 --- a/engine/vcs/vcs_router.go +++ b/engine/vcs/vcs_router.go @@ -49,4 +49,6 @@ func (s *Service) initRouter(ctx context.Context) { r.Handle("/vcs/{name}/repos/{owner}/{repo}/search/pullrequest", nil, r.GET(s.SearchPullRequestHandler)) r.Handle("/vcs/{name}/status", nil, r.POST(s.postStatusHandler)) + + // TOTO yesnault add route v2 to post status, and another to post a comment } diff --git a/sdk/vcs.go b/sdk/vcs.go index ff75b21f84..f9b7b349d4 100644 --- a/sdk/vcs.go +++ b/sdk/vcs.go @@ -301,10 +301,6 @@ type VCSServer interface { GetAuthorizedClient(context.Context, VCSAuth) (VCSAuthorizedClient, error) } -type VCSServerService interface { - GetAuthorizedClient(context.Context, string, string, int64) (VCSAuthorizedClientService, error) -} - type VCSBranchFilters struct { BranchName string Default bool @@ -358,8 +354,7 @@ type VCSAuthorizedClientCommon interface { PullRequestEvents(context.Context, string, []interface{}) ([]VCSPullRequestEvent, error) // Set build status on repository - SetStatus(ctx context.Context, event Event) error - SetDisableStatusDetails(disableStatusDetails bool) + SetStatus(ctx context.Context, buildStatus VCSBuildStatus) error ListStatuses(ctx context.Context, repo string, ref string) ([]VCSCommitStatus, error) // Release @@ -419,3 +414,25 @@ func VCSCommitStatusDescription(projKey, workflowName string, evt EventRunWorkfl ) return fmt.Sprintf("CDS/%s", key) } + +type VCSBuildStatus struct { + // v1:eventNR.NodeName + ": " + eventNR.Status + // v2: Workflow.Name + ": " +Status + Description string `json:"description"` + + // v1: fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", cdsUIURL, event.ProjectKey, event.WorkflowName, eventNR.Number) + // v2: TODO + URLCDS string `json:"url_cds"` // + + // v1: fmt.Sprintf("%s-%s-%s", event.ProjectKey, event.WorkflowName, eventNR.NodeName) + // v2: TODO + Context string `json:"context"` + + Status string `json:"status"` + + RepositoryFullname string `json:"repository_fullname"` + GitHash string `json:"git_hash"` + + // from v1 workflow only + GerritChange *GerritChangeEvent `json:"gerrit_change,omitempty"` +} From e169995bec27cee8be6f354bb7dc888a922cef41 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Wed, 7 Feb 2024 16:22:38 +0000 Subject: [PATCH 07/17] feat: authorizedClient without tx Signed-off-by: Yvonnick Esnault --- engine/api/application.go | 14 +-- engine/api/hook.go | 18 +--- engine/api/repositories_manager.go | 14 +-- engine/api/ui.go | 20 ++-- engine/api/v2_entities.go | 12 +-- engine/api/v2_hooks.go | 41 ++------- engine/api/v2_project_clean_ascode.go | 10 +- engine/api/v2_project_repository.go | 12 +-- engine/api/v2_repository_analyze.go | 107 +++++++--------------- engine/api/v2_workflow_run_craft.go | 35 +------ engine/api/workflow/workflow_run_event.go | 18 +--- engine/api/workflow_application.go | 12 +-- engine/api/workflow_hook.go | 18 +--- engine/vcs/vcs_router.go | 2 - 14 files changed, 77 insertions(+), 256 deletions(-) diff --git a/engine/api/application.go b/engine/api/application.go index 7e4c81832a..09fe4a6aa0 100644 --- a/engine/api/application.go +++ b/engine/api/application.go @@ -191,13 +191,9 @@ func (api *API) getApplicationVCSInfosHandler() service.Handler { applicationName := vars["applicationName"] remote := r.FormValue("remote") - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint + db := api.mustDB() - app, err := application.LoadByName(ctx, tx, projectKey, applicationName, application.LoadOptions.Default) + app, err := application.LoadByName(ctx, db, projectKey, applicationName, application.LoadOptions.Default) if err != nil { return sdk.WrapError(err, "cannot load application %s for project %s from db", applicationName, projectKey) } @@ -212,7 +208,7 @@ func (api *API) getApplicationVCSInfosHandler() service.Handler { return service.WriteJSON(w, resp, http.StatusOK) } - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, projectKey, app.VCSServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, projectKey, app.VCSServer) if err != nil { return sdk.NewErrorWithStack(err, sdk.NewErrorFrom(sdk.ErrNoReposManagerClientAuth, "cannot get vcs server %s for project %s", app.VCSServer, projectKey)) } @@ -246,10 +242,6 @@ func (api *API) getApplicationVCSInfosHandler() service.Handler { }) } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - return service.WriteJSON(w, resp, http.StatusOK) } } diff --git a/engine/api/hook.go b/engine/api/hook.go index ab38b04036..7274d702d7 100644 --- a/engine/api/hook.go +++ b/engine/api/hook.go @@ -34,32 +34,24 @@ func (api *API) getHookPollingVCSEvents() service.Handler { } } - h, err := workflow.LoadHookByUUID(api.mustDB(), uuid) - if err != nil { - return err - } + db := api.mustDB() - proj, err := project.Load(ctx, api.mustDB(), h.Config[sdk.HookConfigProject].Value, nil) + h, err := workflow.LoadHookByUUID(db, uuid) if err != nil { return err } - tx, err := api.mustDB().Begin() + proj, err := project.Load(ctx, db, h.Config[sdk.HookConfigProject].Value, nil) if err != nil { - return sdk.WithStack(err) + return err } - defer tx.Rollback() // nolint //get the client for the repositories manager - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, proj.Key, vcsServerParam) + client, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, proj.Key, vcsServerParam) if err != nil { return err } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - //Check if the polling if disabled if info, err := repositoriesmanager.GetPollingInfos(ctx, client, *proj); err != nil { return sdk.WrapError(err, "cannot check if polling is enabled") diff --git a/engine/api/repositories_manager.go b/engine/api/repositories_manager.go index 9e991c3ea4..163449db10 100644 --- a/engine/api/repositories_manager.go +++ b/engine/api/repositories_manager.go @@ -77,27 +77,17 @@ func (api *API) getRepoFromRepositoriesManagerHandler() service.Handler { return sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("Missing repository name 'repo' as a query parameter")) } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint - - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, projectKey, rmName) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projectKey, rmName) if err != nil { return sdk.NewErrorWithStack(err, sdk.NewErrorFrom(sdk.ErrNoReposManagerClientAuth, "cannot get client got %s %s", projectKey, rmName)) } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - log.Info(ctx, "getRepoFromRepositoriesManagerHandler> Loading repository on %s", rmName) repo, err := client.RepoByFullname(ctx, repoName) if err != nil { - return sdk.WrapError(err, "cannot get repos") + return sdk.WrapError(err, "cannot get repo %v", repoName) } return service.WriteJSON(w, repo, http.StatusOK) diff --git a/engine/api/ui.go b/engine/api/ui.go index b2b3c09bff..174f2da5e5 100644 --- a/engine/api/ui.go +++ b/engine/api/ui.go @@ -38,18 +38,14 @@ func (api *API) getApplicationOverviewHandler() service.Handler { projectKey := vars[permProjectKey] appName := vars["applicationName"] - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint + db := api.mustDB() - app, err := application.LoadByName(ctx, tx, projectKey, appName) + app, err := application.LoadByName(ctx, db, projectKey, appName) if err != nil { return sdk.WrapError(err, "unable to load application") } - usage, err := loadApplicationUsage(ctx, tx, projectKey, appName) + usage, err := loadApplicationUsage(ctx, db, projectKey, appName) if err != nil { return sdk.WrapError(err, "cannot load application usage") } @@ -62,7 +58,7 @@ func (api *API) getApplicationOverviewHandler() service.Handler { if len(srvs) > 0 { // Get metrics - mTest, err := metrics.GetMetrics(ctx, tx, projectKey, app.ID, sdk.MetricKeyUnitTest) + mTest, err := metrics.GetMetrics(ctx, db, projectKey, app.ID, sdk.MetricKeyUnitTest) if err != nil { return sdk.WrapError(err, "cannot list Unit test metrics") } @@ -75,7 +71,7 @@ func (api *API) getApplicationOverviewHandler() service.Handler { if app.VCSServer != "" { // GET VCS URL // Get vcs info to known if we are on the default branch or not - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, projectKey, app.VCSServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, projectKey, app.VCSServer) if err != nil { return sdk.NewErrorWithStack(err, sdk.NewErrorFrom(sdk.ErrNoReposManagerClientAuth, "cannot get repo client %s", app.VCSServer)) @@ -94,7 +90,7 @@ func (api *API) getApplicationOverviewHandler() service.Handler { tagFilter := make(map[string]string, 1) tagFilter["git.branch"] = defaultBranch.DisplayID for _, w := range app.Usage.Workflows { - runs, _, _, _, err := workflow.LoadRunsSummaries(ctx, tx, projectKey, w.Name, 0, 5, tagFilter) + runs, _, _, _, err := workflow.LoadRunsSummaries(ctx, db, projectKey, w.Name, 0, 5, tagFilter) if err != nil { return sdk.WrapError(err, "unable to load runs") } @@ -102,10 +98,6 @@ func (api *API) getApplicationOverviewHandler() service.Handler { } } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - return service.WriteJSON(w, appOverview, http.StatusOK) } } diff --git a/engine/api/v2_entities.go b/engine/api/v2_entities.go index 5e356bc1cf..0dd1ccab83 100644 --- a/engine/api/v2_entities.go +++ b/engine/api/v2_entities.go @@ -30,22 +30,12 @@ func (api *API) getEntityRefFromQueryParams(ctx context.Context, req *http.Reque } if branch == "" { - tx, err := api.mustDB().Begin() + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projKey, vcsName) if err != nil { return "", err } - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, projKey, vcsName) - if err != nil { - _ = tx.Rollback() - return "", err - } defaultBranch, err := vcsClient.Branch(ctx, repoName, sdk.VCSBranchFilters{Default: true}) if err != nil { - _ = tx.Rollback() - return "", err - } - if err := tx.Commit(); err != nil { - _ = tx.Rollback() return "", err } ref = defaultBranch.ID diff --git a/engine/api/v2_hooks.go b/engine/api/v2_hooks.go index 1436555890..6cfd5f2667 100644 --- a/engine/api/v2_hooks.go +++ b/engine/api/v2_hooks.go @@ -86,13 +86,7 @@ func (api *API) postHookEventRetrieveSignKeyHandler() ([]service.RbacChecker, se return err } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint - - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, hookRetrieveSignKey.ProjectKey, hookRetrieveSignKey.VCSServerName) + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, hookRetrieveSignKey.ProjectKey, hookRetrieveSignKey.VCSServerName) if err != nil { return err } @@ -101,9 +95,6 @@ func (api *API) postHookEventRetrieveSignKeyHandler() ([]service.RbacChecker, se log.Info(ctx, "unable to get repository %s/%s for project %s", hookRetrieveSignKey.VCSServerName, hookRetrieveSignKey.RepositoryName, hookRetrieveSignKey.ProjectKey) return err } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } cloneURL := repo.SSHCloneURL if vcsProjectWithSecret.Auth.SSHKeyName == "" { @@ -173,11 +164,13 @@ func (api *API) postRetrieveWorkflowToTriggerHandler() ([]service.RbacChecker, s ctx = context.WithValue(ctx, cdslog.HookEventID, hookRequest.HookEventUUID) + db := api.mustDB() + uniqueWorkflowMap := make(map[string]struct{}) filteredWorkflowHooks := make([]sdk.V2WorkflowHook, 0) // Get repository web hooks - workflowHooks, err := LoadWorkflowHooksWithRepositoryWebHooks(ctx, api.mustDB(), hookRequest) + workflowHooks, err := LoadWorkflowHooksWithRepositoryWebHooks(ctx, db, hookRequest) if err != nil { return err } @@ -190,7 +183,7 @@ func (api *API) postRetrieveWorkflowToTriggerHandler() ([]service.RbacChecker, s } // Get workflow_update hooks - workflowUpdateHooks, err := LoadWorkflowHooksWithWorkflowUpdate(ctx, api.mustDB(), hookRequest) + workflowUpdateHooks, err := LoadWorkflowHooksWithWorkflowUpdate(ctx, db, hookRequest) if err != nil { return err } @@ -203,7 +196,7 @@ func (api *API) postRetrieveWorkflowToTriggerHandler() ([]service.RbacChecker, s } // Get model_update hooks - modelUpdateHooks, err := LoadWorkflowHooksWithModelUpdate(ctx, api.mustDB(), hookRequest) + modelUpdateHooks, err := LoadWorkflowHooksWithModelUpdate(ctx, db, hookRequest) if err != nil { return err } @@ -215,17 +208,11 @@ func (api *API) postRetrieveWorkflowToTriggerHandler() ([]service.RbacChecker, s } } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() - hooksWithReadRight := make([]sdk.V2WorkflowHook, 0) for _, h := range filteredWorkflowHooks { if !hookRequest.AnayzedProjectKeys.Contains(h.ProjectKey) { // Check project right - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, h.ProjectKey, hookRequest.VCSName) + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, h.ProjectKey, hookRequest.VCSName) if err != nil { return err } @@ -237,10 +224,6 @@ func (api *API) postRetrieveWorkflowToTriggerHandler() ([]service.RbacChecker, s hooksWithReadRight = append(hooksWithReadRight, h) } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - return service.WriteJSON(w, hooksWithReadRight, http.StatusOK) } } @@ -377,14 +360,10 @@ func (api *API) getRepositoryWebHookSecretHandler() ([]service.RbacChecker, serv return err } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() + db := api.mustDB() // Check if project has read access - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, pKey, vcsName) + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, pKey, vcsName) if err != nil { return err } @@ -392,7 +371,7 @@ func (api *API) getRepositoryWebHookSecretHandler() ([]service.RbacChecker, serv return err } - srvs, err := services.LoadAllByType(ctx, tx, sdk.TypeHooks) + srvs, err := services.LoadAllByType(ctx, db, sdk.TypeHooks) if err != nil { return err } diff --git a/engine/api/v2_project_clean_ascode.go b/engine/api/v2_project_clean_ascode.go index d017a73c33..f8c2690fb1 100644 --- a/engine/api/v2_project_clean_ascode.go +++ b/engine/api/v2_project_clean_ascode.go @@ -138,13 +138,7 @@ func cleanAscodeProject(ctx context.Context, db *gorp.DbMap, store cache.Store, } func (c *EntitiesCleaner) getBranches(ctx context.Context, db *gorp.DbMap, store cache.Store) error { - tx, err := db.Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() - - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, store, c.projKey, c.vcsName) + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, store, c.projKey, c.vcsName) if err != nil { return err } @@ -166,7 +160,7 @@ func (c *EntitiesCleaner) getBranches(ctx context.Context, db *gorp.DbMap, store for _, t := range tags { c.refs[sdk.GitRefTagPrefix+t.Tag] = struct{}{} } - return sdk.WithStack(tx.Commit()) + return nil } func (c *EntitiesCleaner) cleanEntitiesByRef(ctx context.Context, db *gorp.DbMap, store cache.Store, ref string, entitiesByBranch []sdk.Entity) error { diff --git a/engine/api/v2_project_repository.go b/engine/api/v2_project_repository.go index f51ca3ca70..5eb15d9f42 100644 --- a/engine/api/v2_project_repository.go +++ b/engine/api/v2_project_repository.go @@ -310,15 +310,8 @@ func (api *API) getProjectRepositoryBranchesHandler() ([]service.RbacChecker, se return err } - tx, err := api.mustDB().Begin() - if err != nil { - return err - } - defer tx.Rollback() - - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, pKey, vcsProject.Name) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, pKey, vcsProject.Name) if err != nil { - _ = tx.Rollback() // nolint return err } branches, err := client.Branches(ctx, repo.Name, sdk.VCSBranchesFilter{Limit: int64(limit)}) @@ -326,9 +319,6 @@ func (api *API) getProjectRepositoryBranchesHandler() ([]service.RbacChecker, se return err } - if err := tx.Commit(); err != nil { - return err - } return service.WriteJSON(w, branches, http.StatusOK) } } diff --git a/engine/api/v2_repository_analyze.go b/engine/api/v2_repository_analyze.go index bc45e86c59..b0c047abe3 100644 --- a/engine/api/v2_repository_analyze.go +++ b/engine/api/v2_repository_analyze.go @@ -209,21 +209,10 @@ func (api *API) postRepositoryAnalysisHandler() ([]service.RbacChecker, service. return err } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() - if analysis.Commit == "" || analysis.Ref == "" { // retrieve commit for the given ref - tx, err := api.mustDB().Begin() - if err != nil { - return err - } - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, analysis.ProjectKey, vcs.Name) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, analysis.ProjectKey, vcs.Name) if err != nil { - _ = tx.Rollback() return err } @@ -232,14 +221,12 @@ func (api *API) postRepositoryAnalysisHandler() ([]service.RbacChecker, service. case strings.HasPrefix(analysis.Ref, sdk.GitRefTagPrefix): giventTag, err := client.Tag(ctx, repo.Name, strings.TrimPrefix(analysis.Ref, sdk.GitRefTagPrefix)) if err != nil { - _ = tx.Rollback() return err } analysis.Commit = giventTag.Sha default: givenBranch, err := client.Branch(ctx, repo.Name, sdk.VCSBranchFilters{BranchName: strings.TrimPrefix(analysis.Ref, sdk.GitRefBranchPrefix)}) if err != nil { - _ = tx.Rollback() return err } analysis.Commit = givenBranch.LatestCommit @@ -252,10 +239,6 @@ func (api *API) postRepositoryAnalysisHandler() ([]service.RbacChecker, service. analysis.Ref = defaultBranch.ID analysis.Commit = defaultBranch.LatestCommit } - - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } } var u *sdk.AuthentifiedUser @@ -275,6 +258,13 @@ func (api *API) postRepositoryAnalysisHandler() ([]service.RbacChecker, service. hookEventUUID: analysis.HookEventUUID, user: u, } + + tx, err := api.mustDB().Begin() + if err != nil { + return sdk.WithStack(err) + } + defer tx.Rollback() + a, err := api.createAnalyze(ctx, tx, createAnalysis) if err != nil { return err @@ -549,14 +539,8 @@ skipEntity: entitiesToUpdate = append(entitiesToUpdate, *e) } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint - // Insert / Update entities - vcsAuthClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, analysis.ProjectKey, vcsProjectWithSecret.Name) + vcsAuthClient, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, analysis.ProjectKey, vcsProjectWithSecret.Name) if err != nil { return api.stopAnalysis(ctx, analysis, sdk.NewErrorFrom(sdk.ErrNotFound, "unable to retrieve vcs_server %s on project %s", vcsProjectWithSecret.Name, analysis.ProjectKey)) } @@ -567,6 +551,12 @@ skipEntity: eventInsertedEntities := make([]sdk.Entity, 0) eventUpdatedEntities := make([]sdk.Entity, 0) + + tx, err := api.mustDB().Begin() + if err != nil { + return sdk.WithStack(err) + } + defer tx.Rollback() // nolint for i := range entitiesToUpdate { e := &entities[i] existingEntity, err := entity.LoadByRefTypeName(ctx, tx, e.ProjectRepositoryID, e.Ref, e.Type, e.Name) @@ -884,19 +874,19 @@ func findCommitter(ctx context.Context, cache cache.Store, db *gorp.DbMap, sha, } - // Get commit - tx, err := db.Begin() + client, err := repositoriesmanager.AuthorizedClient(ctx, db, cache, projKey, vcsProjectWithSecret.Name) if err != nil { return nil, "", "", sdk.WithStack(err) } - defer tx.Rollback() // nolint - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, cache, projKey, vcsProjectWithSecret.Name) + var commitUser *sdk.AuthentifiedUser + + // Get commit + tx, err := db.Begin() if err != nil { return nil, "", "", sdk.WithStack(err) } - - var commitUser *sdk.AuthentifiedUser + defer tx.Rollback() // nolint switch vcsProjectWithSecret.Type { case sdk.VCSTypeBitbucketServer, sdk.VCSTypeGitlab: @@ -904,7 +894,7 @@ func findCommitter(ctx context.Context, cache cache.Store, db *gorp.DbMap, sha, if err != nil { return nil, "", "", err } - commitUser, err = user.LoadByUsername(ctx, tx, commit.Committer.Slug) + commitUser, err = user.LoadByUsername(ctx, db, commit.Committer.Slug) if err != nil { if !sdk.ErrorIs(err, sdk.ErrNotFound) { return nil, "", "", sdk.WithStack(sdk.NewErrorFrom(err, "unable to get user %s", commit.Committer.Slug)) @@ -922,7 +912,7 @@ func findCommitter(ctx context.Context, cache cache.Store, db *gorp.DbMap, sha, } // Retrieve user link by external ID - userLink, err := link.LoadUserLinkByTypeAndExternalID(ctx, tx, vcsProjectWithSecret.Type, commit.Committer.ID) + userLink, err := link.LoadUserLinkByTypeAndExternalID(ctx, db, vcsProjectWithSecret.Type, commit.Committer.ID) if err != nil { if !sdk.ErrorIs(err, sdk.ErrNotFound) { return nil, "", "", err @@ -1029,39 +1019,29 @@ func (api *API) analyzeCommitSignatureThroughVcsAPI(ctx context.Context, analysi ctx, next := telemetry.Span(ctx, "api.analyzeCommitSignatureThroughVcsAPI") defer next() - tx, err := api.mustDB().Begin() - if err != nil { - return keyID, analyzesError, sdk.WithStack(err) - } // Check commit signature - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, analysis.ProjectKey, vcsProject.Name) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, analysis.ProjectKey, vcsProject.Name) if err != nil { - _ = tx.Rollback() // nolint return keyID, analyzesError, err } vcsCommit, err := client.Commit(ctx, repoWithSecret.Name, analysis.Commit) if err != nil { - _ = tx.Rollback() // nolint return keyID, analyzesError, err } - if err := tx.Commit(); err != nil { - return keyID, analyzesError, sdk.WithStack(err) - } if vcsCommit.Hash == "" { return keyID, analyzesError, fmt.Errorf("commit %s not found", analysis.Commit) - } else { - if vcsCommit.Signature != "" { - keyId, err := gpg.GetKeyIdFromSignature(vcsCommit.Signature) - if err != nil { - return keyID, analyzesError, fmt.Errorf("unable to extract keyID from signature: %v", err) - } else { - keyID = keyId - } + } + if vcsCommit.Signature != "" { + keyId, err := gpg.GetKeyIdFromSignature(vcsCommit.Signature) + if err != nil { + return keyID, analyzesError, fmt.Errorf("unable to extract keyID from signature: %v", err) } else { - analyzesError = fmt.Sprintf("commit %s is not signed", vcsCommit.Hash) + keyID = keyId } + } else { + analyzesError = fmt.Sprintf("commit %s is not signed", vcsCommit.Hash) } return keyID, analyzesError, nil } @@ -1116,13 +1096,7 @@ func (api *API) getCdsFilesOnVCSDirectory(ctx context.Context, analysis *sdk.Pro ctx, next := telemetry.Span(ctx, "api.getCdsFilesOnVCSDirectory") defer next() - tx, err := api.mustDB().Begin() - if err != nil { - return nil, sdk.WithStack(err) - } - defer tx.Rollback() // nolint - - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, analysis.ProjectKey, vcsName) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, analysis.ProjectKey, vcsName) if err != nil { return nil, sdk.WithStack(err) } @@ -1155,9 +1129,6 @@ func (api *API) getCdsFilesOnVCSDirectory(ctx context.Context, analysis *sdk.Pro } } } - if err := tx.Commit(); err != nil { - return nil, sdk.WithStack(err) - } return filesContent, nil } @@ -1165,13 +1136,7 @@ func (api *API) getCdsArchiveFileOnRepo(ctx context.Context, repo sdk.ProjectRep ctx, next := telemetry.Span(ctx, "api.getCdsArchiveFileOnRepo") defer next() - tx, err := api.mustDB().Begin() - if err != nil { - return nil, sdk.WithStack(err) - } - defer tx.Rollback() // nolint - - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, analysis.ProjectKey, vcsName) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, analysis.ProjectKey, vcsName) if err != nil { return nil, sdk.WithStack(err) } @@ -1206,10 +1171,6 @@ func (api *API) getCdsArchiveFileOnRepo(ctx context.Context, repo sdk.ProjectRep return nil, sdk.NewErrorWithStack(err, sdk.NewErrorFrom(sdk.ErrWrongRequest, "unable to read tar file")) } filesContent[dir+fileName] = buff.Bytes() - - } - if err := tx.Commit(); err != nil { - return nil, sdk.WithStack(err) } return filesContent, nil } diff --git a/engine/api/v2_workflow_run_craft.go b/engine/api/v2_workflow_run_craft.go index 591b420685..02b7aea1e2 100644 --- a/engine/api/v2_workflow_run_craft.go +++ b/engine/api/v2_workflow_run_craft.go @@ -430,23 +430,14 @@ func (wref *WorkflowRunEntityFinder) searchEntity(ctx context.Context, db *gorp. ref = defaultCache } else { // Get default branch - tx, err := db.Begin() + client, err := repositoriesmanager.AuthorizedClient(ctx, db, store, projKey, entityVCS.Name) if err != nil { - return "", nil, sdk.WithStack(err) - } - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, store, projKey, entityVCS.Name) - if err != nil { - _ = tx.Rollback() return "", nil, err } b, err := client.Branch(ctx, entityRepo.Name, sdk.VCSBranchFilters{Default: true}) if err != nil { - _ = tx.Rollback() return "", nil, err } - if err := tx.Commit(); err != nil { - return "", nil, sdk.WithStack(err) - } ref = b.ID wref.repoDefaultRefCache[entityVCS.Name+"/"+entityRepo.Name] = ref } @@ -455,18 +446,12 @@ func (wref *WorkflowRunEntityFinder) searchEntity(ctx context.Context, db *gorp. ref = branchOrTag } else { // Need to known if branchOrTag is a tag or a branch - tx, err := db.Begin() + client, err := repositoriesmanager.AuthorizedClient(ctx, db, store, projKey, entityVCS.Name) if err != nil { - return "", nil, sdk.WithStack(err) - } - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, store, projKey, entityVCS.Name) - if err != nil { - _ = tx.Rollback() return "", nil, err } b, err := client.Branch(ctx, entityRepo.Name, sdk.VCSBranchFilters{BranchName: branchOrTag}) if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) { - _ = tx.Rollback() return "", nil, err } @@ -474,17 +459,12 @@ func (wref *WorkflowRunEntityFinder) searchEntity(ctx context.Context, db *gorp. // try to get tag t, err := client.Tag(ctx, entityRepo.Name, branchOrTag) if err != nil { - _ = tx.Rollback() return "", nil, err } ref = sdk.GitRefTagPrefix + t.Tag } else { ref = b.ID } - if err := tx.Commit(); err != nil { - return "", nil, sdk.WithStack(err) - } - } completePath := fmt.Sprintf("%s/%s/%s/%s", projKey, vcsName, repoName, entityName) @@ -763,12 +743,7 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s gitContext.Repository = repo.Name } - tx, err := db.Begin() - if err != nil { - return nil, sdk.WithStack(err) - } - defer tx.Rollback() - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, tx, store, wr.ProjectKey, workflowVCSServer.Name) + vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, store, wr.ProjectKey, workflowVCSServer.Name) if err != nil { return nil, err } @@ -799,10 +774,6 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s } } - if err := tx.Commit(); err != nil { - return nil, sdk.WithStack(err) - } - // Env context envs := make(map[string]string) for k, v := range wr.WorkflowData.Workflow.Env { diff --git a/engine/api/workflow/workflow_run_event.go b/engine/api/workflow/workflow_run_event.go index 5512e327e0..01f418d555 100644 --- a/engine/api/workflow/workflow_run_event.go +++ b/engine/api/workflow/workflow_run_event.go @@ -48,12 +48,6 @@ func ResyncCommitStatus(ctx context.Context, db *gorp.DbMap, store cache.Store, func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, store cache.Store, proj sdk.Project, wr sdk.WorkflowRun, nodeRun sdk.WorkflowNodeRun, cdsUIURL string) error { ctx = context.WithValue(ctx, cdslog.NodeRunID, nodeRun.ID) - tx, err := db.Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint - if nodeRun.Status == sdk.StatusWaiting { return nil } @@ -101,7 +95,7 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st //Get the RepositoriesManager Client if e.vcsClient == nil { var err error - e.vcsClient, err = repositoriesmanager.AuthorizedClient(ctx, tx, store, proj.Key, vcsServerName) + e.vcsClient, err = repositoriesmanager.AuthorizedClient(ctx, db, store, proj.Key, vcsServerName) if err != nil { return sdk.WrapError(err, "can't get AuthorizedClient for %v/%v", proj.Key, vcsServerName) } @@ -144,7 +138,7 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st if statusFound == nil || statusFound.State == "" { for i := range notifs { log.Info(ctx, "status %q %s not found, sending a new one %+v", expected, nodeRun.Status, notifs[i]) - if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName, cdsUIURL); err != nil { + if err := e.sendVCSEventStatus(ctx, db, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName, cdsUIURL); err != nil { return sdk.WrapError(err, "can't sendVCSEventStatus vcs %v/%v", proj.Key, vcsServerName) } } @@ -175,7 +169,7 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st if !skipStatus { for i := range notifs { log.Info(ctx, "status %q %s not found, sending a new one %+v", expected, nodeRun.Status, notifs[i]) - if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName, cdsUIURL); err != nil { + if err := e.sendVCSEventStatus(ctx, db, store, proj.Key, wr, &nodeRun, notifs[i], vcsServerName, cdsUIURL); err != nil { return sdk.WrapError(err, "can't sendVCSEventStatus vcs %v/%v", proj.Key, vcsServerName) } } @@ -186,15 +180,11 @@ func (e *VCSEventMessenger) SendVCSEvent(ctx context.Context, db *gorp.DbMap, st return nil } for i := range notifs { - if err := e.sendVCSPullRequestComment(ctx, tx, wr, &nodeRun, notifs[i], vcsServerName); err != nil { + if err := e.sendVCSPullRequestComment(ctx, db, wr, &nodeRun, notifs[i], vcsServerName); err != nil { return sdk.WrapError(err, "can't sendVCSPullRequestComment vcs %v/%v", proj.Key, vcsServerName) } } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - return nil } diff --git a/engine/api/workflow_application.go b/engine/api/workflow_application.go index 74cc82d4ad..da7c65dcd8 100644 --- a/engine/api/workflow_application.go +++ b/engine/api/workflow_application.go @@ -59,21 +59,11 @@ func (api *API) releaseApplicationWorkflowHandler() service.Handler { return sdk.NewErrorFrom(sdk.ErrNoReposManager, "app.VCSServer is empty") } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint - - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, proj.Key, app.VCSServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, proj.Key, app.VCSServer) if err != nil { return sdk.WrapError(err, "cannot get client got %s %s", key, app.VCSServer) } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - release, errRelease := client.Release(ctx, app.RepositoryFullname, req.TagName, req.ReleaseTitle, req.ReleaseContent) if errRelease != nil { return sdk.WithStack(errRelease) diff --git a/engine/api/workflow_hook.go b/engine/api/workflow_hook.go index 8e702112d1..c4943ea79c 100644 --- a/engine/api/workflow_hook.go +++ b/engine/api/workflow_hook.go @@ -65,18 +65,14 @@ func (api *API) getWorkflowHookModelsHandler() service.Handler { return err } - tx, err := api.mustDB().Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint + db := api.mustDB() - p, err := project.Load(ctx, tx, key, project.LoadOptions.WithIntegrations) + p, err := project.Load(ctx, db, key, project.LoadOptions.WithIntegrations) if err != nil { return sdk.WithStack(err) } - wf, err := workflow.Load(ctx, tx, api.Cache, *p, workflowName, workflow.LoadOptions{}) + wf, err := workflow.Load(ctx, db, api.Cache, *p, workflowName, workflow.LoadOptions{}) if err != nil { return sdk.WithStack(err) } @@ -86,7 +82,7 @@ func (api *API) getWorkflowHookModelsHandler() service.Handler { return sdk.WithStack(sdk.ErrWorkflowNodeNotFound) } - m, err := workflow.LoadHookModels(tx) + m, err := workflow.LoadHookModels(db) if err != nil { return sdk.WithStack(err) } @@ -100,7 +96,7 @@ func (api *API) getWorkflowHookModelsHandler() service.Handler { var webHookInfo repositoriesmanager.WebhooksInfos if hasRepoManager { // Call VCS to know if repository allows webhook and get the configuration fields - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, p.Key, wf.GetApplication(node.Context.ApplicationID).VCSServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, p.Key, wf.GetApplication(node.Context.ApplicationID).VCSServer) if err == nil { webHookInfo, err = repositoriesmanager.GetWebhooksInfos(ctx, client) if err != nil { @@ -164,10 +160,6 @@ func (api *API) getWorkflowHookModelsHandler() service.Handler { } } - if err := tx.Commit(); err != nil { - return sdk.WithStack(err) - } - return service.WriteJSON(w, models, http.StatusOK) } } diff --git a/engine/vcs/vcs_router.go b/engine/vcs/vcs_router.go index e9684e7c04..bede98183f 100644 --- a/engine/vcs/vcs_router.go +++ b/engine/vcs/vcs_router.go @@ -49,6 +49,4 @@ func (s *Service) initRouter(ctx context.Context) { r.Handle("/vcs/{name}/repos/{owner}/{repo}/search/pullrequest", nil, r.GET(s.SearchPullRequestHandler)) r.Handle("/vcs/{name}/status", nil, r.POST(s.postStatusHandler)) - - // TOTO yesnault add route v2 to post status, and another to post a comment } From 46735ec75e307e740e351e3b83036b0c51b6af3a Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Wed, 7 Feb 2024 17:46:22 +0000 Subject: [PATCH 08/17] feat: post comment from new wf v2 Signed-off-by: Yvonnick Esnault --- engine/api/api.go | 2 +- engine/api/event_v2/event.go | 67 ++++++++++++++----- engine/api/workflow/workflow_run_event.go | 15 +---- .../api/workflow/workflow_run_event_test.go | 40 ----------- engine/api/workflow_queue.go | 2 +- sdk/repositories_manager.go | 19 +++++- sdk/repositories_manager_test.go | 43 ++++++++++++ 7 files changed, 115 insertions(+), 73 deletions(-) create mode 100644 sdk/repositories_manager_test.go diff --git a/engine/api/api.go b/engine/api/api.go index c9f4e4a958..c440375ed6 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -816,7 +816,7 @@ func (a *API) Serve(ctx context.Context) error { }) a.GoRoutines.RunWithRestart(ctx, "event_v2.dequeue", func(ctx context.Context) { - event_v2.Dequeue(ctx, a.mustDB(), a.Cache, a.GoRoutines) + event_v2.Dequeue(ctx, a.mustDB(), a.Cache, a.GoRoutines, a.Config.URL.UI) }) log.Info(ctx, "Initializing internal routines...") diff --git a/engine/api/event_v2/event.go b/engine/api/event_v2/event.go index f436b05b94..1bfe026857 100644 --- a/engine/api/event_v2/event.go +++ b/engine/api/event_v2/event.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "regexp" @@ -39,7 +40,7 @@ func publish(ctx context.Context, store cache.Store, event interface{}) { } // Dequeue runs in a goroutine and dequeue event from cache -func Dequeue(ctx context.Context, db *gorp.DbMap, store cache.Store, goroutines *sdk.GoRoutines) { +func Dequeue(ctx context.Context, db *gorp.DbMap, store cache.Store, goroutines *sdk.GoRoutines, cdsUIURL string) { for { if err := ctx.Err(); err != nil { ctx := sdk.ContextWithStacktrace(ctx, err) @@ -91,7 +92,7 @@ func Dequeue(ctx context.Context, db *gorp.DbMap, store cache.Store, goroutines wg.Add(1) goroutines.Exec(ctx, "event.workflow.notifications", func(ctx context.Context) { defer wg.Done() - if err := workflowNotifications(ctx, db, store, e); err != nil { + if err := workflowNotifications(ctx, db, store, e, cdsUIURL); err != nil { ctx := log.ContextWithStackTrace(ctx, err) log.Error(ctx, "EventV2.workflowNotifications: %v", err) } @@ -118,7 +119,7 @@ func pushToWebsockets(ctx context.Context, store cache.Store, event sdk.FullEven } } -func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Store, event sdk.FullEventV2) error { +func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Store, event sdk.FullEventV2, cdsUIURL string) error { // EventRunEnded if event.Type != sdk.EventRunEnded || event.ProjectKey == "" { return nil @@ -131,16 +132,22 @@ func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Stor } // always send build status on workflow End - vcsClient, err := repositoriesmanager.AuthorizedClient(ctx, db, store, event.ProjectKey, event.VCSName) if err != nil { return sdk.WrapError(err, "can't get AuthorizedClient for %v/%v", event.ProjectKey, event.VCSName) } - // TODO yesnault - vcsClient.SetStatus(ctx, sdk.VCSBuildStatus{}) - - // TODO send build status + buildStatus := sdk.VCSBuildStatus{ + Description: run.WorkflowName + ":" + run.Status, + URLCDS: fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", cdsUIURL, event.ProjectKey, run.WorkflowName, event.RunNumber), + Context: fmt.Sprintf("%s-%s", event.ProjectKey, run.WorkflowName), + Status: event.Status, + RepositoryFullname: event.Repository, + GitHash: run.Contexts.Git.Ref, + } + if err := vcsClient.SetStatus(ctx, buildStatus); err != nil { + return sdk.WrapError(err, "can't send the build status for %v/%v", event.ProjectKey, event.VCSName) + } if run.WorkflowData.Workflow.On == nil { return nil @@ -148,25 +155,55 @@ func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Stor if run.WorkflowData.Workflow.On.PullRequest != nil { if run.WorkflowData.Workflow.On.PullRequest.Comment != "" { - // TODO yesnault send comment - - // if err := e.vcsClient.PullRequestComment(ctx, app.RepositoryFullname, reqComment); err != nil { - // log.ErrorWithStackTrace(ctx, err) - // return err - // } + err := sendVCSPullRequestComment(ctx, vcsClient, run, run.WorkflowData.Workflow.On.PullRequest.Comment) + if err != nil { + return sdk.WrapError(err, "can't send the pull-request comment for %v/%v", event.ProjectKey, event.VCSName) + } } } if run.WorkflowData.Workflow.On.PullRequestComment != nil { if run.WorkflowData.Workflow.On.PullRequestComment.Comment != "" { - // TODO yesnault send comment + err := sendVCSPullRequestComment(ctx, vcsClient, run, run.WorkflowData.Workflow.On.PullRequestComment.Comment) + if err != nil { + return sdk.WrapError(err, "can't send the pull-request comment for %v/%v", event.ProjectKey, event.VCSName) + } } } return nil } +func sendVCSPullRequestComment(ctx context.Context, vcsClient sdk.VCSAuthorizedClientService, run sdk.V2WorkflowRun, comment string) error { + prComment := sdk.VCSPullRequestCommentRequest{ + Revision: run.Contexts.Git.Ref, + Message: comment, + } + + //Check if this branch and this commit is a pullrequest + prs, err := vcsClient.PullRequests(ctx, run.Contexts.Git.Repository) + if err != nil { + log.ErrorWithStackTrace(ctx, err) + return err + } + + // Send comment on pull request + for _, pr := range prs { + if pr.Head.Branch.DisplayID == run.Contexts.Git.RefName && sdk.VCSIsSameCommit(pr.Head.Branch.LatestCommit, run.Contexts.Git.Ref) && !pr.Merged && !pr.Closed { + prComment.ID = pr.ID + log.Info(ctx, "send comment (revision: %v pr: %v) on repo %s", prComment.Revision, prComment.ID, run.Contexts.Git.Repository) + if err := vcsClient.PullRequestComment(ctx, run.Contexts.Git.Repository, prComment); err != nil { + return err + } + break + } else { + log.Info(ctx, "nothing to do on pr %+v for ref %s", pr, run.Contexts.Git.Ref) + } + } + return nil +} + func pushNotifications(ctx context.Context, db *gorp.DbMap, event sdk.FullEventV2) error { if event.ProjectKey == "" { return nil diff --git a/engine/api/workflow/workflow_run_event.go b/engine/api/workflow/workflow_run_event.go index 01f418d555..8d113c5d0d 100644 --- a/engine/api/workflow/workflow_run_event.go +++ b/engine/api/workflow/workflow_run_event.go @@ -396,7 +396,7 @@ func (e *VCSEventMessenger) sendVCSPullRequestComment(ctx context.Context, db go //Send comment on pull request for _, pr := range prs { - if pr.Head.Branch.DisplayID == nodeRun.VCSBranch && IsSameCommit(pr.Head.Branch.LatestCommit, nodeRun.VCSHash) && !pr.Merged && !pr.Closed { + if pr.Head.Branch.DisplayID == nodeRun.VCSBranch && sdk.VCSIsSameCommit(pr.Head.Branch.LatestCommit, nodeRun.VCSHash) && !pr.Merged && !pr.Closed { reqComment.ID = pr.ID log.Info(ctx, "send comment (revision: %v pr: %v) on repo %s", reqComment.Revision, reqComment.ID, app.RepositoryFullname) if err := e.vcsClient.PullRequestComment(ctx, app.RepositoryFullname, reqComment); err != nil { @@ -411,16 +411,3 @@ func (e *VCSEventMessenger) sendVCSPullRequestComment(ctx context.Context, db go } return nil } - -func IsSameCommit(sha1, sha1b string) bool { - if len(sha1) == len(sha1b) { - return sha1 == sha1b - } - if len(sha1) == 12 && len(sha1b) >= 12 { - return sha1 == sha1b[0:len(sha1)] - } - if len(sha1b) == 12 && len(sha1) >= 12 { - return sha1b == sha1[0:len(sha1b)] - } - return false -} diff --git a/engine/api/workflow/workflow_run_event_test.go b/engine/api/workflow/workflow_run_event_test.go index 222e280240..a1ad71a6f2 100644 --- a/engine/api/workflow/workflow_run_event_test.go +++ b/engine/api/workflow/workflow_run_event_test.go @@ -557,43 +557,3 @@ func TestResyncCommitStatusCommitCache(t *testing.T) { err := e.SendVCSEvent(ctx, db.DbMap, cache, *proj, *wr, wr.WorkflowNodeRuns[1][0], "") assert.NoError(t, err) } - -func Test_isSameCommit(t *testing.T) { - type args struct { - sha1 string - sha1b string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "same", - args: args{sha1: "aaaaaa", sha1b: "aaaaaa"}, - want: true, - }, - { - name: "same", - args: args{sha1: "4e269fccb82a", sha1b: "4e269fccb82a1b98a510b172b2c8db8ec9b4abb0"}, - want: true, - }, - { - name: "same", - args: args{sha1: "4e269fccb82a1b98a510b172b2c8db8ec9b4abb0", sha1b: "4e269fccb82a"}, - want: true, - }, - { - name: "not same", - args: args{sha1: "aa4e269fccb82a1b98a510b172b2c8db8ec9b4abb0", sha1b: "aa4e269fccb82a"}, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := workflow.IsSameCommit(tt.args.sha1, tt.args.sha1b); got != tt.want { - t.Errorf("isSameCommit() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 8810fa65cd..b29756d77d 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -900,7 +900,7 @@ func getSinceUntilLimitHeader(ctx context.Context, w http.ResponseWriter, r *htt } func (api *API) postWorkflowJobTestsResultsHandler() service.Handler { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { + return func(ctx context.Context, _ http.ResponseWriter, r *http.Request) error { if isWorker := isWorker(ctx); !isWorker { return sdk.WithStack(sdk.ErrForbidden) } diff --git a/sdk/repositories_manager.go b/sdk/repositories_manager.go index a6782ad1ad..56948b407c 100644 --- a/sdk/repositories_manager.go +++ b/sdk/repositories_manager.go @@ -137,8 +137,10 @@ func (s VCSPullRequestState) IsValid() bool { } type VCSPullRequestCommentRequest struct { - VCSPullRequest - Message string `json:"message"` + ID int `json:"id"` + ChangeID string `json:"change_id"` // gerrit only + Revision string `json:"revision"` + Message string `json:"message"` } // VCSPushEvent represents a push events for polling @@ -188,3 +190,16 @@ type VCSCommitStatus struct { State string `json:"state"` Decription string `json:"description"` } + +func VCSIsSameCommit(sha1, sha1b string) bool { + if len(sha1) == len(sha1b) { + return sha1 == sha1b + } + if len(sha1) == 12 && len(sha1b) >= 12 { + return sha1 == sha1b[0:len(sha1)] + } + if len(sha1b) == 12 && len(sha1) >= 12 { + return sha1b == sha1[0:len(sha1b)] + } + return false +} diff --git a/sdk/repositories_manager_test.go b/sdk/repositories_manager_test.go new file mode 100644 index 0000000000..41121acaf7 --- /dev/null +++ b/sdk/repositories_manager_test.go @@ -0,0 +1,43 @@ +package sdk + +import "testing" + +func Test_isSameCommit(t *testing.T) { + type args struct { + sha1 string + sha1b string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "same", + args: args{sha1: "aaaaaa", sha1b: "aaaaaa"}, + want: true, + }, + { + name: "same", + args: args{sha1: "4e269fccb82a", sha1b: "4e269fccb82a1b98a510b172b2c8db8ec9b4abb0"}, + want: true, + }, + { + name: "same", + args: args{sha1: "4e269fccb82a1b98a510b172b2c8db8ec9b4abb0", sha1b: "4e269fccb82a"}, + want: true, + }, + { + name: "not same", + args: args{sha1: "aa4e269fccb82a1b98a510b172b2c8db8ec9b4abb0", sha1b: "aa4e269fccb82a"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := VCSIsSameCommit(tt.args.sha1, tt.args.sha1b); got != tt.want { + t.Errorf("isSameCommit() = %v, want %v", got, tt.want) + } + }) + } +} From ddf41765234ee0fd4e92a61af8315b0443ade429 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 9 Feb 2024 14:33:59 +0000 Subject: [PATCH 09/17] test: integration test Signed-off-by: Yvonnick Esnault --- engine/hooks/hook_repository_event.go | 18 +- engine/hooks/type_gitea.go | 347 ++++++++++++++++++++++-- engine/vcs/gitea/client_status.go | 61 ++++- sdk/hooks_repository_event.go | 4 +- sdk/log/formatter.go | 13 +- sdk/log/log.go | 2 + sdk/v2_workflow.go | 4 +- tests/08_v2_workflow_on_pullrequest.yml | 132 +++++++++ tests/lib/v2_add_git_repo.yml | 6 +- tests/lib/v2_create_project.yml | 45 ++- 10 files changed, 597 insertions(+), 35 deletions(-) create mode 100644 tests/08_v2_workflow_on_pullrequest.yml diff --git a/engine/hooks/hook_repository_event.go b/engine/hooks/hook_repository_event.go index 029fd63694..5ff24939d8 100644 --- a/engine/hooks/hook_repository_event.go +++ b/engine/hooks/hook_repository_event.go @@ -31,9 +31,6 @@ func (s *Service) extractDataFromGiteaRequest(headers http.Header, body []byte, } repoName := request.Repository.FullName - extractedData.Ref = request.Ref - - extractedData.Commit = request.After // https://github.com/go-gitea/gitea/blob/main/modules/webhook/type.go // https://github.com/go-gitea/gitea/blob/main/services/webhook/deliver.go#L128 @@ -42,12 +39,19 @@ func (s *Service) extractDataFromGiteaRequest(headers http.Header, body []byte, case "push": extractedData.CDSEventName = sdk.WorkflowHookEventPush extractedData.CDSEventType = "" // nothing here + extractedData.Ref = request.Ref + extractedData.Commit = request.After case "pull_request": - extractedData.CDSEventName = sdk.WorkflowHookEventPullRequest - extractedData.CDSEventType = "" // nothing here + extractedData.Ref = "refs/heads/" + request.PullRequest.Base.Ref + extractedData.Commit = request.PullRequest.Base.Sha + switch request.Action { + case "opened": + extractedData.CDSEventName = sdk.WorkflowHookEventPullRequest + extractedData.CDSEventType = sdk.WorkflowHookEventPullRequestTypeOpened + } case "pull_request_comment": - extractedData.CDSEventName = sdk.WorkflowHookEventPullRequestComment - extractedData.CDSEventType = "" // nothing here + // Not managed. Should needs to get the pull-request detail to get the ref / sha from the pull-request + // with a comment event, gitea does not send these details } return repoName, extractedData, nil diff --git a/engine/hooks/type_gitea.go b/engine/hooks/type_gitea.go index 10b495d6bf..9600bbf502 100644 --- a/engine/hooks/type_gitea.go +++ b/engine/hooks/type_gitea.go @@ -3,6 +3,7 @@ package hooks import "time" type GiteaEventPayload struct { + Action string `json:"action"` Secret string `json:"secret"` Ref string `json:"ref"` // refs/heads/branch_name Before string `json:"before"` // commit before @@ -24,32 +25,331 @@ type GiteaEventPayload struct { } `json:"committer"` Timestamp time.Time `json:"timestamp"` } `json:"commits"` + PullRequest struct { + ID int `json:"id"` + URL string `json:"url"` + Number int `json:"number"` + User struct { + ID int `json:"id"` + Login string `json:"login"` + LoginName string `json:"login_name"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin time.Time `json:"last_login"` + Created time.Time `json:"created"` + Restricted bool `json:"restricted"` + Active bool `json:"active"` + ProhibitLogin bool `json:"prohibit_login"` + Location string `json:"location"` + Website string `json:"website"` + Description string `json:"description"` + Visibility string `json:"visibility"` + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + StarredReposCount int `json:"starred_repos_count"` + Username string `json:"username"` + } `json:"user"` + Title string `json:"title"` + Body string `json:"body"` + Labels []any `json:"labels"` + Milestone any `json:"milestone"` + Assignee any `json:"assignee"` + Assignees any `json:"assignees"` + RequestedReviewers any `json:"requested_reviewers"` + State string `json:"state"` + IsLocked bool `json:"is_locked"` + Comments int `json:"comments"` + HTMLURL string `json:"html_url"` + DiffURL string `json:"diff_url"` + PatchURL string `json:"patch_url"` + Mergeable bool `json:"mergeable"` + Merged bool `json:"merged"` + MergedAt any `json:"merged_at"` + MergeCommitSha any `json:"merge_commit_sha"` + MergedBy any `json:"merged_by"` + AllowMaintainerEdit bool `json:"allow_maintainer_edit"` + Base struct { + Label string `json:"label"` + Ref string `json:"ref"` + Sha string `json:"sha"` + RepoID int `json:"repo_id"` + Repo struct { + ID int `json:"id"` + Owner struct { + ID int `json:"id"` + Login string `json:"login"` + LoginName string `json:"login_name"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin time.Time `json:"last_login"` + Created time.Time `json:"created"` + Restricted bool `json:"restricted"` + Active bool `json:"active"` + ProhibitLogin bool `json:"prohibit_login"` + Location string `json:"location"` + Website string `json:"website"` + Description string `json:"description"` + Visibility string `json:"visibility"` + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + StarredReposCount int `json:"starred_repos_count"` + Username string `json:"username"` + } `json:"owner"` + Name string `json:"name"` + FullName string `json:"full_name"` + Description string `json:"description"` + Empty bool `json:"empty"` + Private bool `json:"private"` + Fork bool `json:"fork"` + Template bool `json:"template"` + Parent any `json:"parent"` + Mirror bool `json:"mirror"` + Size int `json:"size"` + Language string `json:"language"` + LanguagesURL string `json:"languages_url"` + HTMLURL string `json:"html_url"` + URL string `json:"url"` + Link string `json:"link"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + OriginalURL string `json:"original_url"` + Website string `json:"website"` + StarsCount int `json:"stars_count"` + ForksCount int `json:"forks_count"` + WatchersCount int `json:"watchers_count"` + OpenIssuesCount int `json:"open_issues_count"` + OpenPrCounter int `json:"open_pr_counter"` + ReleaseCounter int `json:"release_counter"` + DefaultBranch string `json:"default_branch"` + Archived bool `json:"archived"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ArchivedAt time.Time `json:"archived_at"` + Permissions struct { + Admin bool `json:"admin"` + Push bool `json:"push"` + Pull bool `json:"pull"` + } `json:"permissions"` + HasIssues bool `json:"has_issues"` + InternalTracker struct { + EnableTimeTracker bool `json:"enable_time_tracker"` + AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` + EnableIssueDependencies bool `json:"enable_issue_dependencies"` + } `json:"internal_tracker"` + HasWiki bool `json:"has_wiki"` + HasPullRequests bool `json:"has_pull_requests"` + HasProjects bool `json:"has_projects"` + HasReleases bool `json:"has_releases"` + HasPackages bool `json:"has_packages"` + HasActions bool `json:"has_actions"` + IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` + AllowMergeCommits bool `json:"allow_merge_commits"` + AllowRebase bool `json:"allow_rebase"` + AllowRebaseExplicit bool `json:"allow_rebase_explicit"` + AllowSquashMerge bool `json:"allow_squash_merge"` + AllowRebaseUpdate bool `json:"allow_rebase_update"` + DefaultDeleteBranchAfterMerge bool `json:"default_delete_branch_after_merge"` + DefaultMergeStyle string `json:"default_merge_style"` + DefaultAllowMaintainerEdit bool `json:"default_allow_maintainer_edit"` + AvatarURL string `json:"avatar_url"` + Internal bool `json:"internal"` + MirrorInterval string `json:"mirror_interval"` + MirrorUpdated time.Time `json:"mirror_updated"` + RepoTransfer any `json:"repo_transfer"` + } `json:"repo"` + } `json:"base"` + Head struct { + Label string `json:"label"` + Ref string `json:"ref"` + Sha string `json:"sha"` + RepoID int `json:"repo_id"` + Repo struct { + ID int `json:"id"` + Owner struct { + ID int `json:"id"` + Login string `json:"login"` + LoginName string `json:"login_name"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin time.Time `json:"last_login"` + Created time.Time `json:"created"` + Restricted bool `json:"restricted"` + Active bool `json:"active"` + ProhibitLogin bool `json:"prohibit_login"` + Location string `json:"location"` + Website string `json:"website"` + Description string `json:"description"` + Visibility string `json:"visibility"` + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + StarredReposCount int `json:"starred_repos_count"` + Username string `json:"username"` + } `json:"owner"` + Name string `json:"name"` + FullName string `json:"full_name"` + Description string `json:"description"` + Empty bool `json:"empty"` + Private bool `json:"private"` + Fork bool `json:"fork"` + Template bool `json:"template"` + Parent any `json:"parent"` + Mirror bool `json:"mirror"` + Size int `json:"size"` + Language string `json:"language"` + LanguagesURL string `json:"languages_url"` + HTMLURL string `json:"html_url"` + URL string `json:"url"` + Link string `json:"link"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + OriginalURL string `json:"original_url"` + Website string `json:"website"` + StarsCount int `json:"stars_count"` + ForksCount int `json:"forks_count"` + WatchersCount int `json:"watchers_count"` + OpenIssuesCount int `json:"open_issues_count"` + OpenPrCounter int `json:"open_pr_counter"` + ReleaseCounter int `json:"release_counter"` + DefaultBranch string `json:"default_branch"` + Archived bool `json:"archived"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ArchivedAt time.Time `json:"archived_at"` + Permissions struct { + Admin bool `json:"admin"` + Push bool `json:"push"` + Pull bool `json:"pull"` + } `json:"permissions"` + HasIssues bool `json:"has_issues"` + InternalTracker struct { + EnableTimeTracker bool `json:"enable_time_tracker"` + AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` + EnableIssueDependencies bool `json:"enable_issue_dependencies"` + } `json:"internal_tracker"` + HasWiki bool `json:"has_wiki"` + HasPullRequests bool `json:"has_pull_requests"` + HasProjects bool `json:"has_projects"` + HasReleases bool `json:"has_releases"` + HasPackages bool `json:"has_packages"` + HasActions bool `json:"has_actions"` + IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` + AllowMergeCommits bool `json:"allow_merge_commits"` + AllowRebase bool `json:"allow_rebase"` + AllowRebaseExplicit bool `json:"allow_rebase_explicit"` + AllowSquashMerge bool `json:"allow_squash_merge"` + AllowRebaseUpdate bool `json:"allow_rebase_update"` + DefaultDeleteBranchAfterMerge bool `json:"default_delete_branch_after_merge"` + DefaultMergeStyle string `json:"default_merge_style"` + DefaultAllowMaintainerEdit bool `json:"default_allow_maintainer_edit"` + AvatarURL string `json:"avatar_url"` + Internal bool `json:"internal"` + MirrorInterval string `json:"mirror_interval"` + MirrorUpdated time.Time `json:"mirror_updated"` + RepoTransfer any `json:"repo_transfer"` + } `json:"repo"` + } `json:"head"` + MergeBase string `json:"merge_base"` + DueDate any `json:"due_date"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt any `json:"closed_at"` + PinOrder int `json:"pin_order"` + } `json:"pull_request"` Repository struct { - Id int `json:"id"` + ID int `json:"id"` Owner struct { - Id int `json:"id"` - Login string `json:"login"` - FullName string `json:"full_name"` - Email string `json:"email"` - AvatarUrl string `json:"avatar_url"` - Username string `json:"username"` + ID int `json:"id"` + Login string `json:"login"` + LoginName string `json:"login_name"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin time.Time `json:"last_login"` + Created time.Time `json:"created"` + Restricted bool `json:"restricted"` + Active bool `json:"active"` + ProhibitLogin bool `json:"prohibit_login"` + Location string `json:"location"` + Website string `json:"website"` + Description string `json:"description"` + Visibility string `json:"visibility"` + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + StarredReposCount int `json:"starred_repos_count"` + Username string `json:"username"` } `json:"owner"` Name string `json:"name"` FullName string `json:"full_name"` Description string `json:"description"` + Empty bool `json:"empty"` Private bool `json:"private"` Fork bool `json:"fork"` - HtmlUrl string `json:"html_url"` - SshUrl string `json:"ssh_url"` - CloneUrl string `json:"clone_url"` + Template bool `json:"template"` + Parent any `json:"parent"` + Mirror bool `json:"mirror"` + Size int `json:"size"` + Language string `json:"language"` + LanguagesURL string `json:"languages_url"` + HTMLURL string `json:"html_url"` + URL string `json:"url"` + Link string `json:"link"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + OriginalURL string `json:"original_url"` Website string `json:"website"` StarsCount int `json:"stars_count"` ForksCount int `json:"forks_count"` WatchersCount int `json:"watchers_count"` OpenIssuesCount int `json:"open_issues_count"` + OpenPrCounter int `json:"open_pr_counter"` + ReleaseCounter int `json:"release_counter"` DefaultBranch string `json:"default_branch"` + Archived bool `json:"archived"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` + ArchivedAt time.Time `json:"archived_at"` + Permissions struct { + Admin bool `json:"admin"` + Push bool `json:"push"` + Pull bool `json:"pull"` + } `json:"permissions"` + HasIssues bool `json:"has_issues"` + InternalTracker struct { + EnableTimeTracker bool `json:"enable_time_tracker"` + AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` + EnableIssueDependencies bool `json:"enable_issue_dependencies"` + } `json:"internal_tracker"` + HasWiki bool `json:"has_wiki"` + HasPullRequests bool `json:"has_pull_requests"` + HasProjects bool `json:"has_projects"` + HasReleases bool `json:"has_releases"` + HasPackages bool `json:"has_packages"` + HasActions bool `json:"has_actions"` + IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` + AllowMergeCommits bool `json:"allow_merge_commits"` + AllowRebase bool `json:"allow_rebase"` + AllowRebaseExplicit bool `json:"allow_rebase_explicit"` + AllowSquashMerge bool `json:"allow_squash_merge"` + AllowRebaseUpdate bool `json:"allow_rebase_update"` + DefaultDeleteBranchAfterMerge bool `json:"default_delete_branch_after_merge"` + DefaultMergeStyle string `json:"default_merge_style"` + DefaultAllowMaintainerEdit bool `json:"default_allow_maintainer_edit"` + AvatarURL string `json:"avatar_url"` + Internal bool `json:"internal"` + MirrorInterval string `json:"mirror_interval"` + MirrorUpdated time.Time `json:"mirror_updated"` + RepoTransfer any `json:"repo_transfer"` } `json:"repository"` Pusher struct { Id int `json:"id"` @@ -60,11 +360,26 @@ type GiteaEventPayload struct { Username string `json:"username"` } `json:"pusher"` Sender struct { - Id int `json:"id"` - Login string `json:"login"` - FullName string `json:"full_name"` - Email string `json:"email"` - AvatarUrl string `json:"avatar_url"` - Username string `json:"username"` + ID int `json:"id"` + Login string `json:"login"` + LoginName string `json:"login_name"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin time.Time `json:"last_login"` + Created time.Time `json:"created"` + Restricted bool `json:"restricted"` + Active bool `json:"active"` + ProhibitLogin bool `json:"prohibit_login"` + Location string `json:"location"` + Website string `json:"website"` + Description string `json:"description"` + Visibility string `json:"visibility"` + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + StarredReposCount int `json:"starred_repos_count"` + Username string `json:"username"` } `json:"sender"` } diff --git a/engine/vcs/gitea/client_status.go b/engine/vcs/gitea/client_status.go index 1c83770cdd..dd31f6f469 100644 --- a/engine/vcs/gitea/client_status.go +++ b/engine/vcs/gitea/client_status.go @@ -2,12 +2,71 @@ package gitea import ( "context" + "encoding/json" + "fmt" + "strings" + "code.gitea.io/sdk/gitea" "github.com/ovh/cds/sdk" + "github.com/rockbears/log" ) func (client *giteaClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildStatus) error { - return sdk.WithStack(sdk.ErrNotImplemented) + + // POST /repos/{owner}/{repo}/statuses/{sha} + // { + // "context": "string", + // "description": "string", + // "state": "string", + // "target_url": "string" + // } + + if buildStatus.Status == "" { + log.Debug(ctx, "gitea.SetStatus> Do not process event for empty status") + return nil + } + + giteaStatus := gitea.CreateStatusOption{ + Description: buildStatus.Description, + Context: buildStatus.Context, + TargetURL: buildStatus.URLCDS, + } + + switch buildStatus.Status { + case sdk.StatusChecking, sdk.StatusPending, sdk.StatusBuilding: + giteaStatus.State = gitea.StatusPending + case sdk.StatusSuccess: + giteaStatus.State = gitea.StatusSuccess + case sdk.StatusFail: + giteaStatus.State = gitea.StatusFailure + } + + path := fmt.Sprintf("/repos/%s/statuses/%s", buildStatus.RepositoryFullname, buildStatus.GitHash) + + b, err := json.Marshal(giteaStatus) + if err != nil { + return sdk.WrapError(err, "Unable to marshal gitea status") + } + + log.Debug(ctx, "SetStatus> gitea post on %v body:%v", path, string(b)) + + t := strings.Split(buildStatus.RepositoryFullname, "/") + if len(t) != 2 { + return sdk.WrapError(err, "invalid gitRepositoryFullname gitea: %s", buildStatus.RepositoryFullname) + } + s, resp, err := client.client.CreateStatus(t[0], t[1], buildStatus.GitHash, giteaStatus) + if err != nil { + return sdk.WrapError(err, "Unable to post gitea status") + } + + log.Debug(ctx, "SetStatus> gitea response for %v status: %d:", path, resp.StatusCode) + + if resp.StatusCode != 201 { + return sdk.WrapError(err, "Unable to create status on gitea. Status code : %d - Body: %s - context:%s", resp.StatusCode, resp.Body, buildStatus.Context) + } + + log.Debug(ctx, "SetStatus> Status %d %s created at %v", s.ID, s.URL, s.Created) + return nil } func (client *giteaClient) ListStatuses(ctx context.Context, repo string, ref string) ([]sdk.VCSCommitStatus, error) { diff --git a/sdk/hooks_repository_event.go b/sdk/hooks_repository_event.go index c10521cc0a..cdd500dc2d 100644 --- a/sdk/hooks_repository_event.go +++ b/sdk/hooks_repository_event.go @@ -19,13 +19,13 @@ const ( WorkflowHookEventModelUpdate = "model_update" WorkflowHookEventPush = "push" - WorkflowHookEventPullRequest = "pull_request" + WorkflowHookEventPullRequest = "pull-request" WorkflowHookEventPullRequestTypeOpened = "opened" WorkflowHookEventPullRequestTypeReopened = "reopened" WorkflowHookEventPullRequestTypeClosed = "closed" WorkflowHookEventPullRequestTypeEdited = "edited" - WorkflowHookEventPullRequestComment = "pull_request_comment" + WorkflowHookEventPullRequestComment = "pull-request-comment" WorkflowHookEventPullRequestCommentTypeCreated = "created" WorkflowHookEventPullRequestCommentTypeDeleted = "deleted" WorkflowHookEventPullRequestCommentTypeEdited = "edited" diff --git a/sdk/log/formatter.go b/sdk/log/formatter.go index 8f475df6cf..3215fcdc50 100644 --- a/sdk/log/formatter.go +++ b/sdk/log/formatter.go @@ -11,7 +11,9 @@ import ( ) // CDSFormatter ... -type CDSFormatter struct{} +type CDSFormatter struct { + DisabledPrintFields bool +} // Format format a log func (f *CDSFormatter) Format(entry *logrus.Entry) ([]byte, error) { @@ -53,10 +55,13 @@ func (f *CDSFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry, keys [ levelText = "[" + levelText + "]" fmt.Fprintf(b, "%s %s%+5s%s %s", entry.Time.Format("2006-01-02 15:04:05"), levelColor, levelText, ansi.Reset, entry.Message) - for _, k := range keys { - v := entry.Data[k] - fmt.Fprintf(b, " %s%s%s=%+v", levelColor, k, ansi.Reset, v) + if !f.DisabledPrintFields { + for _, k := range keys { + v := entry.Data[k] + fmt.Fprintf(b, " %s%s%s=%+v", levelColor, k, ansi.Reset, v) + } } + } func needsQuoting(text string) bool { diff --git a/sdk/log/log.go b/sdk/log/log.go index b0b9ad9a0a..4b210acce1 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -62,6 +62,8 @@ func Initialize(ctx context.Context, conf *Conf) { logrus.SetOutput(io.Discard) case "json": logrus.SetFormatter(&logrus.JSONFormatter{}) + case "readable": + logrus.SetFormatter(&CDSFormatter{DisabledPrintFields: true}) default: logrus.SetFormatter(&CDSFormatter{}) } diff --git a/sdk/v2_workflow.go b/sdk/v2_workflow.go index fa5f4e9c53..00d8c3a86d 100644 --- a/sdk/v2_workflow.go +++ b/sdk/v2_workflow.go @@ -30,8 +30,8 @@ type V2Workflow struct { type WorkflowOn struct { Push *WorkflowOnPush `json:"push,omitempty"` - PullRequest *WorkflowOnPullRequest `json:"pull_request,omitempty"` - PullRequestComment *WorkflowOnPullRequestComment `json:"pull_request_comment,omitempty"` + PullRequest *WorkflowOnPullRequest `json:"pull-request,omitempty"` + PullRequestComment *WorkflowOnPullRequestComment `json:"pull-request-comment,omitempty"` ModelUpdate *WorkflowOnModelUpdate `json:"model_update,omitempty"` WorkflowUpdate *WorkflowOnWorkflowUpdate `json:"workflow_update,omitempty"` } diff --git a/tests/08_v2_workflow_on_pullrequest.yml b/tests/08_v2_workflow_on_pullrequest.yml new file mode 100644 index 0000000000..b8093449b4 --- /dev/null +++ b/tests/08_v2_workflow_on_pullrequest.yml @@ -0,0 +1,132 @@ +name: Workflow notifications +vars: + cds_project: "ITV2WFNOTIF" + git_repo: "it_v2_workflow_notifications" + cds_workflow: "WorkflowNotificationsLog" +testcases: +- name: Prepare test + steps: + - name: "Verify cdsctl configuration" + script: "{{.cdsctl}} -f {{.cdsctl.config}} user me --format json" + vars: + cdsUser: + from: result.systemoutjson.username + + - name: "Display username" + info: "Connected CDS user is {{.cdsUser}}" + + - type: v2_create_project + cdsctl_command: "{{.cdsctl}} -f {{.cdsctl.config}}" + cds_project: "{{.cds_project}}" + git_host: "{{.git.host}}" + git_user: "{{.git.user}}" + git_password: "{{.git.password}}" + cds_region: "{{.cds.region}}" + + - type: v2_add_git_repo + cdsctl_command: "{{.cdsctl}} -f {{.cdsctl.config}}" + cds_project: "{{.cds_project}}" + cds_hook_url: "{{.gitea.hook.url}}" + git_host: "{{.git.host}}" + git_user: "{{.git.user}}" + git_password: "{{.git.password}}" + git_repo: "{{.git_repo}}" + + - type: v2_install_gpg_key + cdsctl_command: "{{.cdsctl}} -f {{.cdsctl.config}}" + gpg_key_id: "{{.gpg.key_id}}" + git_host: "{{.git.host}}" + git_user: "{{.git.user}}" + git_password: "{{.git.password}}" + vars: + emailaddress: + from: result.emailaddress + +- name: Push workflow file + steps: + - type: v2_push_files_on_repo + git_repo: "{{.git_repo}}" + git_host: "{{.git.host}}" + git_user: "{{.git.user}}" + git_email: "{{.Prepare-test.emailaddress}}" + git_password: "{{.git.password}}" + gpg_key_id: "{{.gpg.key_id}}" + files: + worker-models/debian.yml: + name: docker-debian + osarch: linux/amd64 + type: docker + spec: + image: buildpack-deps:buster + workflows/workflow_notifications.yaml: + name: {{.cds_workflow}} + commit-status: + title: foo + description: bar + on: + pull-request: + comment: "a comment here" + types: ["opened"] + jobs: + myjob: + runs-on: "{{.cds_project}}/my_vcs_server/{{.git.user}}/{{.git_repo}}/docker-debian" + steps: + - run: |- + #!/bin/bash + set -ex + echo "job done" && date + + - name: Create a file on a new branch + script: |- + cd /tmp/repos/{{.git_repo}} + git checkout -b new-branch + echo 'FOO' > foo.md + git add --all + git commit --gpg-sign={{.gpg.key_id}} -m "add file and sign" --author "{{.git.user}} <{{.Prepare-test.emailaddress}}>" + git push origin new-branch + + - name: wait 10s + script: sleep 10 + + - name: Create the pull-request + type: http + method: POST + url: "{{.git.host}}/api/v1/repos/{{.git.user}}/{{.git_repo}}/pulls" + basic_auth_user: "{{.git.user}}" + basic_auth_password: "{{.git.password}}" + headers: + Content-Type: application/json + body: > + { + "base": "master", + "body": "test body pull-request", + "head": "new-branch", + "title": "test pull-request" + } + assertions: + - or: + - result.statuscode ShouldEqual 201 + + - name: Check CDS project analyses status + script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental project analysis list {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} --format json" + assertions: + - result.systemoutjson ShouldHaveLength 3 + - result.systemoutjson.systemoutjson0.status ShouldEqual "Skipped" + - result.systemoutjson.systemoutjson1.status ShouldEqual "Success" + - result.systemoutjson.systemoutjson2.status ShouldEqual "Success" + retry: 10 + delay: 12 + + - name: Check that the CDS workflow has at least one execution and is Success + script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental workflow history {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} {{.cds_workflow}} --format json" + assertions: + - result.systemoutjson ShouldHaveLength 1 + - result.systemoutjson.systemoutjson0.status ShouldEqual "Success" + retry: 20 + delay: 12 + + - name: Download logs + script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental workflow logs download {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} {{.cds_workflow}} 1" + + - name: Check workflowEnv log content + script: "cat {{.cds_workflow}}-*-myjob* | grep \"job done\"" diff --git a/tests/lib/v2_add_git_repo.yml b/tests/lib/v2_add_git_repo.yml index 05f7d1a3fb..62c9ae914b 100644 --- a/tests/lib/v2_add_git_repo.yml +++ b/tests/lib/v2_add_git_repo.yml @@ -70,7 +70,11 @@ steps: "secret": "{{.hookSecret}}", "content_type": "json", "url": "{{.input.cds_hook_url}}/v2/webhook/repository/gitea/my_vcs_server" - } + }, + "events": [ + "push", + "pull_request" + ] } vars: hookID: diff --git a/tests/lib/v2_create_project.yml b/tests/lib/v2_create_project.yml index b0c19dd3fc..2520aad156 100644 --- a/tests/lib/v2_create_project.yml +++ b/tests/lib/v2_create_project.yml @@ -91,6 +91,48 @@ steps: - script: "{{.input.cdsctl_command}} experimental rbac import /tmp/project_rbac.yml --force" # Put the project ssh key as SSH Key on gitea + - type: http + method: DELETE + url: "{{.input.git_host}}/api/v1/user/keys/{{.sshKeyName}}" + basic_auth_user: "{{.input.git_user}}" + basic_auth_password: "{{.input.git_password}}" + headers: + Content-Type: application/json + assertions: + - or: + - result.statuscode ShouldEqual 204 + - result.statuscode ShouldEqual 404 + + # Drop existing ssh key for user + - type: http + method: GET + url: "{{.input.git_host}}/api/v1/user/keys" + headers: + Content-Type: application/json + basic_auth_user: "{{.input.git_user}}" + basic_auth_password: "{{.input.git_password}}" + assertions: + - result.statuscode ShouldEqual 200 + vars: + keys: + from: result.bodyjson + + - script: > + echo '{{.keys}}' | jq -c 'map(select(.title | contains("{{.sshKeyName}}")))' + vars: + keys_filtered: + from: result.systemout + + - range: "{{.keys_filtered}}" + type: http + method: DELETE + url: "{{.input.git_host}}/api/v1/user/keys/{{.value.id}}" + basic_auth_user: "{{.input.git_user}}" + basic_auth_password: "{{.input.git_password}}" + assertions: + - result.statuscode ShouldEqual 204 + + # Add the project ssh key as SSH Key on gitea - type: http method: POST url: "{{.input.git_host}}/api/v1/user/keys" @@ -107,6 +149,5 @@ steps: assertions: - or: - result.statuscode ShouldEqual 201 - - result.statuscode ShouldEqual 422 -output: {} +output: {} \ No newline at end of file From 5482bdbf27feb1db1584d9c850baf70e8af900c0 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 9 Feb 2024 16:56:13 +0000 Subject: [PATCH 10/17] fix: it Signed-off-by: Yvonnick Esnault --- engine/hooks/hook_repository_event.go | 4 +-- engine/vcs/gitea/client_status.go | 22 ++++++++++++----- tests/08_v2_workflow_on_pullrequest.yml | 33 +++++++++++++++---------- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/engine/hooks/hook_repository_event.go b/engine/hooks/hook_repository_event.go index 5ff24939d8..7bf2b4e131 100644 --- a/engine/hooks/hook_repository_event.go +++ b/engine/hooks/hook_repository_event.go @@ -42,8 +42,8 @@ func (s *Service) extractDataFromGiteaRequest(headers http.Header, body []byte, extractedData.Ref = request.Ref extractedData.Commit = request.After case "pull_request": - extractedData.Ref = "refs/heads/" + request.PullRequest.Base.Ref - extractedData.Commit = request.PullRequest.Base.Sha + extractedData.Ref = "refs/heads/" + request.PullRequest.Head.Ref + extractedData.Commit = request.PullRequest.Head.Sha switch request.Action { case "opened": extractedData.CDSEventName = sdk.WorkflowHookEventPullRequest diff --git a/engine/vcs/gitea/client_status.go b/engine/vcs/gitea/client_status.go index dd31f6f469..f163a29187 100644 --- a/engine/vcs/gitea/client_status.go +++ b/engine/vcs/gitea/client_status.go @@ -27,9 +27,19 @@ func (client *giteaClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBui } giteaStatus := gitea.CreateStatusOption{ - Description: buildStatus.Description, - Context: buildStatus.Context, - TargetURL: buildStatus.URLCDS, + Context: buildStatus.Context, + TargetURL: buildStatus.URLCDS, + } + + // gitea display on the UI, the context contact with the description. + // so that, we remove the context from the description: + // description":"WorkflowNotificationsLog:Success","context":"ITV2WFNOTIF-WorkflowNotificationsLog + // we want only ITV2WFNOTIF-WorkflowNotificationsLog:Success display + td := strings.Split(buildStatus.Description, ":") + if len(td) == 2 { + giteaStatus.Description = td[1] + } else { + giteaStatus.Description = buildStatus.Description } switch buildStatus.Status { @@ -45,7 +55,7 @@ func (client *giteaClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBui b, err := json.Marshal(giteaStatus) if err != nil { - return sdk.WrapError(err, "Unable to marshal gitea status") + return sdk.WrapError(err, "unable to marshal gitea status") } log.Debug(ctx, "SetStatus> gitea post on %v body:%v", path, string(b)) @@ -56,13 +66,13 @@ func (client *giteaClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBui } s, resp, err := client.client.CreateStatus(t[0], t[1], buildStatus.GitHash, giteaStatus) if err != nil { - return sdk.WrapError(err, "Unable to post gitea status") + return sdk.WrapError(err, "unable to post gitea status") } log.Debug(ctx, "SetStatus> gitea response for %v status: %d:", path, resp.StatusCode) if resp.StatusCode != 201 { - return sdk.WrapError(err, "Unable to create status on gitea. Status code : %d - Body: %s - context:%s", resp.StatusCode, resp.Body, buildStatus.Context) + return sdk.WrapError(err, "unable to create status on gitea. Status code : %d - Body: %s - context:%s", resp.StatusCode, resp.Body, buildStatus.Context) } log.Debug(ctx, "SetStatus> Status %d %s created at %v", s.ID, s.URL, s.Created) diff --git a/tests/08_v2_workflow_on_pullrequest.yml b/tests/08_v2_workflow_on_pullrequest.yml index b8093449b4..d386617db9 100644 --- a/tests/08_v2_workflow_on_pullrequest.yml +++ b/tests/08_v2_workflow_on_pullrequest.yml @@ -74,9 +74,19 @@ testcases: - run: |- #!/bin/bash set -ex + echo "$GIT_REF" echo "job done" && date - - name: Create a file on a new branch + - name: Check CDS project analyses status after push on master + script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental project analysis list {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} --format json" + assertions: + - result.systemoutjson ShouldHaveLength 2 + - result.systemoutjson.systemoutjson0.status ShouldEqual "Skipped" + - result.systemoutjson.systemoutjson1.status ShouldEqual "Success" + retry: 10 + delay: 12 + + - name: Create a file on a new-branch branch script: |- cd /tmp/repos/{{.git_repo}} git checkout -b new-branch @@ -85,8 +95,15 @@ testcases: git commit --gpg-sign={{.gpg.key_id}} -m "add file and sign" --author "{{.git.user}} <{{.Prepare-test.emailaddress}}>" git push origin new-branch - - name: wait 10s - script: sleep 10 + - name: Check CDS project analyses status after push on new-branch + script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental project analysis list {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} --format json" + assertions: + - result.systemoutjson ShouldHaveLength 3 + - result.systemoutjson.systemoutjson0.status ShouldEqual "Skipped" + - result.systemoutjson.systemoutjson1.status ShouldEqual "Success" + - result.systemoutjson.systemoutjson2.status ShouldEqual "Success" + retry: 10 + delay: 12 - name: Create the pull-request type: http @@ -107,16 +124,6 @@ testcases: - or: - result.statuscode ShouldEqual 201 - - name: Check CDS project analyses status - script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental project analysis list {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} --format json" - assertions: - - result.systemoutjson ShouldHaveLength 3 - - result.systemoutjson.systemoutjson0.status ShouldEqual "Skipped" - - result.systemoutjson.systemoutjson1.status ShouldEqual "Success" - - result.systemoutjson.systemoutjson2.status ShouldEqual "Success" - retry: 10 - delay: 12 - - name: Check that the CDS workflow has at least one execution and is Success script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental workflow history {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} {{.cds_workflow}} --format json" assertions: From 5e023669b0f7053e067dd36a72ddc5aef07a6c41 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 9 Feb 2024 18:08:03 +0000 Subject: [PATCH 11/17] fix: implem gitea comment Signed-off-by: Yvonnick Esnault --- engine/api/event_v2/event.go | 2 +- engine/vcs/gitea/client_pull_request.go | 75 ++++++++++++++++++++++++- sdk/v2_workflow.go | 8 +-- tests/08_v2_workflow_on_pullrequest.yml | 4 ++ 4 files changed, 81 insertions(+), 8 deletions(-) diff --git a/engine/api/event_v2/event.go b/engine/api/event_v2/event.go index 1bfe026857..0a33e2a144 100644 --- a/engine/api/event_v2/event.go +++ b/engine/api/event_v2/event.go @@ -190,7 +190,7 @@ func sendVCSPullRequestComment(ctx context.Context, vcsClient sdk.VCSAuthorizedC // Send comment on pull request for _, pr := range prs { - if pr.Head.Branch.DisplayID == run.Contexts.Git.RefName && sdk.VCSIsSameCommit(pr.Head.Branch.LatestCommit, run.Contexts.Git.Ref) && !pr.Merged && !pr.Closed { + if pr.Head.Branch.DisplayID == run.Contexts.Git.RefName && sdk.VCSIsSameCommit(pr.Head.Branch.LatestCommit, run.Contexts.Git.Sha) && !pr.Merged && !pr.Closed { prComment.ID = pr.ID log.Info(ctx, "send comment (revision: %v pr: %v) on repo %s", prComment.Revision, prComment.ID, run.Contexts.Git.Repository) if err := vcsClient.PullRequestComment(ctx, run.Contexts.Git.Repository, prComment); err != nil { diff --git a/engine/vcs/gitea/client_pull_request.go b/engine/vcs/gitea/client_pull_request.go index 6f2e81d507..00896d8d2f 100644 --- a/engine/vcs/gitea/client_pull_request.go +++ b/engine/vcs/gitea/client_pull_request.go @@ -2,20 +2,89 @@ package gitea import ( "context" + "fmt" + "strconv" + "strings" + + "code.gitea.io/sdk/gitea" "github.com/ovh/cds/sdk" + "github.com/rockbears/log" ) func (g *giteaClient) PullRequest(ctx context.Context, repo string, id string) (sdk.VCSPullRequest, error) { - return sdk.VCSPullRequest{}, sdk.WithStack(sdk.ErrNotImplemented) + ret := sdk.VCSPullRequest{} + t := strings.Split(repo, "/") + if len(t) != 2 { + return ret, fmt.Errorf("invalid repo gitea: %s", repo) + } + + i, _ := strconv.ParseInt(id, 10, 64) + + pr, _, err := g.client.GetPullRequest(t[0], t[1], i) + if err != nil { + return ret, sdk.WrapError(err, "unable to get gitea pull-request repo:%v id:%v", repo, id) + } + + ret = sdk.VCSPullRequest{ + ID: int(pr.Index), + State: string(pr.State), + Head: sdk.VCSPushEvent{ + Branch: sdk.VCSBranch{ + DisplayID: pr.Head.Ref, + LatestCommit: pr.Head.Sha, + }, + }, + } + return ret, nil } func (g *giteaClient) PullRequests(ctx context.Context, repo string, opts sdk.VCSPullRequestOptions) ([]sdk.VCSPullRequest, error) { - return nil, sdk.WithStack(sdk.ErrNotImplemented) + t := strings.Split(repo, "/") + if len(t) != 2 { + return nil, fmt.Errorf("invalid repo gitea: %s", repo) + } + + prs, _, err := g.client.ListRepoPullRequests(t[0], t[1], gitea.ListPullRequestsOptions{}) + if err != nil { + return nil, sdk.WrapError(err, "unable to get gitea pull-requests from repo:%v", repo) + } + + ret := make([]sdk.VCSPullRequest, 0) + for _, pr := range prs { + ret = append(ret, sdk.VCSPullRequest{ + ID: int(pr.Index), + State: string(pr.State), + Head: sdk.VCSPushEvent{ + Branch: sdk.VCSBranch{ + DisplayID: pr.Head.Ref, + LatestCommit: pr.Head.Sha, + }, + }, + }) + } + return ret, nil } // PullRequestComment push a new comment on a pull request func (g *giteaClient) PullRequestComment(ctx context.Context, repo string, prRequest sdk.VCSPullRequestCommentRequest) error { - return sdk.WithStack(sdk.ErrNotImplemented) + t := strings.Split(repo, "/") + if len(t) != 2 { + return fmt.Errorf("invalid repo gitea: %s", repo) + } + + log.Debug(ctx, "PullRequestComment> trying post comment %s", prRequest.Message) + + opt := gitea.CreatePullReviewOptions{ + Body: prRequest.Message, + } + s, _, err := g.client.CreatePullReview(t[0], t[1], int64(prRequest.ID), opt) + if err != nil { + return sdk.WrapError(err, "unable to create pull-request comment on repo:%v id:%v", repo, prRequest.ID) + } + + log.Debug(ctx, "PullRequestComment> comment %d %s", s.ID, s.Body) + + return nil } func (g *giteaClient) PullRequestCreate(ctx context.Context, repo string, pr sdk.VCSPullRequest) (sdk.VCSPullRequest, error) { diff --git a/sdk/v2_workflow.go b/sdk/v2_workflow.go index 00d8c3a86d..bebecd10f1 100644 --- a/sdk/v2_workflow.go +++ b/sdk/v2_workflow.go @@ -44,14 +44,14 @@ type WorkflowOnPush struct { type WorkflowOnPullRequest struct { Branches []string `json:"branches,omitempty"` - Comment string `json:"pr_comment,omitempty"` + Comment string `json:"comment,omitempty"` Paths []string `json:"paths,omitempty"` Types []string `json:"types,omitempty"` } type WorkflowOnPullRequestComment struct { Branches []string `json:"branches,omitempty"` - Comment string `json:"pr_comment,omitempty"` + Comment string `json:"comment,omitempty"` Paths []string `json:"paths,omitempty"` Types []string `json:"types,omitempty"` } @@ -102,13 +102,13 @@ func IsDefaultHooks(on *WorkflowOn) []string { } if on.PullRequest != nil { hookKeys = append(hookKeys, WorkflowHookEventPullRequest) - if len(on.PullRequest.Paths) > 0 || len(on.PullRequest.Branches) > 0 { + if len(on.PullRequest.Paths) > 0 || len(on.PullRequest.Branches) > 0 || on.PullRequest.Comment != "" { return nil } } if on.PullRequestComment != nil { hookKeys = append(hookKeys, WorkflowHookEventPullRequestComment) - if len(on.PullRequest.Paths) > 0 || len(on.PullRequest.Branches) > 0 { + if len(on.PullRequestComment.Paths) > 0 || len(on.PullRequestComment.Branches) > 0 || on.PullRequestComment.Comment != "" { return nil } } diff --git a/tests/08_v2_workflow_on_pullrequest.yml b/tests/08_v2_workflow_on_pullrequest.yml index d386617db9..5ada175105 100644 --- a/tests/08_v2_workflow_on_pullrequest.yml +++ b/tests/08_v2_workflow_on_pullrequest.yml @@ -86,6 +86,8 @@ testcases: retry: 10 delay: 12 +- name: Work on another git branch + steps: - name: Create a file on a new-branch branch script: |- cd /tmp/repos/{{.git_repo}} @@ -124,6 +126,8 @@ testcases: - or: - result.statuscode ShouldEqual 201 +- name: Check if workflow triggered on the new-branch is ok after pull-request creation + steps: - name: Check that the CDS workflow has at least one execution and is Success script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental workflow history {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} {{.cds_workflow}} --format json" assertions: From f36dd5eb210a2eea2283f3851759af0e2b76956a Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Sun, 11 Feb 2024 10:03:53 +0000 Subject: [PATCH 12/17] test: it check comment Signed-off-by: Yvonnick Esnault --- tests/08_v2_workflow_on_pullrequest.yml | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/08_v2_workflow_on_pullrequest.yml b/tests/08_v2_workflow_on_pullrequest.yml index 5ada175105..e476d6f02d 100644 --- a/tests/08_v2_workflow_on_pullrequest.yml +++ b/tests/08_v2_workflow_on_pullrequest.yml @@ -107,7 +107,9 @@ testcases: retry: 10 delay: 12 - - name: Create the pull-request +- name: Create Pull-Request + steps: + - name: Create the pull-request on gitea type: http method: POST url: "{{.git.host}}/api/v1/repos/{{.git.user}}/{{.git_repo}}/pulls" @@ -123,8 +125,10 @@ testcases: "title": "test pull-request" } assertions: - - or: - result.statuscode ShouldEqual 201 + vars: + prNumber: + from: result.bodyjson.number - name: Check if workflow triggered on the new-branch is ok after pull-request creation steps: @@ -141,3 +145,25 @@ testcases: - name: Check workflowEnv log content script: "cat {{.cds_workflow}}-*-myjob* | grep \"job done\"" + +- name: Check comment created on gitea + steps: + - name: Get pull-request from gitea and check comment + type: http + method: GET + url: "{{.git.host}}/api/v1/repos/{{.git.user}}/{{.git_repo}}/pulls/{{.Create-Pull-Request.prNumber}}/reviews" + basic_auth_user: "{{.git.user}}" + basic_auth_password: "{{.git.password}}" + headers: + Content-Type: application/json + body: > + { + "base": "master", + "body": "test body pull-request", + "head": "new-branch", + "title": "test pull-request" + } + assertions: + - result.statuscode ShouldEqual 200 + - result.bodyjson ShouldHaveLength 1 + - result.bodyjson.bodyjson0.body ShouldEqual "a comment here" \ No newline at end of file From 6c50a891867e8825de4aa9c294b62be66c1226aa Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Sun, 11 Feb 2024 12:10:24 +0000 Subject: [PATCH 13/17] fix: github status Signed-off-by: Yvonnick Esnault --- engine/vcs/github/client_status.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/engine/vcs/github/client_status.go b/engine/vcs/github/client_status.go index d17cd093f8..7b8671bed0 100644 --- a/engine/vcs/github/client_status.go +++ b/engine/vcs/github/client_status.go @@ -39,6 +39,18 @@ func (g *githubClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildSt TargetURL: buildStatus.URLCDS, } + switch buildStatus.Status { + case sdk.StatusSuccess: + ghStatus.State = "success" + case sdk.StatusFail: + ghStatus.State = "failure" + case sdk.StatusBuilding: + ghStatus.State = "pending" + default: + log.Debug(ctx, "SetStatus> github setStatus not managed for %s", buildStatus.Status) + return nil + } + path := fmt.Sprintf("/repos/%s/statuses/%s", buildStatus.RepositoryFullname, buildStatus.GitHash) b, err := json.Marshal(ghStatus) From f2486d5b2178137824e06a49d4070a989b20dd69 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Sun, 11 Feb 2024 15:23:08 +0000 Subject: [PATCH 14/17] feat: gitea wf v1 Signed-off-by: Yvonnick Esnault --- engine/api/event_v2/event.go | 2 +- engine/vcs/gitea/client_commit.go | 44 +++++++++++++++++++++---- engine/vcs/gitea/client_forks.go | 31 +++++++++++++++-- engine/vcs/gitea/client_status.go | 35 ++++++++++++++++++-- engine/vcs/gitea/client_tags.go | 38 +++++++++++++++++++-- engine/vcs/gitlab/client_status.go | 3 ++ tests/08_v2_workflow_on_pullrequest.yml | 30 ++++++++++++----- 7 files changed, 160 insertions(+), 23 deletions(-) diff --git a/engine/api/event_v2/event.go b/engine/api/event_v2/event.go index 0a33e2a144..f12b32d3ee 100644 --- a/engine/api/event_v2/event.go +++ b/engine/api/event_v2/event.go @@ -143,7 +143,7 @@ func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Stor Context: fmt.Sprintf("%s-%s", event.ProjectKey, run.WorkflowName), Status: event.Status, RepositoryFullname: event.Repository, - GitHash: run.Contexts.Git.Ref, + GitHash: run.Contexts.Git.Sha, } if err := vcsClient.SetStatus(ctx, buildStatus); err != nil { return sdk.WrapError(err, "can't send the build status for %v/%v", event.ProjectKey, event.VCSName) diff --git a/engine/vcs/gitea/client_commit.go b/engine/vcs/gitea/client_commit.go index eb94ade53d..9e21fe6c43 100644 --- a/engine/vcs/gitea/client_commit.go +++ b/engine/vcs/gitea/client_commit.go @@ -2,15 +2,48 @@ package gitea import ( "context" + "fmt" "strconv" + "strings" - gg "code.gitea.io/sdk/gitea" - + "code.gitea.io/sdk/gitea" "github.com/ovh/cds/sdk" ) -func (g *giteaClient) Commits(_ context.Context, repo, branch, since, until string) ([]sdk.VCSCommit, error) { - return nil, sdk.WithStack(sdk.ErrNotImplemented) +func (g *giteaClient) Commits(_ context.Context, fullname, branch, since, until string) ([]sdk.VCSCommit, error) { + t := strings.Split(fullname, "/") + if len(t) != 2 { + return nil, fmt.Errorf("giteaClient.Commits> invalid fullname %s", fullname) + } + + commits, _, err := g.client.ListRepoCommits(t[0], t[1], gitea.ListCommitOptions{}) + //allCommitBetween(ctx, repo, since, until, theBranch) + if err != nil { + return nil, sdk.WrapError(err, "cannot load commits on branch %s", branch) + } + + commitsResult := make([]sdk.VCSCommit, 0, len(commits)) + //Convert to sdk.VCSCommit + for _, c := range commits { + email := c.Author.Email + commit := sdk.VCSCommit{ + Timestamp: c.Created.Unix() * 1000, + Message: c.RepoCommit.Message, + Hash: c.RepoCommit.Tree.SHA, + URL: c.RepoCommit.URL, + Author: sdk.VCSAuthor{ + DisplayName: c.Author.FullName, + Email: email, + Name: c.Author.UserName, + Avatar: c.Author.AvatarURL, + ID: fmt.Sprintf("%d", c.Author.ID), + }, + } + + commitsResult = append(commitsResult, commit) + } + + return commitsResult, nil } func (g *giteaClient) Commit(_ context.Context, repo, hash string) (sdk.VCSCommit, error) { @@ -32,8 +65,7 @@ func (g *giteaClient) CommitsBetweenRefs(ctx context.Context, repo, base, head s return nil, sdk.WithStack(sdk.ErrNotImplemented) } -func (g *giteaClient) toVCSCommit(commit *gg.Commit) sdk.VCSCommit { - +func (g *giteaClient) toVCSCommit(commit *gitea.Commit) sdk.VCSCommit { vcsCommit := sdk.VCSCommit{ Signature: "", Verified: false, diff --git a/engine/vcs/gitea/client_forks.go b/engine/vcs/gitea/client_forks.go index 53f3ec3554..016b8260f4 100644 --- a/engine/vcs/gitea/client_forks.go +++ b/engine/vcs/gitea/client_forks.go @@ -2,9 +2,36 @@ package gitea import ( "context" + "fmt" + "strings" + + "code.gitea.io/sdk/gitea" "github.com/ovh/cds/sdk" ) -func (g *giteaClient) ListForks(ctx context.Context, repo string) ([]sdk.VCSRepo, error) { - return nil, sdk.WithStack(sdk.ErrNotImplemented) +func (c *giteaClient) ListForks(ctx context.Context, fullname string) ([]sdk.VCSRepo, error) { + t := strings.Split(fullname, "/") + if len(t) != 2 { + return nil, fmt.Errorf("giteaCliet.Tags> invalid fullname %s", fullname) + } + repos, _, err := c.client.ListForks(t[0], t[1], gitea.ListForksOptions{}) + if err != nil { + return nil, err + } + + var ret []sdk.VCSRepo + for _, repo := range repos { + ret = append(ret, sdk.VCSRepo{ + ID: fmt.Sprintf("%d", repo.ID), + Name: repo.Name, + Slug: strings.Split(repo.FullName, "/")[0], + Fullname: repo.FullName, + URL: repo.HTMLURL, + HTTPCloneURL: repo.CloneURL, + SSHCloneURL: repo.SSHURL, + }) + } + + return ret, nil + } diff --git a/engine/vcs/gitea/client_status.go b/engine/vcs/gitea/client_status.go index f163a29187..d691fcd2c0 100644 --- a/engine/vcs/gitea/client_status.go +++ b/engine/vcs/gitea/client_status.go @@ -79,6 +79,37 @@ func (client *giteaClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBui return nil } -func (client *giteaClient) ListStatuses(ctx context.Context, repo string, ref string) ([]sdk.VCSCommitStatus, error) { - return nil, sdk.WithStack(sdk.ErrNotImplemented) +func (client *giteaClient) ListStatuses(ctx context.Context, fullname string, ref string) ([]sdk.VCSCommitStatus, error) { + + t := strings.Split(fullname, "/") + if len(t) != 2 { + return nil, fmt.Errorf("giteaCliet.Tags> invalid fullname %s", fullname) + } + statuses, _, err := client.client.ListStatuses(t[0], t[1], ref, gitea.ListStatusesOption{}) + if err != nil { + return nil, err + } + + var vcsStatuses []sdk.VCSCommitStatus + for _, s := range statuses { + vcsStatuses = append(vcsStatuses, sdk.VCSCommitStatus{ + CreatedAt: s.Created, + Decription: s.Context, + Ref: ref, + State: processGiteaState(s.State), + }) + } + + return vcsStatuses, nil +} + +func processGiteaState(s gitea.StatusState) string { + switch s { + case "success": + return sdk.StatusSuccess + case "error", "failure": + return sdk.StatusFail + default: + return sdk.StatusDisabled + } } diff --git a/engine/vcs/gitea/client_tags.go b/engine/vcs/gitea/client_tags.go index 33855721e7..82a272036f 100644 --- a/engine/vcs/gitea/client_tags.go +++ b/engine/vcs/gitea/client_tags.go @@ -2,14 +2,46 @@ package gitea import ( "context" + "fmt" + "strings" + + "code.gitea.io/sdk/gitea" "github.com/ovh/cds/sdk" ) // Tags retrieve tags -func (g *giteaClient) Tags(ctx context.Context, fullname string) ([]sdk.VCSTag, error) { - return nil, sdk.WithStack(sdk.ErrNotImplemented) +func (c *giteaClient) Tags(ctx context.Context, fullname string) ([]sdk.VCSTag, error) { + t := strings.Split(fullname, "/") + if len(t) != 2 { + return nil, fmt.Errorf("giteaCliet.Tags> invalid fullname %s", fullname) + } + tags, _, err := c.client.ListRepoTags(t[0], t[1], gitea.ListRepoTagsOptions{}) + if err != nil { + return nil, err + } + + var ret []sdk.VCSTag + for _, tag := range tags { + ret = append(ret, sdk.VCSTag{ + Tag: tag.Name, + Sha: tag.Commit.SHA, + }) + } + + return ret, nil } func (c *giteaClient) Tag(ctx context.Context, fullname string, tagName string) (sdk.VCSTag, error) { - return sdk.VCSTag{}, sdk.WithStack(sdk.ErrNotImplemented) + t := strings.Split(fullname, "/") + if len(t) != 2 { + return sdk.VCSTag{}, fmt.Errorf("giteaCliet.Tag> invalid fullname %s", fullname) + } + tag, _, err := c.client.GetTag(t[0], t[1], tagName) + if err != nil { + return sdk.VCSTag{}, err + } + return sdk.VCSTag{ + Tag: tag.Name, + Sha: tag.Commit.SHA, + }, sdk.WrapError(sdk.ErrNotFound, "tag not found") } diff --git a/engine/vcs/gitlab/client_status.go b/engine/vcs/gitlab/client_status.go index ffc7d9d3e4..b7dd43e9ec 100644 --- a/engine/vcs/gitlab/client_status.go +++ b/engine/vcs/gitlab/client_status.go @@ -64,6 +64,8 @@ func (c *gitlabClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildSt return sdk.WrapError(err, "unable to get commit statuses - repo:%s hash:%s", buildStatus.RepositoryFullname, buildStatus.GitHash) } + log.Debug(ctx, "gitlabClient.SetStatus> existing nb statuses: %d", len(val)) + found := false for _, s := range val { sameRequest := s.TargetURL == *opt.TargetURL && // Comparing TargetURL as there is the workflow run number inside @@ -80,6 +82,7 @@ func (c *gitlabClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildSt } } if !found { + log.Debug(ctx, "gitlabClient.SetStatus> gitlab set status on %v hash:%v status:%v", buildStatus.RepositoryFullname, buildStatus.GitHash, buildStatus.Status) if _, _, err := c.client.Commits.SetCommitStatus(buildStatus.RepositoryFullname, buildStatus.GitHash, opt); err != nil { return sdk.WrapError(err, "cannot process event repo:%s hash:%s", buildStatus.RepositoryFullname, buildStatus.GitHash) } diff --git a/tests/08_v2_workflow_on_pullrequest.yml b/tests/08_v2_workflow_on_pullrequest.yml index e476d6f02d..ab83108f8b 100644 --- a/tests/08_v2_workflow_on_pullrequest.yml +++ b/tests/08_v2_workflow_on_pullrequest.yml @@ -74,7 +74,8 @@ testcases: - run: |- #!/bin/bash set -ex - echo "$GIT_REF" + echo "GIT_REF: $GIT_REF" + echo "GIT_SHA: $GIT_SHA" echo "job done" && date - name: Check CDS project analyses status after push on master @@ -129,6 +130,8 @@ testcases: vars: prNumber: from: result.bodyjson.number + gitSha: + from: result.bodyjson.head.sha - name: Check if workflow triggered on the new-branch is ok after pull-request creation steps: @@ -156,14 +159,23 @@ testcases: basic_auth_password: "{{.git.password}}" headers: Content-Type: application/json - body: > - { - "base": "master", - "body": "test body pull-request", - "head": "new-branch", - "title": "test pull-request" - } assertions: - result.statuscode ShouldEqual 200 - result.bodyjson ShouldHaveLength 1 - - result.bodyjson.bodyjson0.body ShouldEqual "a comment here" \ No newline at end of file + - result.bodyjson.bodyjson0.body ShouldEqual "a comment here" + +- name: Check status created on gitea + steps: + - name: Get status from gitea + type: http + method: GET + url: "{{.git.host}}/api/v1/repos/{{.git.user}}/{{.git_repo}}/commits/{{.Create-Pull-Request.gitSha}}/status" + basic_auth_user: "{{.git.user}}" + basic_auth_password: "{{.git.password}}" + headers: + Content-Type: application/json + assertions: + - result.statuscode ShouldEqual 200 + - result.bodyjson.statuses ShouldHaveLength 1 + - result.bodyjson.statuses.statuses0.status ShouldEqual "success" + - result.bodyjson.statuses.statuses0.context ShouldEqual "ITV2WFNOTIF-WorkflowNotificationsLog" \ No newline at end of file From 0a821defc83c753c1b1e9335bba2ca0361634ced Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 12 Feb 2024 08:31:59 +0000 Subject: [PATCH 15/17] fix: cr Signed-off-by: Yvonnick Esnault --- engine/api/event_v2/event.go | 14 +++++- .../repositories_manager.go | 1 - engine/api/v2_repository_analyze_test.go | 4 ++ engine/api/v2_workflow_run.go | 4 +- engine/api/workflow/workflow_run_event.go | 1 + engine/vcs/bitbucketcloud/client_status.go | 2 +- engine/vcs/bitbucketserver/client_status.go | 2 +- engine/vcs/gitlab/client_status.go | 2 +- sdk/hooks_repository_event.go | 4 +- sdk/log/formatter.go | 43 ++----------------- sdk/v2_workflow.go | 10 ++++- sdk/v2_workflow_run.go | 4 +- sdk/v2_workflow_test.go | 8 ++-- sdk/vcs.go | 5 ++- 14 files changed, 47 insertions(+), 57 deletions(-) diff --git a/engine/api/event_v2/event.go b/engine/api/event_v2/event.go index f12b32d3ee..f5be59f73f 100644 --- a/engine/api/event_v2/event.go +++ b/engine/api/event_v2/event.go @@ -137,8 +137,20 @@ func workflowNotifications(ctx context.Context, db *gorp.DbMap, store cache.Stor return sdk.WrapError(err, "can't get AuthorizedClient for %v/%v", event.ProjectKey, event.VCSName) } + title := fmt.Sprintf("%s-%s", event.ProjectKey, run.WorkflowName) + description := run.WorkflowName + ":" + run.Status + if run.WorkflowData.Workflow.CommitStatus != nil { + if run.WorkflowData.Workflow.CommitStatus.Title != "" { + title = run.WorkflowData.Workflow.CommitStatus.Title + } + if run.WorkflowData.Workflow.CommitStatus.Description != "" { + description = run.WorkflowData.Workflow.CommitStatus.Description + } + } + buildStatus := sdk.VCSBuildStatus{ - Description: run.WorkflowName + ":" + run.Status, + Title: title, + Description: description, URLCDS: fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", cdsUIURL, event.ProjectKey, run.WorkflowName, event.RunNumber), Context: fmt.Sprintf("%s-%s", event.ProjectKey, run.WorkflowName), Status: event.Status, diff --git a/engine/api/repositoriesmanager/repositories_manager.go b/engine/api/repositoriesmanager/repositories_manager.go index 0a31d2fec7..eacfe37738 100644 --- a/engine/api/repositoriesmanager/repositories_manager.go +++ b/engine/api/repositoriesmanager/repositories_manager.go @@ -114,7 +114,6 @@ func AuthorizedClient(ctx context.Context, db gorp.SqlExecutor, store cache.Stor } func (c *vcsClient) setAuthHeader(ctx context.Context, req *http.Request) { - log.Debug(ctx, "requesting vcs via vcs project") req.Header.Set(sdk.HeaderXVCSType, base64.StdEncoding.EncodeToString([]byte(c.vcsProject.Type))) req.Header.Set(sdk.HeaderXVCSURL, base64.StdEncoding.EncodeToString([]byte(c.vcsProject.URL))) req.Header.Set(sdk.HeaderXVCSUsername, base64.StdEncoding.EncodeToString([]byte(c.vcsProject.Auth.Username))) diff --git a/engine/api/v2_repository_analyze_test.go b/engine/api/v2_repository_analyze_test.go index f55587d59a..79302b72b4 100644 --- a/engine/api/v2_repository_analyze_test.go +++ b/engine/api/v2_repository_analyze_test.go @@ -1404,6 +1404,10 @@ func TestManageWorkflowHooksAllSameRepo(t *testing.T) { VCSServer: vcsServer.Name, Name: repoDef.Name, }, + CommitStatus: &sdk.CommitStatus{ + Title: "foo", + Description: "bar", + }, On: &sdk.WorkflowOn{ Push: &sdk.WorkflowOnPush{}, WorkflowUpdate: &sdk.WorkflowOnWorkflowUpdate{}, diff --git a/engine/api/v2_workflow_run.go b/engine/api/v2_workflow_run.go index df9195d440..881889381e 100644 --- a/engine/api/v2_workflow_run.go +++ b/engine/api/v2_workflow_run.go @@ -1265,9 +1265,9 @@ func (api *API) startWorkflowV2(ctx context.Context, proj sdk.Project, vcsProjec case runEvent.GitTrigger != nil: msg = fmt.Sprintf("The workflow was triggered by the repository webhook event %s by user %s", runEvent.GitTrigger.EventName, u.Username) case runEvent.WorkflowUpdateTrigger != nil: - msg = fmt.Sprintf("Workflow was triggered by the workflow_update hook by user %s", u.Username) + msg = fmt.Sprintf("Workflow was triggered by the workflow-update hook by user %s", u.Username) case runEvent.ModelUpdateTrigger != nil: - msg = fmt.Sprintf("Workflow was triggered by the model_update hook by user %s", u.Username) + msg = fmt.Sprintf("Workflow was triggered by the model-update hook by user %s", u.Username) default: return nil, sdk.WrapError(sdk.ErrNotImplemented, "event not implemented") } diff --git a/engine/api/workflow/workflow_run_event.go b/engine/api/workflow/workflow_run_event.go index 8d113c5d0d..a1cf5f9cad 100644 --- a/engine/api/workflow/workflow_run_event.go +++ b/engine/api/workflow/workflow_run_event.go @@ -305,6 +305,7 @@ func (e *VCSEventMessenger) sendVCSEventStatus(ctx context.Context, db gorp.SqlE } buildStatus := sdk.VCSBuildStatus{ + Title: fmt.Sprintf("%s-%s-%s", evt.ProjectKey, evt.WorkflowName, eventWNR.NodeName), Description: eventWNR.NodeName + ": " + eventWNR.Status, URLCDS: fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", cdsUIURL, evt.ProjectKey, evt.WorkflowName, eventWNR.Number), Context: fmt.Sprintf("%s-%s-%s", evt.ProjectKey, evt.WorkflowName, eventWNR.NodeName), diff --git a/engine/vcs/bitbucketcloud/client_status.go b/engine/vcs/bitbucketcloud/client_status.go index a92bc48aed..9a55e26860 100644 --- a/engine/vcs/bitbucketcloud/client_status.go +++ b/engine/vcs/bitbucketcloud/client_status.go @@ -45,7 +45,7 @@ func (client *bitbucketcloudClient) SetStatus(ctx context.Context, buildStatus s Description: buildStatus.Description, URL: buildStatus.URLCDS, State: state, - Name: buildStatus.Context, + Name: buildStatus.Title, Key: buildStatus.Context, } diff --git a/engine/vcs/bitbucketserver/client_status.go b/engine/vcs/bitbucketserver/client_status.go index 59f3c06e14..a04f2b67b3 100644 --- a/engine/vcs/bitbucketserver/client_status.go +++ b/engine/vcs/bitbucketserver/client_status.go @@ -21,7 +21,7 @@ func (client *bitbucketClient) SetStatus(ctx context.Context, buildStatus sdk.VC state := getBitbucketStateFromStatus(buildStatus.Status) status := Status{ Key: buildStatus.Context, - Name: buildStatus.Context, + Name: buildStatus.Title, State: state, URL: buildStatus.URLCDS, Description: buildStatus.Description, diff --git a/engine/vcs/gitlab/client_status.go b/engine/vcs/gitlab/client_status.go index b7dd43e9ec..b77f7178fb 100644 --- a/engine/vcs/gitlab/client_status.go +++ b/engine/vcs/gitlab/client_status.go @@ -51,7 +51,7 @@ func (c *gitlabClient) SetStatus(ctx context.Context, buildStatus sdk.VCSBuildSt cds := "CDS" opt := &gitlab.SetCommitStatusOptions{ - Name: &cds, + Name: &buildStatus.Title, Context: &cds, State: getGitlabStateFromStatus(buildStatus.Status), Ref: &buildStatus.GitHash, diff --git a/sdk/hooks_repository_event.go b/sdk/hooks_repository_event.go index cdd500dc2d..143db84423 100644 --- a/sdk/hooks_repository_event.go +++ b/sdk/hooks_repository_event.go @@ -15,8 +15,8 @@ const ( SignHeaderVCSType = "X-Cds-Hooks-Vcs-Type" SignHeaderEventName = "X-Cds-Hooks-Event-Name" - WorkflowHookEventWorkflowUpdate = "workflow_update" - WorkflowHookEventModelUpdate = "model_update" + WorkflowHookEventWorkflowUpdate = "workflow-update" + WorkflowHookEventModelUpdate = "model-update" WorkflowHookEventPush = "push" WorkflowHookEventPullRequest = "pull-request" diff --git a/sdk/log/formatter.go b/sdk/log/formatter.go index 3215fcdc50..4ecd33f079 100644 --- a/sdk/log/formatter.go +++ b/sdk/log/formatter.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/mgutz/ansi" + "github.com/rockbears/log" "github.com/sirupsen/logrus" ) @@ -55,49 +56,13 @@ func (f *CDSFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry, keys [ levelText = "[" + levelText + "]" fmt.Fprintf(b, "%s %s%+5s%s %s", entry.Time.Format("2006-01-02 15:04:05"), levelColor, levelText, ansi.Reset, entry.Message) - if !f.DisabledPrintFields { - for _, k := range keys { + + for _, k := range keys { + if !f.DisabledPrintFields || k == string(log.FieldStackTrace) { v := entry.Data[k] fmt.Fprintf(b, " %s%s%s=%+v", levelColor, k, ansi.Reset, v) } } - -} - -func needsQuoting(text string) bool { - for _, ch := range text { - if !((ch >= 'a' && ch <= 'z') || - (ch >= 'A' && ch <= 'Z') || - (ch >= '0' && ch <= '9') || - ch == '-' || ch == '.') { - return false - } - } - return true -} - -func (f *CDSFormatter) appendKeyValue(b *bytes.Buffer, key string, value interface{}) { - b.WriteString(key) - b.WriteByte('=') - - switch value := value.(type) { - case string: - if needsQuoting(value) { - b.WriteString(value) - } else { - fmt.Fprintf(b, "%q", value) - } - case error: - errmsg := value.Error() - if needsQuoting(errmsg) { - b.WriteString(errmsg) - } else { - fmt.Fprintf(b, "%q", value) - } - default: - fmt.Fprint(b, value) - } - b.WriteByte(' ') } func prefixFieldClashes(data logrus.Fields) { diff --git a/sdk/v2_workflow.go b/sdk/v2_workflow.go index bebecd10f1..39dcc7ab06 100644 --- a/sdk/v2_workflow.go +++ b/sdk/v2_workflow.go @@ -19,6 +19,7 @@ type V2Workflow struct { Name string `json:"name"` Repository *WorkflowRepository `json:"repository,omitempty"` OnRaw json.RawMessage `json:"on,omitempty"` + CommitStatus *CommitStatus `json:"commit-status,omitempty"` On *WorkflowOn `json:"-" yaml:"-"` Stages map[string]WorkflowStage `json:"stages,omitempty"` Gates map[string]V2JobGate `json:"gates,omitempty"` @@ -28,12 +29,17 @@ type V2Workflow struct { VariableSets []string `json:"vars,omitempty"` } +type CommitStatus struct { + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` +} + type WorkflowOn struct { Push *WorkflowOnPush `json:"push,omitempty"` PullRequest *WorkflowOnPullRequest `json:"pull-request,omitempty"` PullRequestComment *WorkflowOnPullRequestComment `json:"pull-request-comment,omitempty"` - ModelUpdate *WorkflowOnModelUpdate `json:"model_update,omitempty"` - WorkflowUpdate *WorkflowOnWorkflowUpdate `json:"workflow_update,omitempty"` + ModelUpdate *WorkflowOnModelUpdate `json:"model-update,omitempty"` + WorkflowUpdate *WorkflowOnWorkflowUpdate `json:"workflow-update,omitempty"` } type WorkflowOnPush struct { diff --git a/sdk/v2_workflow_run.go b/sdk/v2_workflow_run.go index 64ca46fd9c..15de82acc2 100644 --- a/sdk/v2_workflow_run.go +++ b/sdk/v2_workflow_run.go @@ -144,8 +144,8 @@ func (w *V2WorkflowRunJobEvents) Scan(src interface{}) error { type V2WorkflowRunEvent struct { Manual *ManualTrigger `json:"manual,omitempty"` GitTrigger *GitTrigger `json:"git,omitempty"` - WorkflowUpdateTrigger *WorkflowUpdateTrigger `json:"workflow_update,omitempty"` - ModelUpdateTrigger *ModelUpdateTrigger `json:"model_update,omitempty"` + WorkflowUpdateTrigger *WorkflowUpdateTrigger `json:"workflow-update,omitempty"` + ModelUpdateTrigger *ModelUpdateTrigger `json:"model-update,omitempty"` // TODO Scheduler *SchedulerTrigger `json:"scheduler"` diff --git a/sdk/v2_workflow_test.go b/sdk/v2_workflow_test.go index 5676ecf94c..bf50f6441f 100644 --- a/sdk/v2_workflow_test.go +++ b/sdk/v2_workflow_test.go @@ -17,14 +17,14 @@ func TestUnmarshalV2WorkflowHooksDetailed(t *testing.T) { - run: 'echo "Workflow: ${{cds.workflow}}"' name: MyDistantWorkflow "on": - model_update: + model-update: models: - mymodel target_branch: develop push: branches: - master - workflow_update: + workflow-update: target_branch: master ` var w V2Workflow @@ -47,8 +47,8 @@ func TestUnmarshalV2WorkflowHooksShort(t *testing.T) { name: MyDistantWorkflow "on": - push - - workflow_update - - model_update + - workflow-update + - model-update ` var w V2Workflow require.NoError(t, yaml.Unmarshal([]byte(src), &w)) diff --git a/sdk/vcs.go b/sdk/vcs.go index f9b7b349d4..aa54f0b59d 100644 --- a/sdk/vcs.go +++ b/sdk/vcs.go @@ -416,6 +416,9 @@ func VCSCommitStatusDescription(projKey, workflowName string, evt EventRunWorkfl } type VCSBuildStatus struct { + // v2: fmt.Sprintf("%s-%s", event.ProjectKey, event.WorkflowName) + Title string `json:"title"` + // v1:eventNR.NodeName + ": " + eventNR.Status // v2: Workflow.Name + ": " +Status Description string `json:"description"` @@ -425,7 +428,7 @@ type VCSBuildStatus struct { URLCDS string `json:"url_cds"` // // v1: fmt.Sprintf("%s-%s-%s", event.ProjectKey, event.WorkflowName, eventNR.NodeName) - // v2: TODO + // v2: fmt.Sprintf("%s-%s", event.ProjectKey, event.WorkflowName) Context string `json:"context"` Status string `json:"status"` From 14091a9e8acfa940ef0e2d83e2f1b83e4be59e86 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 12 Feb 2024 08:34:27 +0000 Subject: [PATCH 16/17] fix: cr Signed-off-by: Yvonnick Esnault --- engine/hooks/hook_repository_event.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/hooks/hook_repository_event.go b/engine/hooks/hook_repository_event.go index 7bf2b4e131..2d7b42bca7 100644 --- a/engine/hooks/hook_repository_event.go +++ b/engine/hooks/hook_repository_event.go @@ -42,7 +42,7 @@ func (s *Service) extractDataFromGiteaRequest(headers http.Header, body []byte, extractedData.Ref = request.Ref extractedData.Commit = request.After case "pull_request": - extractedData.Ref = "refs/heads/" + request.PullRequest.Head.Ref + extractedData.Ref = sdk.GitRefBranchPrefix + request.PullRequest.Head.Ref extractedData.Commit = request.PullRequest.Head.Sha switch request.Action { case "opened": From c94b0b6b6bb7a46f0fdf272116e071d374051dff Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Tue, 13 Feb 2024 07:42:18 +0000 Subject: [PATCH 17/17] fix: cr Signed-off-by: Yvonnick Esnault --- engine/cmd_start.go | 2 ++ engine/types.go | 8 +++++--- sdk/log/formatter.go | 16 ++++++++++++---- sdk/log/log.go | 12 +++++++++--- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/engine/cmd_start.go b/engine/cmd_start.go index 8e32ed35d1..83ac29bfc1 100644 --- a/engine/cmd_start.go +++ b/engine/cmd_start.go @@ -251,6 +251,8 @@ See $ engine config command for more details. logConf := cdslog.Conf{ Level: conf.Log.Level, Format: conf.Log.Format, + TextFields: conf.Log.TextFields, + SkipTextFields: conf.Log.SkipTextFields, GraylogProtocol: conf.Log.Graylog.Protocol, GraylogHost: conf.Log.Graylog.Host, GraylogPort: fmt.Sprintf("%d", conf.Log.Graylog.Port), diff --git a/engine/types.go b/engine/types.go index f2d879541f..1d2ef665cc 100644 --- a/engine/types.go +++ b/engine/types.go @@ -21,9 +21,11 @@ import ( type Configuration struct { // common Log struct { - Level string `toml:"level" default:"warning" comment:"Log Level: debug, info, warning, notice, error" json:"level"` - Format string `toml:"format" default:"text" comment:"Stdout format: text, json, discard" json:"format"` - Graylog struct { + Level string `toml:"level" default:"warning" comment:"Log Level: debug, info, warning, notice, error" json:"level"` + Format string `toml:"format" default:"text" comment:"Stdout format: text, json, discard" json:"format"` + TextFields []string `toml:"textFields" default:"" json:"textFields" commented:"true" comment:"Can be used only with text format. Empty values = all fields will be displayed, example: [\"request_uri\",\"request_id\",\"stack_trace\",\"status\"]".` + SkipTextFields []string `toml:"skipTextFields" default:"" json:"skipTextFields" commented:"true" comment:"Can be used only with text format. Skip logs with some fields. Empty values = all logs will be displayed, example: [\"handler=api.(*API).postServiceHearbeatHandler-fm.(*API).postServiceHearbeatHandler\"]".` + Graylog struct { Host string `toml:"host" comment:"Example: thot.ovh.com" json:"host"` Port int `toml:"port" comment:"Example: 12202" json:"port"` Protocol string `toml:"protocol" default:"tcp" comment:"tcp or udp" json:"protocol"` diff --git a/sdk/log/formatter.go b/sdk/log/formatter.go index 4ecd33f079..1fd67521b0 100644 --- a/sdk/log/formatter.go +++ b/sdk/log/formatter.go @@ -7,13 +7,12 @@ import ( "strings" "github.com/mgutz/ansi" - "github.com/rockbears/log" "github.com/sirupsen/logrus" ) // CDSFormatter ... type CDSFormatter struct { - DisabledPrintFields bool + Fields []string } // Format format a log @@ -58,13 +57,22 @@ func (f *CDSFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry, keys [ fmt.Fprintf(b, "%s %s%+5s%s %s", entry.Time.Format("2006-01-02 15:04:05"), levelColor, levelText, ansi.Reset, entry.Message) for _, k := range keys { - if !f.DisabledPrintFields || k == string(log.FieldStackTrace) { - v := entry.Data[k] + v := entry.Data[k] + if f.Fields == nil || fieldsInArray(k, f.Fields) { fmt.Fprintf(b, " %s%s%s=%+v", levelColor, k, ansi.Reset, v) } } } +func fieldsInArray(elt string, array []string) bool { + for _, item := range array { + if item == elt { + return true + } + } + return false +} + func prefixFieldClashes(data logrus.Fields) { if _, ok := data["time"]; ok { data["fields.time"] = data["time"] diff --git a/sdk/log/log.go b/sdk/log/log.go index 4b210acce1..e7dc08e65c 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -19,6 +19,8 @@ import ( type Conf struct { Level string Format string + TextFields []string + SkipTextFields []string GraylogHost string GraylogPort string GraylogProtocol string @@ -62,10 +64,14 @@ func Initialize(ctx context.Context, conf *Conf) { logrus.SetOutput(io.Discard) case "json": logrus.SetFormatter(&logrus.JSONFormatter{}) - case "readable": - logrus.SetFormatter(&CDSFormatter{DisabledPrintFields: true}) default: - logrus.SetFormatter(&CDSFormatter{}) + for _, v := range conf.SkipTextFields { + t := strings.SplitN(v, "=", 2) + fieldName := t[0] + fieldValue := t[1] + log.Skip(log.Field(fieldName), fieldValue) + } + logrus.SetFormatter(&CDSFormatter{Fields: conf.TextFields}) } if conf.GraylogHost != "" && conf.GraylogPort != "" {