From 6b1466af7172a8d14096dea29fdc864a4ab4cfb5 Mon Sep 17 00:00:00 2001 From: Gregoire Mahe <31134503+gregoiremahe@users.noreply.github.com> Date: Mon, 10 Jun 2019 18:17:05 +0200 Subject: [PATCH 01/11] feat(contril/helm): replace --decode by -d to be compatible with more distribs / operating systems (#4340) --- contrib/helm/cds/templates/api-deployment.yaml | 2 +- contrib/helm/cds/templates/elasticsearch-deployment.yaml | 2 +- contrib/helm/cds/templates/hooks-deployment.yaml | 2 +- contrib/helm/cds/templates/repositories-deployment.yaml | 2 +- contrib/helm/cds/templates/vcs-deployment.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/helm/cds/templates/api-deployment.yaml b/contrib/helm/cds/templates/api-deployment.yaml index 01cc85a36b..cb70008f24 100644 --- a/contrib/helm/cds/templates/api-deployment.yaml +++ b/contrib/helm/cds/templates/api-deployment.yaml @@ -87,7 +87,7 @@ spec: name: {{ template "cds.fullname" . }} key: cds-api_secrets_key command: ["/bin/sh"] - args: ["-c", "echo $CDS_CONFIG_FILE | base64 --decode > config.toml && /app/cds-engine-linux-amd64 start api --config config.toml"] + args: ["-c", "echo $CDS_CONFIG_FILE | base64 -d > config.toml && /app/cds-engine-linux-amd64 start api --config config.toml"] ports: - name: http containerPort: 8081 diff --git a/contrib/helm/cds/templates/elasticsearch-deployment.yaml b/contrib/helm/cds/templates/elasticsearch-deployment.yaml index 6b1e4f6a5f..d4b9e603ea 100644 --- a/contrib/helm/cds/templates/elasticsearch-deployment.yaml +++ b/contrib/helm/cds/templates/elasticsearch-deployment.yaml @@ -66,7 +66,7 @@ spec: - name: CDS_LOG_LEVEL value: {{ default "" .Values.logLevel | quote }} command: ["/bin/sh"] - args: ["-c", "echo $CDS_CONFIG_FILE | base64 --decode > config.toml && /app/cds-engine-linux-amd64 start elasticsearch --config config.toml"] + args: ["-c", "echo $CDS_CONFIG_FILE | base64 -d > config.toml && /app/cds-engine-linux-amd64 start elasticsearch --config config.toml"] ports: - name: http containerPort: 8084 diff --git a/contrib/helm/cds/templates/hooks-deployment.yaml b/contrib/helm/cds/templates/hooks-deployment.yaml index 174e48cc00..8f6ce96866 100644 --- a/contrib/helm/cds/templates/hooks-deployment.yaml +++ b/contrib/helm/cds/templates/hooks-deployment.yaml @@ -85,7 +85,7 @@ spec: - name: CDS_LOG_LEVEL value: {{ default "" .Values.logLevel | quote }} command: ["/bin/sh"] - args: ["-c", "echo $CDS_CONFIG_FILE | base64 --decode > config.toml && /app/cds-engine-linux-amd64 start hooks --config config.toml"] + args: ["-c", "echo $CDS_CONFIG_FILE | base64 -d > config.toml && /app/cds-engine-linux-amd64 start hooks --config config.toml"] ports: - name: http containerPort: 8084 diff --git a/contrib/helm/cds/templates/repositories-deployment.yaml b/contrib/helm/cds/templates/repositories-deployment.yaml index 0c7dc47731..9ba122f5ea 100644 --- a/contrib/helm/cds/templates/repositories-deployment.yaml +++ b/contrib/helm/cds/templates/repositories-deployment.yaml @@ -71,7 +71,7 @@ spec: - name: CDS_LOG_LEVEL value: {{ default "" .Values.logLevel | quote }} command: ["/bin/sh"] - args: ["-c", "echo $CDS_CONFIG_FILE | base64 --decode > config.toml && /app/cds-engine-linux-amd64 start repositories --config config.toml"] + args: ["-c", "echo $CDS_CONFIG_FILE | base64 -d > config.toml && /app/cds-engine-linux-amd64 start repositories --config config.toml"] ports: - name: http containerPort: 8084 diff --git a/contrib/helm/cds/templates/vcs-deployment.yaml b/contrib/helm/cds/templates/vcs-deployment.yaml index f5b5e80b80..04d11da238 100644 --- a/contrib/helm/cds/templates/vcs-deployment.yaml +++ b/contrib/helm/cds/templates/vcs-deployment.yaml @@ -85,7 +85,7 @@ spec: - name: CDS_LOG_LEVEL value: {{ default "" .Values.logLevel | quote }} command: ["/bin/sh"] - args: ["-c", "echo $CDS_CONFIG_FILE | base64 --decode > config.toml && /app/cds-engine-linux-amd64 start vcs --config config.toml"] + args: ["-c", "echo $CDS_CONFIG_FILE | base64 -d > config.toml && /app/cds-engine-linux-amd64 start vcs --config config.toml"] ports: - name: http containerPort: 8084 From 023723b1a1719594f90b59c0895e5db2a5c638a8 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Tue, 11 Jun 2019 15:14:16 +0200 Subject: [PATCH 02/11] fix(hatchery/openstack): manage multiple images with same name (#4343) * If multiple images (due to probably race-condition): - use the real image on spawn - delete all images with same name when uploading a new ones * Add retries on "Fixed IP address x.x.x.x is already in use on instance" * Apply suggestions from code review --- engine/hatchery/openstack/openstack.go | 41 ++++++++---------- engine/hatchery/openstack/spawn.go | 57 +++++++++++++++----------- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/engine/hatchery/openstack/openstack.go b/engine/hatchery/openstack/openstack.go index e02f707949..aa429c9e69 100644 --- a/engine/hatchery/openstack/openstack.go +++ b/engine/hatchery/openstack/openstack.go @@ -302,23 +302,17 @@ func (h *HatcheryOpenstack) killAwolServers() { } func (h *HatcheryOpenstack) killAwolServersComputeImage(workerModelName, workerModelNameLastModified, serverID, model, flavor string) { - var oldImageID string - var oldDateLastModified string + oldImagesID := []string{} for _, img := range h.getImages() { - w, _ := img.Metadata["worker_model_name"] - if w == workerModelName { - oldImageID = img.ID - if d, ok := img.Metadata["worker_model_last_modified"]; ok { - oldDateLastModified = d.(string) + if w, _ := img.Metadata["worker_model_name"]; w == workerModelName { + oldImagesID = append(oldImagesID, img.ID) + if d, ok := img.Metadata["worker_model_last_modified"]; ok && d.(string) == workerModelNameLastModified { + // no need to recreate an image + return } } } - if oldDateLastModified == workerModelNameLastModified { - // no need to recreate an image - return - } - log.Info("killAwolServersComputeImage> create image before deleting server") imageID, err := servers.CreateImage(h.openstackClient, serverID, servers.CreateImageOpts{ Name: "cds_image_" + workerModelName, @@ -358,12 +352,13 @@ func (h *HatcheryOpenstack) killAwolServersComputeImage(workerModelName, workerM } } - if oldImageID != "" { + for _, oldImageID := range oldImagesID { log.Info("killAwolServersComputeImage> deleting old image for %s with ID %s", workerModelName, oldImageID) if err := images.Delete(h.openstackClient, oldImageID).ExtractErr(); err != nil { log.Error("killAwolServersComputeImage> error while deleting old image %s", oldImageID) } } + h.resetImagesCache() } } @@ -469,21 +464,21 @@ func (h *HatcheryOpenstack) WorkersStartedByModel(model *sdk.Model) int { // NeedRegistration return true if worker model need regsitration func (h *HatcheryOpenstack) NeedRegistration(m *sdk.Model) bool { - var oldDateLastModified string + if m.NeedRegistration { + log.Debug("NeedRegistration> true as worker model.NeedRegistration=true") + return true + } for _, img := range h.getImages() { w, _ := img.Metadata["worker_model_name"] if w == m.Name { if d, ok := img.Metadata["worker_model_last_modified"]; ok { - oldDateLastModified = d.(string) - break + if fmt.Sprintf("%d", m.UserLastModified.Unix()) == d.(string) { + log.Debug("NeedRegistration> false. An image is already available for this workerModel.UserLastModified") + return false + } } } } - - var out bool - if m.NeedRegistration || fmt.Sprintf("%d", m.UserLastModified.Unix()) != oldDateLastModified { - out = true - } - log.Debug("NeedRegistration> %t for %s - m.NeedRegistration:%t m.UserLastModified:%d oldDateLastModified:%s", out, m.Name, m.NeedRegistration, m.UserLastModified.Unix(), oldDateLastModified) - return out + log.Debug("NeedRegistration> true. No existing image found for this worker model") + return true } diff --git a/engine/hatchery/openstack/spawn.go b/engine/hatchery/openstack/spawn.go index 14882666eb..aabe666fdb 100644 --- a/engine/hatchery/openstack/spawn.go +++ b/engine/hatchery/openstack/spawn.go @@ -61,7 +61,8 @@ func (h *HatcheryOpenstack) SpawnWorker(ctx context.Context, spawnArgs hatchery. log.Debug("spawnWorker> call images.List on openstack took %fs, nbImages:%d", time.Since(start).Seconds(), len(imgs)) for _, img := range imgs { workerModelName, _ := img.Metadata["worker_model_name"] - if workerModelName == spawnArgs.Model.Name { + workerModelLastModified, _ := img.Metadata["worker_model_last_modified"] + if workerModelName == spawnArgs.Model.Name && workerModelLastModified == spawnArgs.Model.UserLastModified.Unix() { withExistingImage = true var jobInfo string if spawnArgs.JobID != 0 { @@ -122,31 +123,39 @@ func (h *HatcheryOpenstack) SpawnWorker(ctx context.Context, spawnArgs hatchery. "worker_model_last_modified": fmt.Sprintf("%d", spawnArgs.Model.UserLastModified.Unix()), } - // Ip len(ipsInfos.ips) > 0, specify one of those - var ip string - if len(ipsInfos.ips) > 0 { - var errai error - ip, errai = h.findAvailableIP(name) - if errai != nil { - return "", errai + maxTries := 3 + for try := 1; try <= maxTries; try++ { + // Ip len(ipsInfos.ips) > 0, specify one of those + var ip string + if len(ipsInfos.ips) > 0 { + var errai error + ip, errai = h.findAvailableIP(name) + if errai != nil { + return "", errai + } + log.Debug("Found %s as available IP", ip) } - log.Debug("Found %s as available IP", ip) - } - networks := []servers.Network{{UUID: h.networkID, FixedIP: ip}} - r := servers.Create(h.openstackClient, servers.CreateOpts{ - Name: name, - FlavorRef: flavorID, - ImageRef: imageID, - Metadata: meta, - UserData: []byte(udata64), - Networks: networks, - }) - - server, err := r.Extract() - if err != nil { - return "", fmt.Errorf("SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%s body:%s", name, flavorID, imageID, meta, networks, err, r.Body) + networks := []servers.Network{{UUID: h.networkID, FixedIP: ip}} + r := servers.Create(h.openstackClient, servers.CreateOpts{ + Name: name, + FlavorRef: flavorID, + ImageRef: imageID, + Metadata: meta, + UserData: []byte(udata64), + Networks: networks, + }) + + server, err := r.Extract() + if err != nil { + if strings.Contains(err.Error(), "is already in use on instance") && try < maxTries { // Fixed IP address X.X.X.X is already in use on instance + log.Warning("SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%v body:%s - Try %d/%d", name, flavorID, imageID, meta, networks, err, r.Body, try, maxTries) + continue + } + return "", fmt.Errorf("SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%v body:%s", name, flavorID, imageID, meta, networks, err, r.Body) + } + log.Debug("SpawnWorker> Created Server ID: %s", server.ID) + break } - log.Debug("SpawnWorker> Created Server ID: %s", server.ID) return name, nil } From da579089b09331e71569761472c3e747c9a782db Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Wed, 12 Jun 2019 15:54:57 +0200 Subject: [PATCH 03/11] feat(vcs): add bitbucket cloud support (#4350) --- .../workflow/hooks/git-repo-webhook.md | 2 +- docs/content/docs/integrations/bitbucket.md | 2 +- .../docs/integrations/bitbucketcloud.md | 74 +++ engine/api/admin.go | 4 +- engine/api/application.go | 2 +- engine/api/ascode.go | 4 +- engine/api/event/elasticsearch.go | 4 +- engine/api/hook.go | 2 +- engine/api/metrics/elasticsearch.go | 4 +- engine/api/repositories_manager.go | 29 +- engine/api/repositoriesmanager/dao.go | 23 + engine/api/repositoriesmanager/events.go | 2 +- .../repositories_manager.go | 117 ++++- engine/api/services/external.go | 2 +- engine/api/services/http.go | 48 +- engine/api/ui.go | 2 +- engine/api/workflow.go | 2 +- engine/api/workflow/as_code.go | 2 +- engine/api/workflow/dao_coverage.go | 2 +- engine/api/workflow/dao_node_run.go | 2 +- .../workflow/dao_node_run_vulnerability.go | 2 +- engine/api/workflow/execute_node_run.go | 6 +- engine/api/workflow/hook.go | 10 +- engine/api/workflow/process_node.go | 2 +- engine/api/workflow/process_outgoinghook.go | 2 +- engine/api/workflow/repository.go | 4 +- engine/api/workflow/workflow_run_event.go | 4 +- engine/api/workflow/workflow_vcs.go | 2 +- engine/api/workflow_application.go | 2 +- engine/api/workflow_hook.go | 2 +- engine/api/workflow_queue.go | 2 +- engine/hooks/tasks.go | 7 +- engine/hooks/type_bitbucket_cloud.go | 310 ++++++++++++ ..._bitbucket.go => type_bitbucket_server.go} | 4 +- engine/hooks/webhook.go | 57 ++- engine/vcs/bitbucketcloud/bitbucketcloud.go | 53 ++ .../vcs/bitbucketcloud/bitbucketcloud_test.go | 174 +++++++ .../bitbucketcloud_test_helpers.go | 32 ++ engine/vcs/bitbucketcloud/client_branch.go | 105 ++++ engine/vcs/bitbucketcloud/client_commit.go | 166 +++++++ engine/vcs/bitbucketcloud/client_event.go | 39 ++ engine/vcs/bitbucketcloud/client_forks.go | 54 ++ engine/vcs/bitbucketcloud/client_hook.go | 148 ++++++ .../vcs/bitbucketcloud/client_pull_request.go | 165 +++++++ engine/vcs/bitbucketcloud/client_release.go | 18 + engine/vcs/bitbucketcloud/client_repos.go | 109 ++++ engine/vcs/bitbucketcloud/client_status.go | 179 +++++++ engine/vcs/bitbucketcloud/client_tag.go | 48 ++ engine/vcs/bitbucketcloud/client_user.go | 56 +++ engine/vcs/bitbucketcloud/error.go | 41 ++ engine/vcs/bitbucketcloud/http.go | 218 ++++++++ engine/vcs/bitbucketcloud/oauth_consumer.go | 136 +++++ engine/vcs/bitbucketcloud/status.go | 10 + engine/vcs/bitbucketcloud/types.go | 467 ++++++++++++++++++ .../bitbucketserver.go} | 7 +- .../bitbucketserver_test.go} | 4 +- .../client_branch.go | 2 +- .../client_branch_test.go | 2 +- .../client_commit.go | 2 +- .../client_commit_test.go | 2 +- .../client_event.go | 2 +- .../client_forks.go | 2 +- .../client_hook.go | 2 +- .../client_hook_test.go | 2 +- .../client_pull_request.go | 2 +- .../client_pull_request_test.go | 2 +- .../client_release.go | 2 +- .../client_repos.go | 2 +- .../client_repos_test.go | 2 +- .../client_status.go | 2 +- .../client_status_test.go | 2 +- .../client_tags.go | 2 +- .../client_user.go | 2 +- .../{bitbucket => bitbucketserver}/http.go | 2 +- .../oauth_consumer.go | 4 +- .../oauth_consumer_helper.go | 2 +- .../oauth_consumer_sign.go | 2 +- .../oauth_consumer_token.go | 2 +- .../{bitbucket => bitbucketserver}/types.go | 2 +- .../types_test.go | 2 +- engine/vcs/gerrit/gerrit.go | 6 + engine/vcs/gerrit/gerrit_consumer.go | 2 +- engine/vcs/github/github.go | 6 + engine/vcs/github/github_test.go | 4 +- engine/vcs/github/oauth_consumer.go | 4 +- engine/vcs/gitlab/gitlab.go | 23 +- engine/vcs/gitlab/gitlab_test.go | 4 +- engine/vcs/gitlab/oauth_consumer.go | 2 +- engine/vcs/types.go | 35 +- engine/vcs/vcs.go | 16 +- engine/vcs/vcs_auth.go | 46 +- engine/vcs/vcs_handlers.go | 216 ++++++-- engine/vcs/vcs_handlers_test.go | 38 +- sdk/vcs.go | 12 +- 94 files changed, 3201 insertions(+), 237 deletions(-) create mode 100644 docs/content/docs/integrations/bitbucketcloud.md create mode 100644 engine/hooks/type_bitbucket_cloud.go rename engine/hooks/{type_bitbucket.go => type_bitbucket_server.go} (90%) create mode 100644 engine/vcs/bitbucketcloud/bitbucketcloud.go create mode 100644 engine/vcs/bitbucketcloud/bitbucketcloud_test.go create mode 100644 engine/vcs/bitbucketcloud/bitbucketcloud_test_helpers.go create mode 100644 engine/vcs/bitbucketcloud/client_branch.go create mode 100644 engine/vcs/bitbucketcloud/client_commit.go create mode 100644 engine/vcs/bitbucketcloud/client_event.go create mode 100644 engine/vcs/bitbucketcloud/client_forks.go create mode 100644 engine/vcs/bitbucketcloud/client_hook.go create mode 100644 engine/vcs/bitbucketcloud/client_pull_request.go create mode 100644 engine/vcs/bitbucketcloud/client_release.go create mode 100644 engine/vcs/bitbucketcloud/client_repos.go create mode 100644 engine/vcs/bitbucketcloud/client_status.go create mode 100644 engine/vcs/bitbucketcloud/client_tag.go create mode 100644 engine/vcs/bitbucketcloud/client_user.go create mode 100644 engine/vcs/bitbucketcloud/error.go create mode 100644 engine/vcs/bitbucketcloud/http.go create mode 100644 engine/vcs/bitbucketcloud/oauth_consumer.go create mode 100644 engine/vcs/bitbucketcloud/status.go create mode 100644 engine/vcs/bitbucketcloud/types.go rename engine/vcs/{bitbucket/bitbucket.go => bitbucketserver/bitbucketserver.go} (93%) rename engine/vcs/{bitbucket/bitbucket_test.go => bitbucketserver/bitbucketserver_test.go} (98%) rename engine/vcs/{bitbucket => bitbucketserver}/client_branch.go (98%) rename engine/vcs/{bitbucket => bitbucketserver}/client_branch_test.go (95%) rename engine/vcs/{bitbucket => bitbucketserver}/client_commit.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/client_commit_test.go (95%) rename engine/vcs/{bitbucket => bitbucketserver}/client_event.go (97%) rename engine/vcs/{bitbucket => bitbucketserver}/client_forks.go (98%) rename engine/vcs/{bitbucket => bitbucketserver}/client_hook.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/client_hook_test.go (96%) rename engine/vcs/{bitbucket => bitbucketserver}/client_pull_request.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/client_pull_request_test.go (96%) rename engine/vcs/{bitbucket => bitbucketserver}/client_release.go (95%) rename engine/vcs/{bitbucket => bitbucketserver}/client_repos.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/client_repos_test.go (96%) rename engine/vcs/{bitbucket => bitbucketserver}/client_status.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/client_status_test.go (92%) rename engine/vcs/{bitbucket => bitbucketserver}/client_tags.go (97%) rename engine/vcs/{bitbucket => bitbucketserver}/client_user.go (95%) rename engine/vcs/{bitbucket => bitbucketserver}/http.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/oauth_consumer.go (95%) rename engine/vcs/{bitbucket => bitbucketserver}/oauth_consumer_helper.go (98%) rename engine/vcs/{bitbucket => bitbucketserver}/oauth_consumer_sign.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/oauth_consumer_token.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/types.go (99%) rename engine/vcs/{bitbucket => bitbucketserver}/types_test.go (97%) diff --git a/docs/content/docs/concepts/workflow/hooks/git-repo-webhook.md b/docs/content/docs/concepts/workflow/hooks/git-repo-webhook.md index b54d3d9ab5..ca2e1fbd87 100644 --- a/docs/content/docs/concepts/workflow/hooks/git-repo-webhook.md +++ b/docs/content/docs/concepts/workflow/hooks/git-repo-webhook.md @@ -11,6 +11,6 @@ You have to: * link an application to a git repository * add a Repository Webhook on the root pipeline, this pipeline have the application linked in the [context]({{< relref "/docs/concepts/workflow/pipeline-context.md" >}}) -GitHub / Bitbucket / GitLab are supported by CDS. +GitHub / Github Enterprise / Bitbucket Cloud / Bitbucket Server / GitLab are supported by CDS. > When you add a repository webhook, it will also automatically delete your runs which are linked to a deleted branch (24h after branch deletion). diff --git a/docs/content/docs/integrations/bitbucket.md b/docs/content/docs/integrations/bitbucket.md index fc3250d44c..423e854eb5 100644 --- a/docs/content/docs/integrations/bitbucket.md +++ b/docs/content/docs/integrations/bitbucket.md @@ -5,7 +5,7 @@ card: name: repository-manager --- -The Bitbucket Integration have to be configured on your CDS by a CDS Administrator. +The Bitbucket Server Integration have to be configured on your CDS by a CDS Administrator. This integration allows you to link a Git Repository hosted by your Bitbucket Server to a CDS Application. diff --git a/docs/content/docs/integrations/bitbucketcloud.md b/docs/content/docs/integrations/bitbucketcloud.md new file mode 100644 index 0000000000..230ac013ff --- /dev/null +++ b/docs/content/docs/integrations/bitbucketcloud.md @@ -0,0 +1,74 @@ +--- +title: Bitbucket Cloud +main_menu: true +card: + name: repository-manager +--- + +The Bitbucket Cloud Integration have to be configured on your CDS by a CDS Administrator. + +This integration allows you to link a Git Repository hosted by your Bitbucket Cloud +to a CDS Application. + +This integration enables some features: + + - [Git Repository Webhook]({{}}) + - Easy to use action [CheckoutApplication]({{}}) and [GitClone]({{}}) for advanced usage + - Send [build notifications](https://confluence.atlassian.com/bitbucket/check-build-status-in-a-pull-request-945541505.html) on your Pull-Requests and Commits on Bitbucket Cloud + +## How to configure Bitbucket Cloud integration + ++ Go on [Bitbucket.org](https://bitbucket.org/dashboard/overview) and log in. ++ From your avatar in the bottom left, click ***Bitbucket settings***. ++ Click OAuth from the left navigation. ++ Click the Add consumer button. ++ Bitbucket requests some informations: for the `name` you can simply write CDS, `description` is optional, `callback url` must be the URL of your CDS -> {CDS_UI_URL}/cdsapi/repositories_manager/oauth2/callback (if you are in development mode you have to omit /cdsapi and replace {CDS_UI_URL} with your API URL), `URL` is optional. ++ Click on Save and toggle the consumer name to see the generated `Key` and `Secret`. It correspond to `clientId` and `clientSecret` in the CDS config.toml file. + +### Complete CDS Configuration File + +Set value to `clientId` and `clientSecret`. + +```yaml + [vcs.servers] + [vcs.servers.bitbucketcloud] + + [vcs.servers.bitbucketcloud.bitbucketcloud] + + # Bitbucket Cloud OAuth Key + clientId = "XXXX" + + # Bitbucket Cloud OAuth Secret + clientSecret = "XXXX" + + # Does webhooks are supported by VCS Server + disableWebHooks = false + + # Does webhooks creation are supported by VCS Server + disableWebHooksCreation = false + + #proxyWebhook = "https://myproxy.com/" + + [vcs.servers.bitbucketcloud.bitbucketcloud.Status] + + # Set to true if you don't want CDS to push statuses on the VCS server + disable = false + + # Set to true if you don't want CDS to push CDS URL in statuses on the VCS server + showDetail = false +``` + +See how to generate **[Configuration File]({{}})** + +## Start the vcs µService + +```bash +$ engine start vcs + +# you can also start CDS api and vcs in the same process: +$ engine start api vcs +``` + +## Vcs events + +For now, CDS supports push events. CDS uses this push event to remove existing runs for deleted branches (24h after branch deletion). diff --git a/engine/api/admin.go b/engine/api/admin.go index 75bfc0cc48..d47349d43e 100644 --- a/engine/api/admin.go +++ b/engine/api/admin.go @@ -134,7 +134,7 @@ func selectDeleteAdminServiceCallHandler(api *API, method string) service.Handle } query := r.FormValue("query") - btes, code, err := services.DoRequest(ctx, srvs, method, query, nil) + btes, _, code, err := services.DoRequest(ctx, srvs, method, query, nil) if err != nil { return sdk.NewError(sdk.Error{ Status: code, @@ -165,7 +165,7 @@ func putPostAdminServiceCallHandler(api *API, method string) service.Handler { } defer r.Body.Close() - btes, code, err := services.DoRequest(ctx, srvs, method, query, body) + btes, _, code, err := services.DoRequest(ctx, srvs, method, query, body) if err != nil { return sdk.NewError(sdk.Error{ Status: code, diff --git a/engine/api/application.go b/engine/api/application.go index 5322da6381..24b0088e2a 100644 --- a/engine/api/application.go +++ b/engine/api/application.go @@ -179,7 +179,7 @@ func (api *API) getApplicationVCSInfosHandler() service.Handler { } vcsServer := repositoriesmanager.GetProjectVCSServer(proj, app.VCSServer) - client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projectKey, vcsServer) if erra != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "getApplicationVCSInfosHandler> Cannot get client got %s %s : %s", projectKey, app.VCSServer, erra) } diff --git a/engine/api/ascode.go b/engine/api/ascode.go index 82c0c62270..8bb47af344 100644 --- a/engine/api/ascode.go +++ b/engine/api/ascode.go @@ -48,7 +48,7 @@ func (api *API) postImportAsCodeHandler() service.Handler { } vcsServer := repositoriesmanager.GetProjectVCSServer(p, ope.VCSServer) - client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, p.Key, vcsServer) if erra != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "postImportAsCodeHandler> Cannot get client for %s %s : %s", key, ope.VCSServer, erra) } @@ -153,7 +153,7 @@ func (api *API) postPerformImportAsCodeHandler() service.Handler { // Grant CDS as a repository collaborator // TODO for this moment, this step is not mandatory. If it's failed, continue the ascode process vcsServer := repositoriesmanager.GetProjectVCSServer(proj, ope.VCSServer) - client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, proj.Key, vcsServer) if erra != nil { log.Error("postPerformImportAsCodeHandler> Cannot get client for %s %s : %s", proj.Key, ope.VCSServer, erra) } else { diff --git a/engine/api/event/elasticsearch.go b/engine/api/event/elasticsearch.go index 1e562403ee..e89d2b9aea 100644 --- a/engine/api/event/elasticsearch.go +++ b/engine/api/event/elasticsearch.go @@ -41,7 +41,7 @@ func PushInElasticSearch(c context.Context, db gorp.SqlExecutor, store cache.Sto continue } e.Payload = nil - code, errD := services.DoJSONRequest(context.Background(), esServices, "POST", "/events", e, nil) + _, code, errD := services.DoJSONRequest(context.Background(), esServices, "POST", "/events", e, nil) if code >= 400 || errD != nil { log.Error("PushInElasticSearch> Unable to send event %s to elasticsearch [%d]: %v", e.EventType, code, errD) continue @@ -58,7 +58,7 @@ func GetEvents(db gorp.SqlExecutor, store cache.Store, filters sdk.EventFilter) } var esEvents []elastic.SearchHit - if _, err := services.DoJSONRequest(context.Background(), srvs, "GET", "/events", filters, &esEvents); err != nil { + if _, _, err := services.DoJSONRequest(context.Background(), srvs, "GET", "/events", filters, &esEvents); err != nil { return nil, sdk.WrapError(err, "Unable to get events") } diff --git a/engine/api/hook.go b/engine/api/hook.go index f579c852c7..1b03f9382d 100644 --- a/engine/api/hook.go +++ b/engine/api/hook.go @@ -49,7 +49,7 @@ func (api *API) getHookPollingVCSEvents() service.Handler { //get the client for the repositories manager vcsServer := repositoriesmanager.GetProjectVCSServer(proj, vcsServerParam) - client, errR := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, errR := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, proj.Key, vcsServer) if errR != nil { return sdk.WrapError(errR, "getHookPollingVCSEvents> Unable to get client for %s %s", proj.Key, vcsServerParam) } diff --git a/engine/api/metrics/elasticsearch.go b/engine/api/metrics/elasticsearch.go index c52faa8be0..2837319107 100644 --- a/engine/api/metrics/elasticsearch.go +++ b/engine/api/metrics/elasticsearch.go @@ -40,7 +40,7 @@ func Init(c context.Context, DBFunc func() *gorp.DbMap) { continue } - code, errD := services.DoJSONRequest(context.Background(), esServices, "POST", "/metrics", e, nil) + _, code, errD := services.DoJSONRequest(context.Background(), esServices, "POST", "/metrics", e, nil) if code >= 400 || errD != nil { log.Error("metrics.pushInElasticSearch> Unable to send metrics to elasticsearch [%d]: %v", code, errD) continue @@ -63,7 +63,7 @@ func GetMetrics(db gorp.SqlExecutor, key string, appID int64, metricName string) } var esMetrics []elastic.SearchHit - if _, err := services.DoJSONRequest(context.Background(), srvs, "GET", "/metrics", metricsRequest, &esMetrics); err != nil { + if _, _, err := services.DoJSONRequest(context.Background(), srvs, "GET", "/metrics", metricsRequest, &esMetrics); err != nil { return nil, sdk.WrapError(err, "Unable to get metrics") } diff --git a/engine/api/repositories_manager.go b/engine/api/repositories_manager.go index 2735ca20c8..e476b96ad5 100644 --- a/engine/api/repositories_manager.go +++ b/engine/api/repositories_manager.go @@ -79,9 +79,9 @@ func (api *API) repositoriesManagerAuthorizeHandler() service.Handler { "project_key": proj.Key, "last_modified": strconv.FormatInt(time.Now().Unix(), 10), "repositories_manager": rmName, - "url": url, - "request_token": token, - "username": deprecatedGetUser(ctx).Username, + "url": url, + "request_token": token, + "username": deprecatedGetUser(ctx).Username, } if token != "" { @@ -153,8 +153,9 @@ func (api *API) repositoriesManagerOAuthCallbackHandler() service.Handler { Name: rmName, Username: username, Data: map[string]string{ - "token": token, - "secret": secret, + "token": token, + "secret": secret, + "created": fmt.Sprintf("%d", time.Now().Unix()), }, } @@ -214,8 +215,9 @@ func (api *API) repositoriesManagerAuthorizeBasicHandler() service.Handler { Name: rmName, Username: deprecatedGetUser(ctx).Username, Data: map[string]string{ - "token": username, - "secret": secret, + "token": username, + "secret": secret, + "created": fmt.Sprintf("%d", time.Now().Unix()), }, } @@ -223,7 +225,7 @@ func (api *API) repositoriesManagerAuthorizeBasicHandler() service.Handler { return sdk.WrapError(err, "unable to set repository manager data for project %s", projectKey) } - client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, vcsServerForProject) + client, err := repositoriesmanager.AuthorizedClient(ctx, tx, api.Cache, proj.Key, vcsServerForProject) if err != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "cannot get client for project %s: %v", proj.Key, err) } @@ -292,8 +294,9 @@ func (api *API) repositoriesManagerAuthorizeCallbackHandler() service.Handler { Name: rmName, Username: deprecatedGetUser(ctx).Username, Data: map[string]string{ - "token": token, - "secret": secret, + "token": token, + "secret": secret, + "created": fmt.Sprintf("%d", time.Now().Unix()), }, } @@ -381,7 +384,7 @@ func (api *API) getReposFromRepositoriesManagerHandler() service.Handler { log.Debug("getReposFromRepositoriesManagerHandler> Loading repo for %s; ok", vcsServer.Name) var errAuthClient error - client, errAuthClient := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, errAuthClient := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projectKey, vcsServer) if errAuthClient != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "getReposFromRepositoriesManagerHandler> Cannot get client got %s %s: %v", projectKey, rmName, errAuthClient) } @@ -427,7 +430,7 @@ func (api *API) getRepoFromRepositoriesManagerHandler() service.Handler { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "getReposFromRepositoriesManagerHandler> Cannot get client got %s %s", projectKey, rmName) } - client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projectKey, vcsServer) if err != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "getRepoFromRepositoriesManagerHandler> Cannot get client got %s %s : %s", projectKey, rmName, err) } @@ -464,7 +467,7 @@ func (api *API) attachRepositoriesManagerHandler() service.Handler { } //Get an authorized Client - client, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, rm) + client, err := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, projectKey, rm) if err != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "attachRepositoriesManager> Cannot get client got %s %s : %s", projectKey, rmName, err) } diff --git a/engine/api/repositoriesmanager/dao.go b/engine/api/repositoriesmanager/dao.go index a8113f5526..1e16588e7e 100644 --- a/engine/api/repositoriesmanager/dao.go +++ b/engine/api/repositoriesmanager/dao.go @@ -44,6 +44,29 @@ func InsertForProject(db gorp.SqlExecutor, proj *sdk.Project, vcsServer *sdk.Pro return nil } +//UpdateForProject update the link of project with a repository manager +func UpdateForProject(db gorp.SqlExecutor, proj *sdk.Project, vcsServers []sdk.ProjectVCSServer) error { + b1, err := yaml.Marshal(vcsServers) + if err != nil { + return sdk.WithStack(err) + } + + log.Debug("repositoriesmanager.UpdateForProject> %s %s", proj.Key, string(b1)) + + encryptedVCSServerStr, err := secret.Encrypt(b1) + if err != nil { + return err + } + + if _, err := db.Exec("update project set vcs_servers = $2 where projectkey = $1", proj.Key, encryptedVCSServerStr); err != nil { + return sdk.WithStack(err) + } + + proj.VCSServers = vcsServers + + return nil +} + //DeleteForProject unlink a project with a repository manager func DeleteForProject(db gorp.SqlExecutor, proj *sdk.Project, vcsServer *sdk.ProjectVCSServer) error { servers, err := LoadAllForProject(db, proj.Key) diff --git a/engine/api/repositoriesmanager/events.go b/engine/api/repositoriesmanager/events.go index f963227e73..106fa4a07c 100644 --- a/engine/api/repositoriesmanager/events.go +++ b/engine/api/repositoriesmanager/events.go @@ -65,7 +65,7 @@ func processEvent(ctx context.Context, db *gorp.DbMap, event sdk.Event, store ca return fmt.Errorf("repositoriesmanager>processEvent> AuthorizedClient (%s, %s) > err:%s", event.ProjectKey, eventWNR.RepositoryManagerName, err) } - c, errC = AuthorizedClient(ctx, db, store, vcsServer) + c, errC = AuthorizedClient(ctx, db, store, event.ProjectKey, vcsServer) if errC != nil { return fmt.Errorf("repositoriesmanager>processEvent> AuthorizedClient (%s, %s) > err:%s", event.ProjectKey, eventWNR.RepositoryManagerName, errC) } diff --git a/engine/api/repositoriesmanager/repositories_manager.go b/engine/api/repositoriesmanager/repositories_manager.go index 47ab1fc9d9..ad6f20df0c 100644 --- a/engine/api/repositoriesmanager/repositories_manager.go +++ b/engine/api/repositoriesmanager/repositories_manager.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "sync" "time" @@ -28,7 +29,7 @@ func LoadByName(ctx context.Context, db gorp.SqlExecutor, vcsName string) (sdk.V if err != nil { return vcsServer, sdk.WrapError(err, "Unable to load services") } - if _, err := services.DoJSONRequest(ctx, srvs, "GET", fmt.Sprintf("/vcs/%s", vcsName), nil, &vcsServer); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, "GET", fmt.Sprintf("/vcs/%s", vcsName), nil, &vcsServer); err != nil { return vcsServer, sdk.WithStack(err) } return vcsServer, nil @@ -42,7 +43,7 @@ func LoadAll(ctx context.Context, db *gorp.DbMap, store cache.Store) (map[string } vcsServers := make(map[string]sdk.VCSConfiguration) - if _, err := services.DoJSONRequest(ctx, srvs, "GET", "/vcs", nil, &vcsServers); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, "GET", "/vcs", nil, &vcsServers); err != nil { return nil, sdk.WithStack(err) } return vcsServers, nil @@ -56,11 +57,14 @@ type vcsConsumer struct { } type vcsClient struct { - name string - token string - secret string - srvs []sdk.Service - cache *gocache.Cache + name string + token string + secret string + projectKey string + created int64 //Timestamp .Unix() of creation + srvs []sdk.Service + cache *gocache.Cache + db gorp.SqlExecutor } func (c *vcsClient) Cache() *gocache.Cache { @@ -98,7 +102,7 @@ func (c *vcsConsumer) AuthorizeRedirect(ctx context.Context) (string, string, er res := map[string]string{} path := fmt.Sprintf("/vcs/%s/authorize", c.name) log.Info("Performing request on %s", path) - if _, err := services.DoJSONRequest(ctx, srv, "GET", path, nil, &res); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srv, "GET", path, nil, &res); err != nil { return "", "", sdk.WithStack(err) } @@ -118,14 +122,14 @@ func (c *vcsConsumer) AuthorizeToken(ctx context.Context, token string, secret s res := map[string]string{} path := fmt.Sprintf("/vcs/%s/authorize", c.name) - if _, err := services.DoJSONRequest(ctx, srv, "POST", path, body, &res); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srv, "POST", path, body, &res); err != nil { return "", "", sdk.WithStack(err) } return res["token"], res["secret"], nil } -func (c *vcsConsumer) GetAuthorizedClient(ctx context.Context, token string, secret string) (sdk.VCSAuthorizedClient, error) { +func (c *vcsConsumer) GetAuthorizedClient(ctx context.Context, token, secret string, created int64) (sdk.VCSAuthorizedClient, error) { s := GetProjectVCSServer(c.proj, c.name) if s == nil { return nil, sdk.ErrNoReposManagerClientAuth @@ -137,11 +141,14 @@ func (c *vcsConsumer) GetAuthorizedClient(ctx context.Context, token string, sec } return &vcsClient{ - name: c.name, - token: token, - secret: secret, - srvs: srvs, - cache: gocache.New(5*time.Second, 60*time.Second), + name: c.name, + token: token, + projectKey: c.proj.Key, + created: created, + secret: secret, + srvs: srvs, + cache: gocache.New(5*time.Second, 60*time.Second), + db: c.dbFunc(), }, nil } @@ -165,6 +172,17 @@ func (c *localAuthorizedClientCache) Set(repo *sdk.ProjectVCSServer, vcs sdk.VCS c.cache[hash] = vcs } +func (c *localAuthorizedClientCache) Delete(repo *sdk.ProjectVCSServer) { + c.mutex.Lock() + defer c.mutex.Unlock() + + hash := repo.Hash() + if hash == 0 { + return + } + delete(c.cache, hash) +} + func (c *localAuthorizedClientCache) Get(repo *sdk.ProjectVCSServer) (sdk.VCSAuthorizedClient, bool) { c.mutex.RLock() defer c.mutex.RUnlock() @@ -174,7 +192,7 @@ func (c *localAuthorizedClientCache) Get(repo *sdk.ProjectVCSServer) (sdk.VCSAut } //AuthorizedClient returns an implementation of AuthorizedClient wrapping calls to vcs uService -func AuthorizedClient(ctx context.Context, db gorp.SqlExecutor, store cache.Store, repo *sdk.ProjectVCSServer) (sdk.VCSAuthorizedClient, error) { +func AuthorizedClient(ctx context.Context, db gorp.SqlExecutor, store cache.Store, projectKey string, repo *sdk.ProjectVCSServer) (sdk.VCSAuthorizedClient, error) { if repo == nil { return nil, sdk.ErrNoReposManagerClientAuth } @@ -188,21 +206,35 @@ func AuthorizedClient(ctx context.Context, db gorp.SqlExecutor, store cache.Stor if err != nil { return nil, sdk.WithStack(err) } + var created int64 + + if _, ok := repo.Data["created"]; ok { + created, err = strconv.ParseInt(repo.Data["created"], 10, 64) + if err != nil { + return nil, sdk.WithStack(err) + } + } vcs = &vcsClient{ - name: repo.Name, - token: repo.Data["token"], - secret: repo.Data["secret"], - srvs: srvs, + name: repo.Name, + token: repo.Data["token"], + secret: repo.Data["secret"], + created: created, + srvs: srvs, + db: db, + projectKey: projectKey, } local.Set(repo, vcs) return vcs, nil } func (c *vcsClient) doJSONRequest(ctx context.Context, method, path string, in interface{}, out interface{}) (int, error) { - code, err := services.DoJSONRequest(ctx, c.srvs, method, path, in, out, func(req *http.Request) { - req.Header.Set("X-CDS-ACCESS-TOKEN", base64.StdEncoding.EncodeToString([]byte(c.token))) - req.Header.Set("X-CDS-ACCESS-TOKEN-SECRET", base64.StdEncoding.EncodeToString([]byte(c.secret))) + headers, code, err := services.DoJSONRequest(ctx, c.srvs, method, path, in, out, func(req *http.Request) { + req.Header.Set(sdk.HeaderXAccessToken, base64.StdEncoding.EncodeToString([]byte(c.token))) + req.Header.Set(sdk.HeaderXAccessTokenSecret, base64.StdEncoding.EncodeToString([]byte(c.secret))) + if c.created != 0 { + req.Header.Set(sdk.HeaderXAccessTokenCreated, base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%d", c.created)))) + } }) if code >= 400 { @@ -220,16 +252,47 @@ func (c *vcsClient) doJSONRequest(ctx context.Context, method, path string, in i } } + if err != nil { + return code, sdk.WithStack(err) + } + err = c.checkAccessToken(ctx, headers) + return code, sdk.WithStack(err) } func (c *vcsClient) postMultipart(ctx context.Context, path string, fileContent []byte, out interface{}) (int, error) { return services.PostMultipart(ctx, c.srvs, "POST", path, fileContent, out, func(req *http.Request) { - req.Header.Set("X-CDS-ACCESS-TOKEN", base64.StdEncoding.EncodeToString([]byte(c.token))) - req.Header.Set("X-CDS-ACCESS-TOKEN-SECRET", base64.StdEncoding.EncodeToString([]byte(c.secret))) + req.Header.Set(sdk.HeaderXAccessToken, base64.StdEncoding.EncodeToString([]byte(c.token))) + req.Header.Set(sdk.HeaderXAccessTokenSecret, base64.StdEncoding.EncodeToString([]byte(c.secret))) }) } +func (c *vcsClient) checkAccessToken(ctx context.Context, header http.Header) error { + if newAccessToken := header.Get(sdk.HeaderXAccessToken); newAccessToken != "" { + c.token = newAccessToken + + vcsservers, err := LoadAllForProject(c.db, c.projectKey) + if err != nil { + return sdk.WrapError(err, "cannot load vcs servers for project %s", c.projectKey) + } + + for i := range vcsservers { + if vcsservers[i].Name == c.name { + vcsservers[i].Data["token"] = c.token + vcsservers[i].Data["created"] = fmt.Sprintf("%d", time.Now().Unix()) + local.Delete(&vcsservers[i]) + break + } + } + + if err := UpdateForProject(c.db, &sdk.Project{Key: c.projectKey}, vcsservers); err != nil { + return sdk.WrapError(err, "cannot update vcsservers for project %s", c.projectKey) + } + } + + return nil +} + func (c *vcsClient) Repos(ctx context.Context) ([]sdk.VCSRepo, error) { items, has := c.Cache().Get("/repos") if has { @@ -543,6 +606,10 @@ func (c *vcsClient) GrantWritePermission(ctx context.Context, repo string) error return nil } +func (c *vcsClient) GetAccessToken(_ context.Context) string { + return "" +} + // WebhooksInfos is a set of info about webhooks type WebhooksInfos struct { WebhooksSupported bool `json:"webhooks_supported"` diff --git a/engine/api/services/external.go b/engine/api/services/external.go index 8ca05a52fb..a4215fdfe6 100644 --- a/engine/api/services/external.go +++ b/engine/api/services/external.go @@ -70,7 +70,7 @@ func ping(db gorp.SqlExecutor, s sdk.ExternalService) error { } else { pingURL = fmt.Sprintf("%s:%s", s.HealthURL, s.HealthPort) } - _, code, err := doRequest(context.Background(), pingURL, "", "GET", s.HealthPath, nil) + _, _, code, err := doRequest(context.Background(), pingURL, "", "GET", s.HealthPath, nil) if err != nil || code >= 400 { mon.Lines[0].Status = sdk.MonitoringStatusWarn mon.Lines[0].Value = "Health: KO" diff --git a/engine/api/services/http.go b/engine/api/services/http.go index baa10787b1..c0651dcf78 100644 --- a/engine/api/services/http.go +++ b/engine/api/services/http.go @@ -67,7 +67,7 @@ func DoMultiPartRequest(ctx context.Context, srvs []sdk.Service, method, path st attempt++ for i := range srvs { srv := &srvs[i] - res, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, method, path, body.Bytes(), mods...) + res, _, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, method, path, body.Bytes(), mods...) if err != nil { return code, sdk.WrapError(err, "Unable to perform request on service %s (%s)", srv.Name, srv.Type) } @@ -90,7 +90,7 @@ func DoMultiPartRequest(ctx context.Context, srvs []sdk.Service, method, path st } // DoJSONRequest performs an http request on a service -func DoJSONRequest(ctx context.Context, srvs []sdk.Service, method, path string, in interface{}, out interface{}, mods ...sdk.RequestModifier) (int, error) { +func DoJSONRequest(ctx context.Context, srvs []sdk.Service, method, path string, in interface{}, out interface{}, mods ...sdk.RequestModifier) (http.Header, int, error) { var lastErr error var lastCode int var attempt int @@ -98,9 +98,9 @@ func DoJSONRequest(ctx context.Context, srvs []sdk.Service, method, path string, attempt++ for i := range srvs { srv := &srvs[i] - code, err := doJSONRequest(ctx, srv, method, path, in, out, mods...) + headers, code, err := doJSONRequest(ctx, srv, method, path, in, out, mods...) if err == nil { - return code, nil + return headers, code, nil } lastErr = err lastCode = code @@ -109,34 +109,34 @@ func DoJSONRequest(ctx context.Context, srvs []sdk.Service, method, path string, break } } - return lastCode, lastErr + return nil, lastCode, lastErr } // DoJSONRequest performs an http request on service -func doJSONRequest(ctx context.Context, srv *sdk.Service, method, path string, in interface{}, out interface{}, mods ...sdk.RequestModifier) (int, error) { +func doJSONRequest(ctx context.Context, srv *sdk.Service, method, path string, in interface{}, out interface{}, mods ...sdk.RequestModifier) (http.Header, int, error) { var b = []byte{} var err error if in != nil { b, err = json.Marshal(in) if err != nil { - return 0, sdk.WrapError(err, "Unable to marshal input") + return nil, 0, sdk.WrapError(err, "Unable to marshal input") } } mods = append(mods, sdk.SetHeader("Content-Type", "application/json")) - res, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, method, path, b, mods...) + res, headers, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, method, path, b, mods...) if err != nil { - return code, sdk.ErrorWithFallback(err, sdk.ErrUnknownError, "Unable to perform request on service %s (%s)", srv.Name, srv.Type) + return headers, code, sdk.ErrorWithFallback(err, sdk.ErrUnknownError, "Unable to perform request on service %s (%s)", srv.Name, srv.Type) } if out != nil { if err := json.Unmarshal(res, out); err != nil { - return code, sdk.WrapError(err, "Unable to marshal output") + return headers, code, sdk.WrapError(err, "Unable to marshal output") } } - return code, nil + return headers, code, nil } // PostMultipart post a file content through multipart upload @@ -161,7 +161,7 @@ func PostMultipart(ctx context.Context, srvs []sdk.Service, path string, filenam attempt++ for i := range srvs { srv := &srvs[i] - res, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, "POST", path, body.Bytes(), mods...) + res, _, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, "POST", path, body.Bytes(), mods...) lastCode = code lastErr = err @@ -183,7 +183,7 @@ func PostMultipart(ctx context.Context, srvs []sdk.Service, path string, filenam } // DoRequest performs an http request on a service -func DoRequest(ctx context.Context, srvs []sdk.Service, method, path string, args []byte, mods ...sdk.RequestModifier) ([]byte, int, error) { +func DoRequest(ctx context.Context, srvs []sdk.Service, method, path string, args []byte, mods ...sdk.RequestModifier) ([]byte, http.Header, int, error) { var lastErr error var lastCode int var attempt int @@ -191,9 +191,9 @@ func DoRequest(ctx context.Context, srvs []sdk.Service, method, path string, arg attempt++ for i := range srvs { srv := &srvs[i] - btes, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, method, path, args, mods...) + btes, headers, code, err := doRequest(ctx, srv.HTTPURL, srv.Hash, method, path, args, mods...) if err == nil { - return btes, code, nil + return btes, headers, code, nil } lastErr = err lastCode = code @@ -202,11 +202,11 @@ func DoRequest(ctx context.Context, srvs []sdk.Service, method, path string, arg break } } - return nil, lastCode, lastErr + return nil, nil, lastCode, lastErr } // doRequest performs an http request on service -func doRequest(ctx context.Context, httpURL string, hash string, method, path string, args []byte, mods ...sdk.RequestModifier) ([]byte, int, error) { +func doRequest(ctx context.Context, httpURL string, hash string, method, path string, args []byte, mods ...sdk.RequestModifier) ([]byte, http.Header, int, error) { if HTTPClient == nil { HTTPClient = &http.Client{ Timeout: 60 * time.Second, @@ -215,7 +215,7 @@ func doRequest(ctx context.Context, httpURL string, hash string, method, path st callURL, err := url.ParseRequestURI(httpURL + path) if err != nil { - return nil, 0, err + return nil, nil, 0, err } var requestError error @@ -226,7 +226,7 @@ func doRequest(ctx context.Context, httpURL string, hash string, method, path st req, requestError = http.NewRequest(method, callURL.String(), nil) } if requestError != nil { - return nil, 0, requestError + return nil, nil, 0, requestError } req = req.WithContext(ctx) @@ -255,27 +255,27 @@ func doRequest(ctx context.Context, httpURL string, hash string, method, path st //Do the request resp, errDo := HTTPClient.Do(req) if errDo != nil { - return nil, 0, sdk.WrapError(errDo, "services.DoRequest> Request failed") + return nil, nil, 0, sdk.WrapError(errDo, "services.DoRequest> Request failed") } defer resp.Body.Close() // Read the body body, errBody := ioutil.ReadAll(resp.Body) if errBody != nil { - return nil, resp.StatusCode, sdk.WrapError(errBody, "services.DoRequest> Unable to read body") + return nil, resp.Header, resp.StatusCode, sdk.WrapError(errBody, "services.DoRequest> Unable to read body") } log.Debug("services.DoRequest> response code:%d body:%s", resp.StatusCode, string(body)) // if everything is fine, return body if resp.StatusCode < 400 { - return body, resp.StatusCode, nil + return body, resp.Header, resp.StatusCode, nil } // Try to catch the CDS Error if cdserr := sdk.DecodeError(body); cdserr != nil { - return nil, resp.StatusCode, cdserr + return nil, resp.Header, resp.StatusCode, cdserr } - return nil, resp.StatusCode, fmt.Errorf("Request Failed") + return nil, resp.Header, resp.StatusCode, fmt.Errorf("Request Failed") } diff --git a/engine/api/ui.go b/engine/api/ui.go index f3ec49fefa..312ffb093a 100644 --- a/engine/api/ui.go +++ b/engine/api/ui.go @@ -85,7 +85,7 @@ func (api *API) getApplicationOverviewHandler() service.Handler { // GET VCS URL // Get vcs info to known if we are on the default branch or not if projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, app.VCSServer); projectVCSServer != nil { - client, erra := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, projectVCSServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, db, api.Cache, p.Key, projectVCSServer) if erra != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "getApplicationOverviewHandler> Cannot get repo client %s: %v", app.VCSServer, erra) } diff --git a/engine/api/workflow.go b/engine/api/workflow.go index b84e1c81db..2d33ca3f2d 100644 --- a/engine/api/workflow.go +++ b/engine/api/workflow.go @@ -659,7 +659,7 @@ func (api *API) getWorkflowHookHandler() service.Handler { path := fmt.Sprintf("/task/%s/execution", uuid) task := sdk.Task{} - if _, err := services.DoJSONRequest(ctx, srvs, "GET", path, nil, &task); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, "GET", path, nil, &task); err != nil { return sdk.WrapError(err, "Unable to get hook %s task and executions", uuid) } diff --git a/engine/api/workflow/as_code.go b/engine/api/workflow/as_code.go index 1450f2e082..ba1222cd88 100644 --- a/engine/api/workflow/as_code.go +++ b/engine/api/workflow/as_code.go @@ -196,7 +196,7 @@ func UpdateWorkflowAsCodeResult(ctx context.Context, db *gorp.DbMap, store cache ope.Error = "No vcsServer found" return } - client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, vcsServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, vcsServer) if errclient != nil { log.Error("postWorkflowAsCodeHandler> unable to create repositories manager client: %v", errclient) ope.Status = sdk.OperationStatusError diff --git a/engine/api/workflow/dao_coverage.go b/engine/api/workflow/dao_coverage.go index ba4a2a98f1..dc0d0c1541 100644 --- a/engine/api/workflow/dao_coverage.go +++ b/engine/api/workflow/dao_coverage.go @@ -172,7 +172,7 @@ func ComputeLatestDefaultBranchReport(ctx context.Context, db gorp.SqlExecutor, // Get report latest report on previous branch var defaultBranch string projectVCSServer := repositoriesmanager.GetProjectVCSServer(proj, wnr.VCSServer) - client, erra := repositoriesmanager.AuthorizedClient(ctx, db, cache, projectVCSServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, db, cache, proj.Key, projectVCSServer) if erra != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "ComputeLatestDefaultBranchReport> Cannot get repo client %s : %s", wnr.VCSServer, erra) } diff --git a/engine/api/workflow/dao_node_run.go b/engine/api/workflow/dao_node_run.go index 57f41822ef..c1c5e9a8b2 100644 --- a/engine/api/workflow/dao_node_run.go +++ b/engine/api/workflow/dao_node_run.go @@ -557,7 +557,7 @@ func GetNodeRunBuildCommits(ctx context.Context, db gorp.SqlExecutor, store cach res := []sdk.VCSCommit{} //Get the RepositoriesManager Client - client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, vcsServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, vcsServer) if errclient != nil { return nil, cur, sdk.WrapError(errclient, "GetNodeRunBuildCommits> Cannot get client") } diff --git a/engine/api/workflow/dao_node_run_vulnerability.go b/engine/api/workflow/dao_node_run_vulnerability.go index 7f708f1499..e1aaeef9c3 100644 --- a/engine/api/workflow/dao_node_run_vulnerability.go +++ b/engine/api/workflow/dao_node_run_vulnerability.go @@ -23,7 +23,7 @@ func HandleVulnerabilityReport(ctx context.Context, db gorp.SqlExecutor, cache c if nr.VCSServer != "" { // Get vcs info to known if we are on the default branch or not projectVCSServer := repositoriesmanager.GetProjectVCSServer(proj, nr.VCSServer) - client, erra := repositoriesmanager.AuthorizedClient(ctx, db, cache, projectVCSServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, db, cache, proj.Key, projectVCSServer) if erra != nil { return sdk.WrapError(sdk.ErrNoReposManagerClientAuth, "HandleVulnerabilityReport> Cannot get repo client %s : %v", nr.VCSServer, erra) } diff --git a/engine/api/workflow/execute_node_run.go b/engine/api/workflow/execute_node_run.go index e26d3ee598..a4dabaa6f6 100644 --- a/engine/api/workflow/execute_node_run.go +++ b/engine/api/workflow/execute_node_run.go @@ -742,7 +742,7 @@ func stopWorkflowNodeOutGoingHook(ctx context.Context, dbFunc func() *gorp.DbMap if nodeRun.HookExecutionID != "" { path := fmt.Sprintf("/task/%s/execution/%d/stop", nodeRun.HookExecutionID, nodeRun.HookExecutionTimeStamp) - if _, err := services.DoJSONRequest(ctx, srvs, "POST", path, nil, nil); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, "POST", path, nil, nil); err != nil { return fmt.Errorf("unable to stop task execution: %v", err) } } @@ -907,7 +907,7 @@ func (i vcsInfos) String() string { return fmt.Sprintf("%s:%s:%s:%s", i.Server, i.Repository, i.Branch, i.Hash) } -func getVCSInfos(ctx context.Context, db gorp.SqlExecutor, store cache.Store, vcsServer *sdk.ProjectVCSServer, gitValues map[string]string, applicationName, applicationVCSServer, applicationRepositoryFullname string) (*vcsInfos, error) { +func getVCSInfos(ctx context.Context, db gorp.SqlExecutor, store cache.Store, projectKey string, vcsServer *sdk.ProjectVCSServer, gitValues map[string]string, applicationName, applicationVCSServer, applicationRepositoryFullname string) (*vcsInfos, error) { var vcsInfos vcsInfos vcsInfos.Repository = gitValues[tagGitRepository] vcsInfos.Branch = gitValues[tagGitBranch] @@ -942,7 +942,7 @@ func getVCSInfos(ctx context.Context, db gorp.SqlExecutor, store cache.Store, vc } //Get the RepositoriesManager Client - client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, vcsServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, projectKey, vcsServer) if errclient != nil { return nil, sdk.WrapError(errclient, "cannot get client") } diff --git a/engine/api/workflow/hook.go b/engine/api/workflow/hook.go index f5795ba994..80933c3745 100644 --- a/engine/api/workflow/hook.go +++ b/engine/api/workflow/hook.go @@ -128,7 +128,7 @@ func HookRegistration(ctx context.Context, db gorp.SqlExecutor, store cache.Stor } // Create hook on µservice - code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/task/bulk", hookToUpdate, &hookToUpdate) + _, code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/task/bulk", hookToUpdate, &hookToUpdate) if errHooks != nil || code >= 400 { return sdk.WrapError(errHooks, "HookRegistration> Unable to create hooks [%d]", code) } @@ -168,7 +168,7 @@ func DeleteHookConfiguration(ctx context.Context, db gorp.SqlExecutor, store cac // Call VCS to know if repository allows webhook and get the configuration fields projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value) if projectVCSServer != nil { - client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, projectVCSServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, projectVCSServer) if errclient != nil { return sdk.WrapError(errclient, "deleteHookConfiguration> Cannot get vcs client") } @@ -197,7 +197,7 @@ func DeleteHookConfiguration(ctx context.Context, db gorp.SqlExecutor, store cac if err != nil { return sdk.WrapError(err, "Unable to get services dao") } - code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodDelete, "/task/bulk", hookToDelete, nil) + _, code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodDelete, "/task/bulk", hookToDelete, nil) if errHooks != nil || code >= 400 { // if we return an error, transaction will be rollbacked => hook will in database be not anymore on gitlab/bitbucket/github. // so, it's just a warn log @@ -216,7 +216,7 @@ func createVCSConfiguration(ctx context.Context, db gorp.SqlExecutor, store cach return nil } - client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, projectVCSServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, projectVCSServer) if errclient != nil { return sdk.WrapError(errclient, "createVCSConfiguration> Cannot get vcs client") } @@ -326,7 +326,7 @@ func DefaultPayload(ctx context.Context, db gorp.SqlExecutor, store cache.Store, defaultBranch := "master" projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, wf.Root.Context.Application.VCSServer) if projectVCSServer != nil { - client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, projectVCSServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, projectVCSServer) if errclient != nil { return wf.Root.Context.DefaultPayload, sdk.WrapError(errclient, "DefaultPayload> Cannot get authorized client") } diff --git a/engine/api/workflow/process_node.go b/engine/api/workflow/process_node.go index a7dfa1cbc5..8f6f5ab0ca 100644 --- a/engine/api/workflow/process_node.go +++ b/engine/api/workflow/process_node.go @@ -233,7 +233,7 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr var errVcs error if needVCSInfo { vcsServer := repositoriesmanager.GetProjectVCSServer(proj, app.VCSServer) - vcsInf, errVcs = getVCSInfos(ctx, db, store, vcsServer, currentJobGitValues, app.Name, app.VCSServer, app.RepositoryFullname) + vcsInf, errVcs = getVCSInfos(ctx, db, store, proj.Key, vcsServer, currentJobGitValues, app.Name, app.VCSServer, app.RepositoryFullname) if errVcs != nil { AddWorkflowRunInfo(wr, true, sdk.SpawnMsg{ ID: sdk.MsgWorkflowError.ID, diff --git a/engine/api/workflow/process_outgoinghook.go b/engine/api/workflow/process_outgoinghook.go index 5c4eef18a9..909632f799 100644 --- a/engine/api/workflow/process_outgoinghook.go +++ b/engine/api/workflow/process_outgoinghook.go @@ -107,7 +107,7 @@ func processNodeOutGoingHook(ctx context.Context, db gorp.SqlExecutor, store cac } var task sdk.Task - if _, err := services.DoJSONRequest(ctx, srvs, "POST", "/task/execute", hookRun, &task); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, "POST", "/task/execute", hookRun, &task); err != nil { log.Warning("outgoing hook execution failed: %v", err) hookRun.Status = sdk.StatusFail.String() } diff --git a/engine/api/workflow/repository.go b/engine/api/workflow/repository.go index aae726d9ca..43c426c9b8 100644 --- a/engine/api/workflow/repository.go +++ b/engine/api/workflow/repository.go @@ -334,7 +334,7 @@ func PostRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, prj sdk.P } if multipartData == nil { - if _, err := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/operations", ope, ope); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/operations", ope, ope); err != nil { return sdk.WrapError(err, "Unable to perform operation") } return nil @@ -352,7 +352,7 @@ func GetRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, ope *sdk.O return sdk.WrapError(err, "Unable to found repositories service") } - if _, err := services.DoJSONRequest(ctx, srvs, http.MethodGet, "/operations/"+ope.UUID, nil, ope); err != nil { + if _, _, err := services.DoJSONRequest(ctx, srvs, http.MethodGet, "/operations/"+ope.UUID, nil, ope); err != nil { return sdk.WrapError(err, "Unable to get operation") } return nil diff --git a/engine/api/workflow/workflow_run_event.go b/engine/api/workflow/workflow_run_event.go index 85d8d24c1e..5a7274a974 100644 --- a/engine/api/workflow/workflow_run_event.go +++ b/engine/api/workflow/workflow_run_event.go @@ -104,7 +104,7 @@ func ResyncCommitStatus(ctx context.Context, db gorp.SqlExecutor, store cache.St details := fmt.Sprintf("on project:%s workflow:%s node:%s num:%d sub:%d vcs:%s", proj.Name, wr.Workflow.Name, nodeRun.WorkflowNodeName, nodeRun.Number, nodeRun.SubNumber, vcsServer.Name) //Get the RepositoriesManager Client - client, errClient := repositoriesmanager.AuthorizedClient(ctx, db, store, vcsServer) + client, errClient := repositoriesmanager.AuthorizedClient(ctx, db, store, proj.Key, vcsServer) if errClient != nil { return sdk.WrapError(errClient, "resyncCommitStatus> Cannot get client %s", details) } @@ -213,7 +213,7 @@ func sendVCSEventStatus(ctx context.Context, db gorp.SqlExecutor, store cache.St } //Get the RepositoriesManager Client - client, errClient := repositoriesmanager.AuthorizedClient(ctx, db, store, vcsServer) + client, errClient := repositoriesmanager.AuthorizedClient(ctx, db, store, proj.Key, vcsServer) if errClient != nil { return sdk.WrapError(errClient, "sendVCSEventStatus> Cannot get client") } diff --git a/engine/api/workflow/workflow_vcs.go b/engine/api/workflow/workflow_vcs.go index 6682226b2d..0eba1acadf 100644 --- a/engine/api/workflow/workflow_vcs.go +++ b/engine/api/workflow/workflow_vcs.go @@ -21,5 +21,5 @@ func createVCSClientFromRootNode(ctx context.Context, db gorp.SqlExecutor, store if vcsServer == nil { return nil, sdk.WithStack(fmt.Errorf("no vcsServer found")) } - return repositoriesmanager.AuthorizedClient(ctx, db, store, vcsServer) + return repositoriesmanager.AuthorizedClient(ctx, db, store, proj.Key, vcsServer) } diff --git a/engine/api/workflow_application.go b/engine/api/workflow_application.go index b3730231ee..8c959f875f 100644 --- a/engine/api/workflow_application.go +++ b/engine/api/workflow_application.go @@ -77,7 +77,7 @@ func (api *API) releaseApplicationWorkflowHandler() service.Handler { return sdk.WrapError(sdk.ErrNoReposManager, "releaseApplicationWorkflowHandler") } - client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, rm) + client, err := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, proj.Key, rm) if err != nil { return sdk.WrapError(err, "Cannot get client got %s %s", key, app.VCSServer) } diff --git a/engine/api/workflow_hook.go b/engine/api/workflow_hook.go index 999c41f99a..ca092fe3a3 100644 --- a/engine/api/workflow_hook.go +++ b/engine/api/workflow_hook.go @@ -77,7 +77,7 @@ func (api *API) getWorkflowHookModelsHandler() service.Handler { // Call VCS to know if repository allows webhook and get the configuration fields vcsServer := repositoriesmanager.GetProjectVCSServer(p, wf.GetApplication(node.Context.ApplicationID).VCSServer) if vcsServer != nil { - client, errclient := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, vcsServer) + client, errclient := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, p.Key, vcsServer) if errclient != nil { return sdk.WrapError(errclient, "getWorkflowHookModelsHandler> Cannot get vcs client") } diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 19f4265ff8..0206b7b9d8 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -972,7 +972,7 @@ func (api *API) postWorkflowJobTestsResultsHandler() service.Handler { // Get vcs info to known if we are on the default branch or not projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, nr.VCSServer) - client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projectVCSServer) + client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, p.Key, projectVCSServer) if erra != nil { log.Error("postWorkflowJobTestsResultsHandler> Cannot get repo client %s : %v", nr.VCSServer, erra) return nil diff --git a/engine/hooks/tasks.go b/engine/hooks/tasks.go index 51271ca137..977a057595 100644 --- a/engine/hooks/tasks.go +++ b/engine/hooks/tasks.go @@ -27,9 +27,10 @@ const ( TypeOutgoingWebHook = "OutgoingWebhook" TypeOutgoingWorkflow = "OutgoingWorkflow" - GithubHeader = "X-Github-Event" - GitlabHeader = "X-Gitlab-Event" - BitbucketHeader = "X-Event-Key" + GithubHeader = "X-Github-Event" + GitlabHeader = "X-Gitlab-Event" + BitbucketHeader = "X-Event-Key" + BitbucketCloudHeader = "X-Event-Key_Cloud" // Fake header, do not use to fetch header, just to return custom header ConfigNumber = "Number" ConfigSubNumber = "SubNumber" diff --git a/engine/hooks/type_bitbucket_cloud.go b/engine/hooks/type_bitbucket_cloud.go new file mode 100644 index 0000000000..8b5a71fff5 --- /dev/null +++ b/engine/hooks/type_bitbucket_cloud.go @@ -0,0 +1,310 @@ +package hooks + +import "time" + +// BitbucketServerPushEvent represents payload send by bitbucket cloud on a push event +type BitbucketCloudPushEvent struct { + Push struct { + Changes []struct { + Forced bool `json:"forced"` + Old struct { + Name string `json:"name"` + Links struct { + Commits struct { + Href string `json:"href"` + } `json:"commits"` + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + DefaultMergeStrategy string `json:"default_merge_strategy"` + MergeStrategies []string `json:"merge_strategies"` + Type string `json:"type"` + Target struct { + Rendered struct { + } `json:"rendered"` + Hash string `json:"hash"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + Author struct { + Raw string `json:"raw"` + Type string `json:"type"` + User struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + Nickname string `json:"nickname"` + Type string `json:"type"` + AccountID string `json:"account_id"` + } `json:"user"` + } `json:"author"` + Summary struct { + Raw string `json:"raw"` + Markup string `json:"markup"` + HTML string `json:"html"` + Type string `json:"type"` + } `json:"summary"` + Parents []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + } `json:"parents"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` + Properties struct { + } `json:"properties"` + } `json:"target"` + } `json:"old"` + Links struct { + Commits struct { + Href string `json:"href"` + } `json:"commits"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Diff struct { + Href string `json:"href"` + } `json:"diff"` + } `json:"links"` + Created bool `json:"created"` + Commits []struct { + Rendered struct { + } `json:"rendered"` + Hash string `json:"hash"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + Comments struct { + Href string `json:"href"` + } `json:"comments"` + Patch struct { + Href string `json:"href"` + } `json:"patch"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Diff struct { + Href string `json:"href"` + } `json:"diff"` + Approve struct { + Href string `json:"href"` + } `json:"approve"` + Statuses struct { + Href string `json:"href"` + } `json:"statuses"` + } `json:"links"` + Author struct { + Raw string `json:"raw"` + Type string `json:"type"` + User struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + Nickname string `json:"nickname"` + Type string `json:"type"` + AccountID string `json:"account_id"` + } `json:"user"` + } `json:"author"` + Summary struct { + Raw string `json:"raw"` + Markup string `json:"markup"` + HTML string `json:"html"` + Type string `json:"type"` + } `json:"summary"` + Parents []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + } `json:"parents"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` + Properties struct { + } `json:"properties"` + } `json:"commits"` + Truncated bool `json:"truncated"` + Closed bool `json:"closed"` + New struct { + Name string `json:"name"` + Links struct { + Commits struct { + Href string `json:"href"` + } `json:"commits"` + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + DefaultMergeStrategy string `json:"default_merge_strategy"` + MergeStrategies []string `json:"merge_strategies"` + Type string `json:"type"` + Target struct { + Rendered struct { + } `json:"rendered"` + Hash string `json:"hash"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + Author struct { + Raw string `json:"raw"` + Type string `json:"type"` + User struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + Nickname string `json:"nickname"` + Type string `json:"type"` + AccountID string `json:"account_id"` + } `json:"user"` + } `json:"author"` + Summary struct { + Raw string `json:"raw"` + Markup string `json:"markup"` + HTML string `json:"html"` + Type string `json:"type"` + } `json:"summary"` + Parents []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + } `json:"links"` + } `json:"parents"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` + Properties struct { + } `json:"properties"` + } `json:"target"` + } `json:"new"` + } `json:"changes"` + } `json:"push"` + Actor struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + Nickname string `json:"nickname"` + Type string `json:"type"` + AccountID string `json:"account_id"` + } `json:"actor"` + Repository struct { + Scm string `json:"scm"` + Website string `json:"website"` + Name string `json:"name"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + FullName string `json:"full_name"` + Owner struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + Nickname string `json:"nickname"` + Type string `json:"type"` + AccountID string `json:"account_id"` + } `json:"owner"` + Type string `json:"type"` + IsPrivate bool `json:"is_private"` + UUID string `json:"uuid"` + } `json:"repository"` +} diff --git a/engine/hooks/type_bitbucket.go b/engine/hooks/type_bitbucket_server.go similarity index 90% rename from engine/hooks/type_bitbucket.go rename to engine/hooks/type_bitbucket_server.go index 0b35281584..2dd81e9660 100644 --- a/engine/hooks/type_bitbucket.go +++ b/engine/hooks/type_bitbucket_server.go @@ -1,7 +1,7 @@ package hooks -// BitbucketPushEvent represents payload send by github on a push event -type BitbucketPushEvent struct { +// BitbucketServerPushEvent represents payload send by bitbucket server on a push event +type BitbucketServerPushEvent struct { EventKey string `json:"eventKey"` Date string `json:"date"` Actor struct { diff --git a/engine/hooks/webhook.go b/engine/hooks/webhook.go index 094b1411fa..f920dc5bed 100644 --- a/engine/hooks/webhook.go +++ b/engine/hooks/webhook.go @@ -34,6 +34,9 @@ func getRepositoryHeader(whe *sdk.WebHookExecution) string { return GitlabHeader } else if v, ok := whe.RequestHeader[BitbucketHeader]; ok && v[0] == "repo:refs_changed" { return BitbucketHeader + } else if v, ok := whe.RequestHeader[BitbucketHeader]; ok && v[0] == "repo:push" { + // We return a fake header to make a difference between server and cloud version + return BitbucketCloudHeader } return "" } @@ -132,7 +135,7 @@ func (s *Service) executeRepositoryWebHook(t *sdk.TaskExecution) ([]sdk.Workflow payload["payload"] = string(payloadStr) payloads = append(payloads, payload) case BitbucketHeader: - var pushEvent BitbucketPushEvent + var pushEvent BitbucketServerPushEvent if err := json.Unmarshal(t.WebHook.RequestBody, &pushEvent); err != nil { return nil, sdk.WrapError(err, "unable ro read bitbucket request: %s", string(t.WebHook.RequestBody)) } @@ -176,6 +179,58 @@ func (s *Service) executeRepositoryWebHook(t *sdk.TaskExecution) ([]sdk.Workflow payload["payload"] = string(payloadStr) payloads = append(payloads, payload) } + case BitbucketCloudHeader: + var event BitbucketCloudPushEvent + if err := json.Unmarshal(t.WebHook.RequestBody, &event); err != nil { + return nil, sdk.WrapError(err, "unable ro read bitbucket request: %s", string(t.WebHook.RequestBody)) + } + pushEvent := event.Push + if len(pushEvent.Changes) == 0 { + return nil, nil + } + + for _, pushChange := range pushEvent.Changes { + if pushChange.Closed { + if pushChange.Old.Type == "branch" { + err := s.enqueueBranchDeletion(projectKey, workflowName, strings.TrimPrefix(pushChange.Old.Name, "refs/heads/")) + if err != nil { + log.Error("cannot enqueue branch deletion: %v", err) + } + } + continue + } + payload := make(map[string]interface{}) + payload["git.author"] = event.Actor.DisplayName + if len(pushChange.New.Target.Message) > 0 { + payload["git.message"] = pushChange.New.Target.Message + } + + if pushChange.New.Type == "branch" { + payload["git.branch"] = strings.TrimPrefix(pushChange.New.Name, "refs/heads/") + } else if pushChange.New.Type == "tag" { + payload["git.tag"] = strings.TrimPrefix(pushChange.New.Name, "refs/tags/") + } else { + log.Warning("Uknown push type: %s", pushChange.New.Type) + continue + } + payload["git.hash.before"] = pushChange.Old.Target.Hash + payload["git.hash"] = pushChange.New.Target.Hash + hashShort := pushChange.New.Target.Hash + if len(hashShort) >= 7 { + hashShort = hashShort[:7] + } + payload["git.hash.short"] = hashShort + payload["git.repository"] = event.Repository.FullName + + payload["cds.triggered_by.username"] = event.Actor.Username + payload["cds.triggered_by.fullname"] = event.Actor.DisplayName + payloadStr, err := json.Marshal(pushEvent) + if err != nil { + log.Error("Unable to marshal payload: %v", err) + } + payload["payload"] = string(payloadStr) + payloads = append(payloads, payload) + } default: log.Warning("executeRepositoryWebHook> Repository manager not found. Cannot read %s", string(t.WebHook.RequestBody)) return nil, fmt.Errorf("Repository manager not found. Cannot read request body") diff --git a/engine/vcs/bitbucketcloud/bitbucketcloud.go b/engine/vcs/bitbucketcloud/bitbucketcloud.go new file mode 100644 index 0000000000..dc69fcc875 --- /dev/null +++ b/engine/vcs/bitbucketcloud/bitbucketcloud.go @@ -0,0 +1,53 @@ +package bitbucketcloud + +import ( + "context" + + "github.com/ovh/cds/engine/api/cache" + "github.com/ovh/cds/sdk" +) + +const rootURL = "https://api.bitbucket.org/2.0" + +// bitbucketcloudClient is a https://bitbucket.org wrapper for CDS vcs. interface +type bitbucketcloudClient struct { + ClientID string + OAuthToken string + RefreshToken string + DisableStatus bool + DisableStatusDetail bool + Cache cache.Store + apiURL string + uiURL string + proxyURL string +} + +//bitbucketcloudConsumer implements vcs.Server and it's used to instanciate a githubClient +type bitbucketcloudConsumer struct { + ClientID string `json:"client-id"` + ClientSecret string `json:"-"` + Cache cache.Store + uiURL string + apiURL string + proxyURL string + disableStatus bool + disableStatusDetail bool +} + +//New creates a new GithubConsumer +func New(ClientID, ClientSecret, apiURL, uiURL, proxyURL string, store cache.Store, disableStatus, disableStatusDetail bool) sdk.VCSServer { + return &bitbucketcloudConsumer{ + ClientID: ClientID, + ClientSecret: ClientSecret, + Cache: store, + apiURL: apiURL, + uiURL: uiURL, + proxyURL: proxyURL, + disableStatus: disableStatus, + disableStatusDetail: disableStatusDetail, + } +} + +func (c *bitbucketcloudClient) GetAccessToken(_ context.Context) string { + return c.OAuthToken +} diff --git a/engine/vcs/bitbucketcloud/bitbucketcloud_test.go b/engine/vcs/bitbucketcloud/bitbucketcloud_test.go new file mode 100644 index 0000000000..a889f3fc0c --- /dev/null +++ b/engine/vcs/bitbucketcloud/bitbucketcloud_test.go @@ -0,0 +1,174 @@ +package bitbucketcloud + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/ovh/cds/sdk" + + "github.com/pkg/browser" + "github.com/stretchr/testify/assert" + + "github.com/ovh/cds/engine/api/cache" + "github.com/ovh/cds/engine/api/test" + "github.com/ovh/cds/sdk/log" +) + +var currentAccessToken string +var currentRefreshToken string + +// TestNew needs bitbucketCloudClientID and bitbucketCloudClientSecret +func TestNewClient(t *testing.T) { + bbConsumer := getNewConsumer(t) + assert.NotNil(t, bbConsumer) +} + +func getNewConsumer(t *testing.T) sdk.VCSServer { + log.SetLogger(t) + cfg := test.LoadTestingConf(t) + clientID := cfg["bitbucketCloudClientID"] + clientSecret := cfg["bitbucketCloudClientSecret"] + redisHost := cfg["redisHost"] + redisPassword := cfg["redisPassword"] + + if clientID == "" && clientSecret == "" { + t.Logf("Unable to read bitbucket cloud configuration. Skipping this tests.") + t.SkipNow() + } + + cache, err := cache.New(redisHost, redisPassword, 30) + if err != nil { + t.Fatalf("Unable to init cache (%s): %v", redisHost, err) + } + + bbConsumer := New(clientID, clientSecret, "http://localhost", "", "", cache, true, true) + return bbConsumer +} + +func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { + log.SetLogger(t) + cfg := test.LoadTestingConf(t) + clientID := cfg["bitbucketCloudClientID"] + clientSecret := cfg["bitbucketCloudClientSecret"] + redisHost := cfg["redisHost"] + redisPassword := cfg["redisPassword"] + + if clientID == "" && clientSecret == "" { + t.Logf("Unable to read github configuration. Skipping this tests.") + t.SkipNow() + } + + cache, err := cache.New(redisHost, redisPassword, 30) + if err != nil { + t.Fatalf("Unable to init cache (%s): %v", redisHost, err) + } + + bbConsumer := New(clientID, clientSecret, "http://localhost", "", "", cache, true, true) + cli, err := bbConsumer.GetAuthorizedClient(context.Background(), currentAccessToken, currentRefreshToken, time.Now().Unix()) + if err != nil { + t.Fatalf("Unable to init authorized client (%s): %v", redisHost, err) + } + + return cli +} + +func TestClientAuthorizeToken(t *testing.T) { + bbConsumer := getNewConsumer(t) + token, url, err := bbConsumer.AuthorizeRedirect(context.Background()) + t.Logf("token: %s", token) + t.Logf("url: %s", url) + assert.NotEmpty(t, token) + assert.NotEmpty(t, url) + test.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + out := make(chan http.Request, 1) + + go callbackServer(ctx, t, out) + err = browser.OpenURL(url) + test.NoError(t, err) + + r, ok := <-out + t.Logf("Chan request closed? %v", !ok) + t.Logf("OAuth request 2: %+v", r) + assert.NotNil(t, r) + + cberr := r.FormValue("error") + errDescription := r.FormValue("error_description") + errURI := r.FormValue("error_uri") + + assert.Empty(t, cberr) + assert.Empty(t, errDescription) + assert.Empty(t, errURI) + + code := r.FormValue("code") + + assert.NotEmpty(t, code) + + accessToken, refreshToken, err := bbConsumer.AuthorizeToken(context.Background(), "", code) + assert.NotEmpty(t, accessToken) + assert.NotEmpty(t, refreshToken) + test.NoError(t, err) + + currentAccessToken = accessToken + currentRefreshToken = refreshToken + t.Logf("Token is %s", accessToken) + + bbClient, err := bbConsumer.GetAuthorizedClient(context.Background(), accessToken, refreshToken, time.Now().Unix()) + test.NoError(t, err) + assert.NotNil(t, bbClient) +} + +func TestAuthorizedClient(t *testing.T) { + bbClient := getNewAuthorizedClient(t) + assert.NotNil(t, bbClient) +} + +func TestRepos(t *testing.T) { + bbClient := getNewAuthorizedClient(t) + assert.NotNil(t, bbClient) + + repos, err := bbClient.Repos(context.Background()) + test.NoError(t, err) + assert.NotEmpty(t, repos) +} + +func TestRepoByFullname(t *testing.T) { + bbClient := getNewAuthorizedClient(t) + assert.NotNil(t, bbClient) + + repo, err := bbClient.RepoByFullname(context.Background(), "bnjjj/test") + test.NoError(t, err) + assert.NotNil(t, repo) +} + +func TestBranches(t *testing.T) { + bbClient := getNewAuthorizedClient(t) + assert.NotNil(t, bbClient) + + branches, err := bbClient.Branches(context.Background(), "bnjjj/test") + test.NoError(t, err) + assert.NotEmpty(t, branches) +} + +func TestBranch(t *testing.T) { + bbClient := getNewAuthorizedClient(t) + assert.NotNil(t, bbClient) + + branch, err := bbClient.Branch(context.Background(), "bnjjj/test", "master") + test.NoError(t, err) + assert.NotNil(t, branch) +} + +func TestCommits(t *testing.T) { + bbClient := getNewAuthorizedClient(t) + assert.NotNil(t, bbClient) + + commits, err := bbClient.Commits(context.Background(), "bnjjj/test", "master", "HEAD", "HEAD") + test.NoError(t, err) + assert.NotNil(t, commits) +} diff --git a/engine/vcs/bitbucketcloud/bitbucketcloud_test_helpers.go b/engine/vcs/bitbucketcloud/bitbucketcloud_test_helpers.go new file mode 100644 index 0000000000..492a1def7f --- /dev/null +++ b/engine/vcs/bitbucketcloud/bitbucketcloud_test_helpers.go @@ -0,0 +1,32 @@ +package bitbucketcloud + +import ( + "context" + "fmt" + "io" + "net/http" + "testing" +) + +func callbackServer(ctx context.Context, t *testing.T, out chan http.Request) { + srv := &http.Server{Addr: ":8081"} + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + out <- *r + _, _ = io.WriteString(w, "Yeah !\n") + fmt.Println("Handler") + }) + + go func() { + fmt.Println("Starting server") + if err := srv.ListenAndServe(); err != nil { + // cannot panic, because this probably is an intentional close + t.Logf("Httpserver: ListenAndServe() error: %s", err) + } + close(out) + }() + + <-ctx.Done() + fmt.Println("Stopping server") + _ = srv.Shutdown(ctx) +} diff --git a/engine/vcs/bitbucketcloud/client_branch.go b/engine/vcs/bitbucketcloud/client_branch.go new file mode 100644 index 0000000000..5cb8ef4b75 --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_branch.go @@ -0,0 +1,105 @@ +package bitbucketcloud + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +// Branches returns list of branches for a repo +func (client *bitbucketcloudClient) Branches(ctx context.Context, fullname string) ([]sdk.VCSBranch, error) { + repo, err := client.repoByFullname(fullname) + if err != nil { + return nil, sdk.WrapError(err, "cannot get repo by fullname") + } + + var branches []Branch + path := fmt.Sprintf("/repositories/%s/refs/branches", fullname) + params := url.Values{} + params.Set("pagelen", "100") + params.Set("sort", "-target.date") + nextPage := 1 + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response Branches + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get branches") + } + if cap(branches) == 0 { + branches = make([]Branch, 0, response.Size) + } + + branches = append(branches, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + + branchesResult := make([]sdk.VCSBranch, 0, len(branches)) + for _, b := range branches { + branch := sdk.VCSBranch{ + DisplayID: b.Name, + ID: b.Name, + LatestCommit: b.Target.Hash, + Default: b.Name == repo.Mainbranch.Name, + } + for _, p := range b.Target.Parents { + branch.Parents = append(branch.Parents, p.Hash) + } + branchesResult = append(branchesResult, branch) + } + + return branchesResult, nil +} + +// Branch returns only detail of a branch +func (client *bitbucketcloudClient) Branch(ctx context.Context, fullname, theBranch string) (*sdk.VCSBranch, error) { + repo, err := client.repoByFullname(fullname) + if err != nil { + return nil, err + } + + url := fmt.Sprintf("/repositories/%s/refs/branches/%s", fullname, theBranch) + status, body, _, err := client.get(url) + if err != nil { + return nil, err + } + if status >= 400 { + return nil, sdk.NewError(sdk.ErrUnknownError, errorAPI(body)) + } + + var branch Branch + if err := json.Unmarshal(body, &branch); err != nil { + log.Warning("bitbucketcloudClient.Branch> Unable to parse github branch: %s", err) + return nil, err + } + + if branch.Name == "" { + return nil, fmt.Errorf("bitbucketcloudClient.Branch > Cannot find branch %s", theBranch) + } + + branchResult := &sdk.VCSBranch{ + DisplayID: branch.Name, + ID: branch.Name, + LatestCommit: branch.Target.Hash, + Default: branch.Name == repo.Mainbranch.Name, + } + + if branch.Target.Hash != "" { + for _, p := range branch.Target.Parents { + branchResult.Parents = append(branchResult.Parents, p.Hash) + } + } + + return branchResult, nil +} diff --git a/engine/vcs/bitbucketcloud/client_commit.go b/engine/vcs/bitbucketcloud/client_commit.go new file mode 100644 index 0000000000..4083121e0f --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_commit.go @@ -0,0 +1,166 @@ +package bitbucketcloud + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +// Commits returns the commits list on a branch between a commit SHA (since) until another commit SHA (until). The branch is given by the branch of the first commit SHA (since) +func (client *bitbucketcloudClient) Commits(ctx context.Context, repo, theBranch, since, until string) ([]sdk.VCSCommit, error) { + var commitsResult []sdk.VCSCommit + //Get Commit List + theCommits, err := client.allCommitBetween(ctx, repo, since, until, theBranch) + if err != nil { + return nil, sdk.WrapError(err, "cannot load all commit between since=%s and until=%s on branch %s", since, until, theBranch) + } + + commitsResult = make([]sdk.VCSCommit, 0, len(theCommits)) + //Convert to sdk.VCSCommit + for _, c := range theCommits { + email := strings.Trim(rawEmailCommitRegexp.FindString(c.Author.Raw), "<>") + commit := sdk.VCSCommit{ + Timestamp: c.Date.Unix() * 1000, + Message: c.Message, + Hash: c.Hash, + URL: c.Links.HTML.Href, + Author: sdk.VCSAuthor{ + DisplayName: c.Author.User.DisplayName, + Email: email, + Name: c.Author.User.Username, + Avatar: c.Author.User.Links.Avatar.Href, + }, + } + + commitsResult = append(commitsResult, commit) + } + + return commitsResult, nil +} + +func (client *bitbucketcloudClient) allCommitBetween(ctx context.Context, repo, sinceCommit, untilCommit, branch string) ([]Commit, error) { + var commits []Commit + params := url.Values{} + params.Add("exclude", sinceCommit) + path := fmt.Sprintf("/repositories/%s/commits/%s", repo, untilCommit) + nextPage := 1 + + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response Commits + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get commits") + } + if cap(commits) == 0 { + commits = make([]Commit, 0, response.Size) + } + commits = append(commits, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + + return commits, nil +} + +// Commit Get a single commit +func (client *bitbucketcloudClient) Commit(ctx context.Context, repo, hash string) (sdk.VCSCommit, error) { + var commit sdk.VCSCommit + url := fmt.Sprintf("/repositories/%s/commit/%s", repo, hash) + status, body, _, err := client.get(url) + if err != nil { + log.Warning("bitbucketcloudClient.Commit> Error %s", err) + return commit, err + } + if status >= 400 { + return commit, sdk.NewError(sdk.ErrRepoNotFound, errorAPI(body)) + } + var c Commit + if err := json.Unmarshal(body, &c); err != nil { + log.Warning("bitbucketcloudClient.Commit> Unable to parse bitbucket cloud commit: %s", err) + return sdk.VCSCommit{}, err + } + + email := strings.Trim(rawEmailCommitRegexp.FindString(c.Author.Raw), "<>") + commit = sdk.VCSCommit{ + Timestamp: c.Date.Unix() * 1000, + Message: c.Message, + Hash: c.Hash, + URL: c.Links.HTML.Href, + Author: sdk.VCSAuthor{ + DisplayName: c.Author.User.DisplayName, + Email: email, + Name: c.Author.User.Username, + Avatar: c.Author.User.Links.Avatar.Href, + }, + } + + return commit, nil +} + +func (client *bitbucketcloudClient) CommitsBetweenRefs(ctx context.Context, repo, base, head string) ([]sdk.VCSCommit, error) { + var commits []Commit + if base == "" { + base = "HEAD" + } + if head == "" { + head = "HEAD" + } + params := url.Values{} + params.Add("exclude", base) + path := fmt.Sprintf("/repositories/%s/commits/%s", repo, head) + nextPage := 1 + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response Commits + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get commits") + } + if cap(commits) == 0 { + commits = make([]Commit, 0, response.Size) + } + commits = append(commits, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + + commitsResult := make([]sdk.VCSCommit, 0, len(commits)) + //Convert to sdk.VCSCommit + for _, c := range commits { + email := strings.Trim(rawEmailCommitRegexp.FindString(c.Author.Raw), "<>") + commit := sdk.VCSCommit{ + Timestamp: c.Date.Unix() * 1000, + Message: c.Message, + Hash: c.Hash, + URL: c.Links.HTML.Href, + Author: sdk.VCSAuthor{ + DisplayName: c.Author.User.DisplayName, + Email: email, + Name: c.Author.User.Username, + Avatar: c.Author.User.Links.Avatar.Href, + }, + } + + commitsResult = append(commitsResult, commit) + } + + return commitsResult, nil +} diff --git a/engine/vcs/bitbucketcloud/client_event.go b/engine/vcs/bitbucketcloud/client_event.go new file mode 100644 index 0000000000..5ef3694c3d --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_event.go @@ -0,0 +1,39 @@ +package bitbucketcloud + +import ( + "context" + "fmt" + "time" + + "github.com/ovh/cds/sdk" +) + +// ErrNoNewEvents for no new events +var ( + ErrNoNewEvents = fmt.Errorf("No new events") +) + +//GetEvents returns events from bitbucket cloud +func (client *bitbucketcloudClient) GetEvents(ctx context.Context, fullname string, dateRef time.Time) ([]interface{}, time.Duration, error) { + return nil, 0, sdk.WithStack(sdk.ErrNotImplemented) +} + +//PushEvents returns push events as commits +func (client *bitbucketcloudClient) PushEvents(ctx context.Context, fullname string, iEvents []interface{}) ([]sdk.VCSPushEvent, error) { + return nil, sdk.WithStack(sdk.ErrNotImplemented) +} + +//CreateEvents checks create events from a event list +func (client *bitbucketcloudClient) CreateEvents(ctx context.Context, fullname string, iEvents []interface{}) ([]sdk.VCSCreateEvent, error) { + return nil, sdk.WithStack(sdk.ErrNotImplemented) +} + +//DeleteEvents checks delete events from a event list +func (client *bitbucketcloudClient) DeleteEvents(ctx context.Context, fullname string, iEvents []interface{}) ([]sdk.VCSDeleteEvent, error) { + return nil, sdk.WithStack(sdk.ErrNotImplemented) +} + +//PullRequestEvents checks pull request events from a event list +func (client *bitbucketcloudClient) PullRequestEvents(ctx context.Context, fullname string, iEvents []interface{}) ([]sdk.VCSPullRequestEvent, error) { + return nil, sdk.WithStack(sdk.ErrNotImplemented) +} diff --git a/engine/vcs/bitbucketcloud/client_forks.go b/engine/vcs/bitbucketcloud/client_forks.go new file mode 100644 index 0000000000..96c94a42c3 --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_forks.go @@ -0,0 +1,54 @@ +package bitbucketcloud + +import ( + "context" + "fmt" + "net/url" + + "github.com/ovh/cds/sdk" +) + +func (client *bitbucketcloudClient) ListForks(ctx context.Context, repo string) ([]sdk.VCSRepo, error) { + var repos []Repository + path := fmt.Sprintf("/repositories/%s/forks", repo) + params := url.Values{} + params.Set("pagelen", "100") + nextPage := 1 + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response Repositories + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get repos") + } + if cap(repos) == 0 { + repos = make([]Repository, 0, response.Size) + } + + repos = append(repos, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + + responseRepos := make([]sdk.VCSRepo, 0, len(repos)) + for _, repo := range repos { + r := sdk.VCSRepo{ + ID: repo.UUID, + Name: repo.Name, + Slug: repo.Slug, + Fullname: repo.FullName, + URL: repo.Links.HTML.Href, + HTTPCloneURL: repo.Links.Clone[0].Href, + SSHCloneURL: repo.Links.Clone[1].Href, + } + responseRepos = append(responseRepos, r) + } + + return responseRepos, nil +} diff --git a/engine/vcs/bitbucketcloud/client_hook.go b/engine/vcs/bitbucketcloud/client_hook.go new file mode 100644 index 0000000000..7877ad1fc3 --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_hook.go @@ -0,0 +1,148 @@ +package bitbucketcloud + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + "strings" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +func (client *bitbucketcloudClient) CreateHook(ctx context.Context, repo string, hook *sdk.VCSHook) error { + url := fmt.Sprintf("/repositories/%s/hooks", repo) + if client.proxyURL != "" { + lastIndexSlash := strings.LastIndex(hook.URL, "/") + if client.proxyURL[len(client.proxyURL)-1] == '/' { + lastIndexSlash++ + } + hook.URL = client.proxyURL + hook.URL[lastIndexSlash:] + } + + r := WebhookCreate{ + Description: "CDS webhook - " + hook.Name, + Active: true, + Events: []string{"repo:push"}, + URL: hook.URL, + } + b, err := json.Marshal(r) + if err != nil { + return sdk.WrapError(err, "Cannot marshal body %+v", r) + } + res, err := client.post(url, "application/json", bytes.NewBuffer(b), nil) + if err != nil { + return sdk.WrapError(err, "bitbucketcloud.CreateHook") + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return sdk.WrapError(err, "ReadAll") + } + if res.StatusCode != 201 { + err := fmt.Errorf("Unable to create webhook on bitbucketcloud. Status code : %d - Body: %s. ", res.StatusCode, body) + return sdk.WrapError(err, "bitbucketcloud.CreateHook. Data : %s", b) + } + + var webhook Webhook + if err := json.Unmarshal(body, &webhook); err != nil { + return sdk.WrapError(err, "Cannot unmarshal response") + } + hook.ID = webhook.UUID + return nil +} + +func (client *bitbucketcloudClient) getHooks(ctx context.Context, fullname string) ([]Webhook, error) { + var webhooks []Webhook + path := fmt.Sprintf("/repositories/%s/hooks", fullname) + params := url.Values{} + params.Set("pagelen", "100") + nextPage := 1 + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response Webhooks + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get repos") + } + if cap(webhooks) == 0 { + webhooks = make([]Webhook, 0, response.Size) + } + + webhooks = append(webhooks, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + return webhooks, nil +} + +func (client *bitbucketcloudClient) GetHook(ctx context.Context, fullname, webhookURL string) (sdk.VCSHook, error) { + var hook sdk.VCSHook + hooks, err := client.getHooks(ctx, fullname) + if err != nil { + return hook, sdk.WithStack(err) + } + + for _, h := range hooks { + log.Info("hooks: %s (expecting: %s)", h.URL, webhookURL) + if h.URL == webhookURL { + return sdk.VCSHook{ + Name: h.Description, + Events: h.Events, + URL: h.URL, + ID: h.UUID, + }, nil + } + } + + return hook, sdk.WithStack(sdk.ErrNotFound) +} +func (client *bitbucketcloudClient) UpdateHook(ctx context.Context, repo, id string, hook sdk.VCSHook) error { + url := fmt.Sprintf("/repositories/%s/hooks", repo) + if client.proxyURL != "" { + lastIndexSlash := strings.LastIndex(hook.URL, "/") + if client.proxyURL[len(client.proxyURL)-1] == '/' { + lastIndexSlash++ + } + hook.URL = client.proxyURL + hook.URL[lastIndexSlash:] + } + + r := WebhookCreate{ + Description: "CDS webhook - " + hook.Name, + Active: true, + Events: []string{"repo:push"}, + URL: hook.URL, + } + b, err := json.Marshal(r) + if err != nil { + return sdk.WrapError(err, "Cannot marshal body %+v", r) + } + res, err := client.put(url, "application/json", bytes.NewBuffer(b), nil) + if err != nil { + return sdk.WrapError(err, "bitbucketcloud.UpdateHook") + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return sdk.WrapError(err, "ReadAll") + } + if res.StatusCode != 200 { + err := fmt.Errorf("Unable to update webhook on bitbucketcloud. Status code : %d - Body: %s. ", res.StatusCode, body) + return sdk.WrapError(err, "bitbucketcloud.CreateHook. Data : %s", b) + } + + return nil +} + +func (client *bitbucketcloudClient) DeleteHook(ctx context.Context, repo string, hook sdk.VCSHook) error { + return client.delete(fmt.Sprintf("/repositories/%s/hooks/%s", repo, hook.ID)) +} diff --git a/engine/vcs/bitbucketcloud/client_pull_request.go b/engine/vcs/bitbucketcloud/client_pull_request.go new file mode 100644 index 0000000000..ef366e0d17 --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_pull_request.go @@ -0,0 +1,165 @@ +package bitbucketcloud + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +func (client *bitbucketcloudClient) PullRequest(ctx context.Context, fullname string, id int) (sdk.VCSPullRequest, error) { + url := fmt.Sprintf("/repositories/%s/pullrequests/%d", fullname, id) + status, body, _, err := client.get(url) + if err != nil { + log.Warning("bitbucketcloudClient.Pullrequest> Error %s", err) + return sdk.VCSPullRequest{}, err + } + if status >= 400 { + return sdk.VCSPullRequest{}, sdk.NewError(sdk.ErrRepoNotFound, errorAPI(body)) + } + var pullrequest PullRequest + if err := json.Unmarshal(body, &pullrequest); err != nil { + log.Warning("bitbucketcloudClient.PullRequest> Unable to parse bitbucket cloud commit: %s", err) + return sdk.VCSPullRequest{}, err + } + + return pullrequest.ToVCSPullRequest(), nil +} + +// PullRequests fetch all the pull request for a repository +func (client *bitbucketcloudClient) PullRequests(ctx context.Context, fullname string) ([]sdk.VCSPullRequest, error) { + var pullrequests []PullRequest + path := fmt.Sprintf("/repositories/%s/pullrequests", fullname) + params := url.Values{} + params.Set("pagelen", "50") + nextPage := 1 + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response PullRequests + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get pull requests") + } + if cap(pullrequests) == 0 { + pullrequests = make([]PullRequest, 0, response.Size) + } + + pullrequests = append(pullrequests, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + + responsePullRequest := make([]sdk.VCSPullRequest, 0, len(pullrequests)) + for _, pr := range pullrequests { + responsePullRequest = append(responsePullRequest, pr.ToVCSPullRequest()) + } + + return responsePullRequest, nil +} + +// PullRequestComment push a new comment on a pull request +func (client *bitbucketcloudClient) PullRequestComment(ctx context.Context, repo string, id int, text string) error { + if client.DisableStatus { + log.Warning("bitbucketcloud.PullRequestComment> ⚠ bitbucketcloud statuses are disabled") + return nil + } + + path := fmt.Sprintf("/repos/%s/issues/%d/comments", repo, id) + payload := map[string]string{ + "body": text, + } + values, _ := json.Marshal(payload) + res, err := client.post(path, "application/json", bytes.NewReader(values), &postOptions{skipDefaultBaseURL: false, asUser: true}) + if err != nil { + return sdk.WrapError(err, "Unable to post status") + } + + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return sdk.WrapError(err, "Unable to read body") + } + + log.Debug("%v", string(body)) + + if res.StatusCode != 201 { + return sdk.WrapError(err, "Unable to create status on bitbucketcloud. Status code : %d - Body: %s", res.StatusCode, body) + } + + return nil +} + +func (client *bitbucketcloudClient) PullRequestCreate(ctx context.Context, repo string, pr sdk.VCSPullRequest) (sdk.VCSPullRequest, error) { + path := fmt.Sprintf("/repos/%s/pulls", repo) + payload := map[string]string{ + "title": pr.Title, + "head": pr.Head.Branch.DisplayID, + "base": pr.Base.Branch.DisplayID, + } + values, _ := json.Marshal(payload) + res, err := client.post(path, "application/json", bytes.NewReader(values), &postOptions{skipDefaultBaseURL: false, asUser: true}) + if err != nil { + return sdk.VCSPullRequest{}, sdk.WrapError(err, "Unable to post status") + } + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return sdk.VCSPullRequest{}, sdk.WrapError(err, "Unable to read body") + } + + var prResponse PullRequest + if err := json.Unmarshal(body, &prResponse); err != nil { + return sdk.VCSPullRequest{}, sdk.WrapError(err, "Unable to unmarshal pullrequest %s", string(body)) + } + + return prResponse.ToVCSPullRequest(), nil +} + +func (pullr PullRequest) ToVCSPullRequest() sdk.VCSPullRequest { + return sdk.VCSPullRequest{ + ID: pullr.ID, + Base: sdk.VCSPushEvent{ + Repo: pullr.Destination.Repository.FullName, + Branch: sdk.VCSBranch{ + ID: pullr.Destination.Branch.Name, + DisplayID: pullr.Destination.Branch.Name, + LatestCommit: pullr.Destination.Commit.Hash, + }, + Commit: sdk.VCSCommit{ + Hash: pullr.Destination.Commit.Hash, + }, + }, + Head: sdk.VCSPushEvent{ + Repo: pullr.Source.Repository.FullName, + Branch: sdk.VCSBranch{ + ID: pullr.Source.Branch.Name, + DisplayID: pullr.Source.Branch.Name, + LatestCommit: pullr.Source.Commit.Hash, + }, + Commit: sdk.VCSCommit{ + Hash: pullr.Source.Commit.Hash, + }, + }, + URL: pullr.Links.HTML.Href, + User: sdk.VCSAuthor{ + Avatar: pullr.Author.Links.Avatar.Href, + DisplayName: pullr.Author.DisplayName, + Name: pullr.Author.Username, + }, + Closed: pullr.State == "SUPERSEDED", + Merged: pullr.State == "MERGED", + } +} diff --git a/engine/vcs/bitbucketcloud/client_release.go b/engine/vcs/bitbucketcloud/client_release.go new file mode 100644 index 0000000000..b70da604aa --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_release.go @@ -0,0 +1,18 @@ +package bitbucketcloud + +import ( + "context" + "io" + + "github.com/ovh/cds/sdk" +) + +// Release Create a release +func (client *bitbucketcloudClient) Release(ctx context.Context, fullname string, tagName string, title string, releaseNote string) (*sdk.VCSRelease, error) { + return nil, sdk.WithStack(sdk.ErrNotImplemented) +} + +// UploadReleaseFile Attach a file into the release +func (client *bitbucketcloudClient) UploadReleaseFile(ctx context.Context, repo string, releaseName string, uploadURL string, artifactName string, r io.ReadCloser) error { + return sdk.WithStack(sdk.ErrNotImplemented) +} diff --git a/engine/vcs/bitbucketcloud/client_repos.go b/engine/vcs/bitbucketcloud/client_repos.go new file mode 100644 index 0000000000..364c05588e --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_repos.go @@ -0,0 +1,109 @@ +package bitbucketcloud + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +// Repos list repositories that are accessible to the authenticated user +func (client *bitbucketcloudClient) Repos(ctx context.Context) ([]sdk.VCSRepo, error) { + var repos []Repository + + user, err := client.CurrentUser(ctx) + if err != nil { + return nil, sdk.WrapError(err, "cannot load user info") + } + path := fmt.Sprintf("/repositories/%s", user.Username) + params := url.Values{} + params.Set("pagelen", "100") + params.Set("role", "contributor") + nextPage := 1 + for { + if nextPage != 1 { + params.Set("page", fmt.Sprintf("%d", nextPage)) + } + + var response Repositories + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get repos") + } + if cap(repos) == 0 { + repos = make([]Repository, 0, response.Size) + } + + repos = append(repos, response.Values...) + + if response.Next == "" { + break + } else { + nextPage++ + } + } + + responseRepos := make([]sdk.VCSRepo, 0, len(repos)) + for _, repo := range repos { + r := sdk.VCSRepo{ + ID: repo.UUID, + Name: repo.Name, + Slug: repo.Slug, + Fullname: repo.FullName, + URL: repo.Links.HTML.Href, + HTTPCloneURL: repo.Links.Clone[0].Href, + SSHCloneURL: repo.Links.Clone[1].Href, + } + responseRepos = append(responseRepos, r) + } + + return responseRepos, nil +} + +// RepoByFullname Get only one repo +func (client *bitbucketcloudClient) RepoByFullname(ctx context.Context, fullname string) (sdk.VCSRepo, error) { + repo, err := client.repoByFullname(fullname) + if err != nil { + return sdk.VCSRepo{}, err + } + + if repo.UUID == "" { + return sdk.VCSRepo{}, err + } + + r := sdk.VCSRepo{ + ID: repo.UUID, + Name: repo.Name, + Slug: repo.Slug, + Fullname: repo.FullName, + URL: repo.Links.HTML.Href, + HTTPCloneURL: repo.Links.Clone[0].Href, + SSHCloneURL: repo.Links.Clone[1].Href, + } + return r, nil +} + +func (client *bitbucketcloudClient) repoByFullname(fullname string) (Repository, error) { + var repo Repository + url := fmt.Sprintf("/repositories/%s", fullname) + status, body, _, err := client.get(url) + if err != nil { + log.Warning("bitbucketcloudClient.Repos> Error %s", err) + return repo, err + } + if status >= 400 { + return repo, sdk.NewError(sdk.ErrRepoNotFound, errorAPI(body)) + } + + if err := json.Unmarshal(body, &repo); err != nil { + return repo, sdk.WrapError(err, "Unable to parse github repository") + } + + return repo, nil +} + +func (client *bitbucketcloudClient) GrantWritePermission(ctx context.Context, fullname string) error { + return sdk.ErrNotImplemented +} diff --git a/engine/vcs/bitbucketcloud/client_status.go b/engine/vcs/bitbucketcloud/client_status.go new file mode 100644 index 0000000000..e0e7e4d726 --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_status.go @@ -0,0 +1,179 @@ +package bitbucketcloud + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "strings" + + "github.com/mitchellh/mapstructure" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +type statusData struct { + pipName string + desc string + status string + repoFullName string + hash string + urlPipeline string + context string +} + +//SetStatus Users with push access can create commit statuses for a given ref: +func (client *bitbucketcloudClient) SetStatus(ctx context.Context, event sdk.Event) error { + if client.DisableStatus { + log.Warning("bitbucketcloud.SetStatus> ⚠ bitbucketcloud statuses are disabled") + return nil + } + + var data statusData + var err error + switch event.EventType { + case fmt.Sprintf("%T", sdk.EventRunWorkflowNode{}): + data, err = processEventWorkflowNodeRun(event, client.uiURL, client.DisableStatusDetail) + default: + log.Error("bitbucketcloud.SetStatus> Unknown event %v", event) + return nil + } + if err != nil { + return sdk.WrapError(err, "Cannot process Event") + } + + if data.status == "" { + log.Debug("bitbucketcloud.SetStatus> Do not process event for current status: %v", event) + return nil + } + + bbStatus := Status{ + Description: data.desc, + URL: data.urlPipeline, + State: data.status, + Name: data.context, + Key: data.context, + } + + path := fmt.Sprintf("/repositories/%s/commit/%s/statuses/build", data.repoFullName, data.hash) + b, err := json.Marshal(bbStatus) + if err != nil { + return sdk.WrapError(err, "Unable to marshal github status") + } + buf := bytes.NewBuffer(b) + + res, err := client.post(path, "application/json", buf, nil) + if err != nil { + return sdk.WrapError(err, "Unable to post status") + } + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return sdk.WrapError(err, "Unable to read body") + } + if res.StatusCode != 201 && res.StatusCode != 200 { + return fmt.Errorf("Unable to create status on bitbucket cloud. Status code : %d - Body: %s - target:%s", res.StatusCode, body, data.urlPipeline) + } + + var resp Status + if err := json.Unmarshal(body, &resp); err != nil { + return sdk.WrapError(err, "Unable to unmarshal body") + } + + log.Debug("bitbucketcloud.SetStatus> Status %s %s created at %v", resp.UUID, resp.Links.Self.Href, resp.CreatedOn) + + return nil +} + +func (client *bitbucketcloudClient) ListStatuses(ctx context.Context, repo string, ref string) ([]sdk.VCSCommitStatus, error) { + url := fmt.Sprintf("/repositories/%s/commit/%s/statuses", repo, ref) + status, body, _, err := client.get(url) + if err != nil { + return []sdk.VCSCommitStatus{}, sdk.WrapError(err, "bitbucketcloudClient.ListStatuses") + } + if status >= 400 { + return []sdk.VCSCommitStatus{}, sdk.NewError(sdk.ErrRepoNotFound, errorAPI(body)) + } + var ss Statuses + if err := json.Unmarshal(body, &ss); err != nil { + return []sdk.VCSCommitStatus{}, sdk.WrapError(err, "Unable to parse bitbucket cloud commit: %s", ref) + } + + vcsStatuses := make([]sdk.VCSCommitStatus, 0, ss.Size) + for _, s := range ss.Values { + if !strings.HasPrefix(s.Name, "CDS/") { + continue + } + vcsStatuses = append(vcsStatuses, sdk.VCSCommitStatus{ + CreatedAt: s.CreatedOn, + Decription: s.Description, + Ref: ref, + State: processBbitbucketState(s), + }) + } + + return vcsStatuses, nil +} + +func processBbitbucketState(s Status) string { + switch s.State { + case "SUCCESSFUL": + return sdk.StatusSuccess.String() + case "FAILED": + return sdk.StatusFail.String() + case "STOPPED": + return sdk.StatusStopped.String() + default: + return sdk.StatusBuilding.String() + } +} + +func processEventWorkflowNodeRun(event sdk.Event, cdsUIURL string, disabledStatusDetail bool) (statusData, error) { + data := statusData{} + var eventNR sdk.EventRunWorkflowNode + if err := mapstructure.Decode(event.Payload, &eventNR); err != nil { + return data, sdk.WrapError(err, "Error durring consumption") + } + //We only manage status Success, Failure and Stopped + if eventNR.Status == sdk.StatusChecking.String() || + eventNR.Status == sdk.StatusDisabled.String() || + eventNR.Status == sdk.StatusNeverBuilt.String() || + eventNR.Status == sdk.StatusSkipped.String() || + eventNR.Status == sdk.StatusUnknown.String() || + eventNR.Status == sdk.StatusWaiting.String() { + return data, nil + } + + switch eventNR.Status { + case sdk.StatusFail.String(): + data.status = "FAILED" + case sdk.StatusSuccess.String(): + data.status = "SUCCESSFUL" + case sdk.StatusStopped.String(): + data.status = "STOPPED" + default: + data.status = "INPROGRESS" + } + data.hash = eventNR.Hash + data.repoFullName = eventNR.RepositoryFullName + data.pipName = eventNR.NodeName + + data.urlPipeline = fmt.Sprintf("%s/project/%s/workflow/%s/run/%d", + cdsUIURL, + event.ProjectKey, + event.WorkflowName, + eventNR.Number, + ) + + //CDS can avoid sending bitbucket target url in status, if it's disable + if disabledStatusDetail { + data.urlPipeline = "https://ovh.github.io/cds/" // because it's mandatory + } + + data.context = sdk.VCSCommitStatusDescription(event.ProjectKey, event.WorkflowName, eventNR) + data.desc = eventNR.NodeName + ": " + eventNR.Status + return data, nil +} diff --git a/engine/vcs/bitbucketcloud/client_tag.go b/engine/vcs/bitbucketcloud/client_tag.go new file mode 100644 index 0000000000..41229668aa --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_tag.go @@ -0,0 +1,48 @@ +package bitbucketcloud + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/ovh/cds/sdk" +) + +// Tags returns list of tags for a repo +func (client *bitbucketcloudClient) Tags(ctx context.Context, fullname string) ([]sdk.VCSTag, error) { + var tags []Tag + path := fmt.Sprintf("/repositories/%s/refs/tags", fullname) + params := url.Values{} + params.Set("pagelen", "100") + + var response Tags + if err := client.do(ctx, "GET", "core", path, params, nil, &response); err != nil { + return nil, sdk.WrapError(err, "Unable to get tags") + } + if cap(tags) == 0 { + tags = make([]Tag, 0, response.Size) + } + + tags = append(tags, response.Values...) + + responseTags := make([]sdk.VCSTag, 0, len(tags)) + for _, tag := range tags { + email := strings.Trim(rawEmailCommitRegexp.FindString(tag.Target.Author.Raw), "<>") + t := sdk.VCSTag{ + Tag: tag.Name, + Hash: tag.Target.Hash, + Message: tag.Message, + Sha: tag.Target.Hash, + Tagger: sdk.VCSAuthor{ + Avatar: tag.Target.Author.User.Links.Avatar.Href, + DisplayName: tag.Target.Author.User.DisplayName, + Email: email, + Name: tag.Target.Author.User.Nickname, + }, + } + responseTags = append(responseTags, t) + } + + return responseTags, nil +} diff --git a/engine/vcs/bitbucketcloud/client_user.go b/engine/vcs/bitbucketcloud/client_user.go new file mode 100644 index 0000000000..b66a317098 --- /dev/null +++ b/engine/vcs/bitbucketcloud/client_user.go @@ -0,0 +1,56 @@ +package bitbucketcloud + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/ovh/cds/engine/api/cache" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +// User Get a single user +func (client *bitbucketcloudClient) User(ctx context.Context, username string) (User, error) { + var user User + url := fmt.Sprintf("/users/%s", username) + status, body, _, err := client.get(url) + if err != nil { + log.Warning("bitbucketcloudClient.User> Error %s", err) + return user, err + } + if status >= 400 { + return user, sdk.NewError(sdk.ErrRepoNotFound, errorAPI(body)) + } + if err := json.Unmarshal(body, &user); err != nil { + log.Warning("bitbucketcloudClient.User> Unable to parse bitbucket cloud commit: %s", err) + return user, err + } + + return user, nil +} + +// User Get a current user +func (client *bitbucketcloudClient) CurrentUser(ctx context.Context) (User, error) { + var user User + url := "/user" + cacheKey := cache.Key("vcs", "bitbucketcloud", "users", client.OAuthToken, url) + + if !client.Cache.Get(cacheKey, &user) { + status, body, _, err := client.get(url) + if err != nil { + log.Warning("bitbucketcloudClient.CurrentUser> Error %s", err) + return user, sdk.WithStack(err) + } + if status >= 400 { + return user, sdk.NewError(sdk.ErrUserNotFound, errorAPI(body)) + } + if err := json.Unmarshal(body, &user); err != nil { + return user, sdk.WithStack(err) + } + //Put the body on cache for 1 hour + client.Cache.SetWithTTL(cacheKey, user, 60*60) + } + + return user, nil +} diff --git a/engine/vcs/bitbucketcloud/error.go b/engine/vcs/bitbucketcloud/error.go new file mode 100644 index 0000000000..18be96441e --- /dev/null +++ b/engine/vcs/bitbucketcloud/error.go @@ -0,0 +1,41 @@ +package bitbucketcloud + +import ( + "encoding/json" + "fmt" +) + +//Error wraps bitbucketcloud error format +type Error struct { + Type string `json:"type"` + ErrorStruct ErrorDetails `json:"error"` +} + +type ErrorDetails struct { + Details string `json:"details"` + Message string `json:"message"` +} + +func (e Error) Error() string { + return fmt.Sprintf("(bitbucketcloud_%s) %s", e.Type, e.ErrorStruct.Message) +} + +func (e Error) String() string { + return e.Error() +} + +var ( + ErrorUnauthorized = &Error{ + Type: "bad_credentials", + ErrorStruct: ErrorDetails{ + Message: "Bad credentials", + }, + } +) + +//errorAPI creates a new error +func errorAPI(body []byte) error { + var res Error + _ = json.Unmarshal(body, &res) + return res +} diff --git a/engine/vcs/bitbucketcloud/http.go b/engine/vcs/bitbucketcloud/http.go new file mode 100644 index 0000000000..fc3dd55000 --- /dev/null +++ b/engine/vcs/bitbucketcloud/http.go @@ -0,0 +1,218 @@ +package bitbucketcloud + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + "time" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/cdsclient" + "github.com/ovh/cds/sdk/log" +) + +//Github http var +var ( + httpClient = cdsclient.NewHTTPClient(time.Second*30, false) +) + +func (consumer *bitbucketcloudConsumer) postForm(url string, data url.Values, headers map[string][]string) (int, []byte, error) { + body := strings.NewReader(data.Encode()) + + req, err := http.NewRequest(http.MethodPost, url, body) + if err != nil { + return 0, nil, err + } + req.SetBasicAuth(consumer.ClientID, consumer.ClientSecret) + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + for k, h := range headers { + for i := range h { + req.Header.Add(k, h[i]) + } + } + + res, err := httpClient.Do(req) + if err != nil { + return 0, nil, err + } + defer res.Body.Close() + resBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return res.StatusCode, nil, err + } + + if res.StatusCode > 400 { + var errBb Error + if err := json.Unmarshal(resBody, &errBb); err == nil { + return res.StatusCode, resBody, errBb + } + } + + return res.StatusCode, resBody, nil +} + +type postOptions struct { + skipDefaultBaseURL bool + asUser bool +} + +func (client *bitbucketcloudClient) post(path string, bodyType string, body io.Reader, opts *postOptions) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, rootURL+path, body) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", bodyType) + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", client.OAuthToken)) + + log.Debug("Bitbucket Cloud API>> Request URL %s", req.URL.String()) + + return httpClient.Do(req) +} + +func (client *bitbucketcloudClient) put(path string, bodyType string, body io.Reader, opts *postOptions) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, rootURL+path, body) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", bodyType) + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", client.OAuthToken)) + + log.Debug("Bitbucket Cloud API>> Request URL %s", req.URL.String()) + + return httpClient.Do(req) +} + +func (client *bitbucketcloudClient) get(path string) (int, []byte, http.Header, error) { + callURL, err := url.ParseRequestURI(rootURL + path) + if err != nil { + return 0, nil, nil, sdk.WithStack(err) + } + + req, err := http.NewRequest(http.MethodGet, callURL.String(), nil) + if err != nil { + return 0, nil, nil, sdk.WithStack(err) + } + + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", client.OAuthToken)) + + res, err := httpClient.Do(req) + if err != nil { + return 0, nil, nil, err + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusNotModified: + return res.StatusCode, nil, res.Header, nil + case http.StatusMovedPermanently, http.StatusTemporaryRedirect, http.StatusFound: + location := res.Header.Get("Location") + if location != "" { + log.Debug("Bitbucket Cloud API>> Response Follow redirect :%s", location) + return client.get(location) + } + case http.StatusUnauthorized: + return res.StatusCode, nil, nil, sdk.WithStack(ErrorUnauthorized) + } + + resBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return res.StatusCode, nil, nil, sdk.WithStack(err) + } + + return res.StatusCode, resBody, res.Header, nil +} + +func (client *bitbucketcloudClient) delete(path string) error { + req, err := http.NewRequest(http.MethodDelete, rootURL+path, nil) + if err != nil { + return sdk.WithStack(err) + } + + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", client.OAuthToken)) + log.Debug("Bitbucket Cloud API>> Request URL %s", req.URL.String()) + + res, err := httpClient.Do(req) + if err != nil { + return sdk.WrapError(err, "Cannot do delete request") + } + + if res.StatusCode != 204 { + return fmt.Errorf("Bitbucket cloud>delete wrong status code %d on url %s", res.StatusCode, path) + } + return nil +} + +func (client *bitbucketcloudClient) do(ctx context.Context, method, api, path string, params url.Values, values []byte, v interface{}) error { + // create the URI + uri, err := url.Parse(rootURL + path) + if err != nil { + return sdk.WithStack(err) + } + + if params != nil && len(params) > 0 { + uri.RawQuery = params.Encode() + } + + // create the request + req := &http.Request{ + URL: uri, + Method: method, + ProtoMajor: 1, + ProtoMinor: 1, + Close: true, + Header: http.Header{}, + } + + if len(values) > 0 { + buf := bytes.NewBuffer(values) + req.Body = ioutil.NopCloser(buf) + req.ContentLength = int64(buf.Len()) + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", client.OAuthToken)) + + // ensure the appropriate content-type is set for POST, + // assuming the field is not populated + if (req.Method == "POST" || req.Method == "PUT") && len(req.Header.Get("Content-Type")) == 0 { + req.Header.Set("Content-Type", "application/json") + } + + // make the request using the default http client + resp, err := httpClient.Do(req) + if err != nil { + return sdk.WrapError(err, "HTTP Error") + } + + // Read the bytes from the body (make sure we defer close the body) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return sdk.WithStack(err) + } + + // Check for an http error status (ie not 200 StatusOK) + switch resp.StatusCode { + case 404: + return sdk.WithStack(sdk.ErrNotFound) + case 403: + return sdk.WithStack(sdk.ErrForbidden) + case 401: + return sdk.WithStack(sdk.ErrUnauthorized) + case 400: + log.Warning("bitbucketClient.do> %s", string(body)) + return sdk.WithStack(sdk.ErrWrongRequest) + } + + return sdk.WithStack(json.Unmarshal(body, v)) +} diff --git a/engine/vcs/bitbucketcloud/oauth_consumer.go b/engine/vcs/bitbucketcloud/oauth_consumer.go new file mode 100644 index 0000000000..9ac91baadb --- /dev/null +++ b/engine/vcs/bitbucketcloud/oauth_consumer.go @@ -0,0 +1,136 @@ +package bitbucketcloud + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "time" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +const ( + bitbucketCloudAccessTokenURL = "https://bitbucket.org/site/oauth2/access_token" +) + +//AuthorizeRedirect returns the request token, the Authorize Bitbucket cloud +func (consumer *bitbucketcloudConsumer) AuthorizeRedirect(ctx context.Context) (string, string, error) { + requestToken, err := sdk.GenerateHash() + if err != nil { + return "", "", err + } + + val := url.Values{} + val.Add("client_id", consumer.ClientID) + val.Add("response_type", "code") + + authorizeURL := fmt.Sprintf("https://bitbucket.org/site/oauth2/authorize?%s", val.Encode()) + + return requestToken, authorizeURL, nil +} + +//AuthorizeToken returns the authorized token (and its refresh_token) +//from the request token and the verifier got on authorize url +func (consumer *bitbucketcloudConsumer) AuthorizeToken(ctx context.Context, _, code string) (string, string, error) { + log.Debug("AuthorizeToken> Bitbucketcloud send code %s", code) + + params := url.Values{} + params.Add("code", code) + params.Add("grant_type", "authorization_code") + + headers := map[string][]string{} + headers["Accept"] = []string{"application/json"} + + status, res, err := consumer.postForm(bitbucketCloudAccessTokenURL, params, headers) + if err != nil { + return "", "", err + } + + if status < 200 || status >= 400 { + return "", "", fmt.Errorf("Bitbucket cloud error (%d) %s ", status, string(res)) + } + + var resp AccessToken + if err := json.Unmarshal(res, &resp); err != nil { + return "", "", fmt.Errorf("Unable to parse bitbucketcloud response (%d) %s ", status, string(res)) + } + + return resp.AccessToken, resp.RefreshToken, nil +} + +//RefreshToken returns the refreshed authorized token +func (consumer *bitbucketcloudConsumer) RefreshToken(ctx context.Context, refreshToken string) (string, string, error) { + params := url.Values{} + params.Add("refresh_token", refreshToken) + params.Add("grant_type", "refresh_token") + + headers := map[string][]string{} + headers["Accept"] = []string{"application/json"} + + status, res, err := consumer.postForm(bitbucketCloudAccessTokenURL, params, headers) + if err != nil { + return "", "", err + } + + if status < 200 || status >= 400 { + return "", "", fmt.Errorf("Bitbucket cloud error (%d) %s ", status, string(res)) + } + + var resp AccessToken + if err := json.Unmarshal(res, &resp); err != nil { + return "", "", fmt.Errorf("Unable to parse bitbucketcloud response (%d) %s ", status, string(res)) + } + + return resp.AccessToken, resp.RefreshToken, nil +} + +//keep client in memory +var instancesAuthorizedClient = map[string]*bitbucketcloudClient{} + +//GetAuthorized returns an authorized client +func (consumer *bitbucketcloudConsumer) GetAuthorizedClient(ctx context.Context, accessToken, refreshToken string, created int64) (sdk.VCSAuthorizedClient, error) { + createdTime := time.Unix(created, 0) + + c, ok := instancesAuthorizedClient[accessToken] + if createdTime.Add(2 * time.Hour).Before(time.Now()) { + if ok { + delete(instancesAuthorizedClient, accessToken) + } + newAccessToken, _, err := consumer.RefreshToken(ctx, refreshToken) + if err != nil { + return nil, sdk.WrapError(err, "cannot refresh token") + } + c = &bitbucketcloudClient{ + ClientID: consumer.ClientID, + OAuthToken: newAccessToken, + RefreshToken: refreshToken, + Cache: consumer.Cache, + apiURL: consumer.apiURL, + uiURL: consumer.uiURL, + DisableStatus: consumer.disableStatus, + DisableStatusDetail: consumer.disableStatusDetail, + proxyURL: consumer.proxyURL, + } + instancesAuthorizedClient[newAccessToken] = c + } else { + if !ok { + c = &bitbucketcloudClient{ + ClientID: consumer.ClientID, + OAuthToken: accessToken, + RefreshToken: refreshToken, + Cache: consumer.Cache, + apiURL: consumer.apiURL, + uiURL: consumer.uiURL, + DisableStatus: consumer.disableStatus, + DisableStatusDetail: consumer.disableStatusDetail, + proxyURL: consumer.proxyURL, + } + instancesAuthorizedClient[accessToken] = c + } + + } + + return c, nil +} diff --git a/engine/vcs/bitbucketcloud/status.go b/engine/vcs/bitbucketcloud/status.go new file mode 100644 index 0000000000..4217fc6a4e --- /dev/null +++ b/engine/vcs/bitbucketcloud/status.go @@ -0,0 +1,10 @@ +package bitbucketcloud + +import ( + "github.com/ovh/cds/sdk" +) + +// GetStatus returns bitbucketcloud status +func GetStatus() []sdk.MonitoringStatusLine { + return []sdk.MonitoringStatusLine{} +} diff --git a/engine/vcs/bitbucketcloud/types.go b/engine/vcs/bitbucketcloud/types.go new file mode 100644 index 0000000000..34dfbdd519 --- /dev/null +++ b/engine/vcs/bitbucketcloud/types.go @@ -0,0 +1,467 @@ +package bitbucketcloud + +import ( + "regexp" + "time" + + "github.com/ovh/cds/sdk" +) + +var ( + _ sdk.VCSAuthorizedClient = &bitbucketcloudClient{} + _ sdk.VCSServer = &bitbucketcloudConsumer{} + rawEmailCommitRegexp = regexp.MustCompile(`<(.*)>`) +) + +// WebhookCreate represent struct to create a webhook +type WebhookCreate struct { + Description string `json:"description"` + URL string `json:"url"` + Active bool `json:"active"` + Events []string `json:"events"` +} + +type Webhook struct { + ReadOnly bool `json:"read_only"` + Description string `json:"description"` + Links struct { + Self Link `json:"self"` + } `json:"links"` + URL string `json:"url"` + CreatedAt time.Time `json:"created_at"` + SkipCertVerification bool `json:"skip_cert_verification"` + Source string `json:"source"` + HistoryEnabled bool `json:"history_enabled"` + Active bool `json:"active"` + Subject struct { + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + } `json:"links"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + } `json:"subject"` + Type string `json:"type"` + Events []string `json:"events"` + UUID string `json:"uuid"` +} + +type Webhooks struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []Webhook `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +// User represents a public bitbucketcloud user. +type User struct { + Username string `json:"username"` + Website string `json:"website"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Hooks Link `json:"hooks"` + Self Link `json:"self"` + Repositories Link `json:"repositories"` + HTML Link `json:"html"` + Followers Link `json:"followers"` + Avatar Link `json:"avatar"` + Following Link `json:"following"` + Snippets Link `json:"snippets"` + } `json:"links"` + Nickname string `json:"nickname"` + CreatedOn time.Time `json:"created_on"` + IsStaff bool `json:"is_staff"` + Location string `json:"location"` + AccountStatus string `json:"account_status"` + Type string `json:"type"` + AccountID string `json:"account_id"` +} + +// PullRequest represents pull request from github api +type PullRequest struct { + Description string `json:"description"` + Links struct { + Decline Link `json:"decline"` + Commits Link `json:"commits"` + Self Link `json:"self"` + Comments Link `json:"comments"` + Merge Link `json:"merge"` + HTML Link `json:"html"` + Activity Link `json:"activity"` + Diff Link `json:"diff"` + Approve Link `json:"approve"` + Statuses Link `json:"statuses"` + } `json:"links"` + Title string `json:"title"` + CloseSourceBranch bool `json:"close_source_branch"` + Type string `json:"type"` + ID int `json:"id"` + Destination struct { + Commit struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + } `json:"links"` + } `json:"commit"` + Repository struct { + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + } `json:"links"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + } `json:"repository"` + Branch struct { + Name string `json:"name"` + } `json:"branch"` + } `json:"destination"` + CreatedOn time.Time `json:"created_on"` + Summary struct { + Raw string `json:"raw"` + Markup string `json:"markup"` + HTML string `json:"html"` + Type string `json:"type"` + } `json:"summary"` + Source struct { + Commit struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + } `json:"links"` + } `json:"commit"` + Repository struct { + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + } `json:"links"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + } `json:"repository"` + Branch struct { + Name string `json:"name"` + } `json:"branch"` + } `json:"source"` + CommentCount int `json:"comment_count"` + State string `json:"state"` + TaskCount int `json:"task_count"` + Reason string `json:"reason"` + UpdatedOn time.Time `json:"updated_on"` + Author User `json:"author"` + MergeCommit struct { + Hash string `json:"hash"` + } `json:"merge_commit"` +} + +type PullRequests struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []PullRequest `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +type AccessToken struct { + AccessToken string `json:"access_token"` + Scopes string `json:"scopes"` + ExpiresIn int64 `json:"expires_in"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` +} + +type Link struct { + Href string `json:"href"` + Name string `json:"name"` +} + +type Repositories struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []Repository `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +type Repository struct { + Scm string `json:"scm"` + Website string `json:"website"` + HasWiki bool `json:"has_wiki"` + Name string `json:"name"` + Links struct { + Watchers Link `json:"watchers"` + Branches Link `json:"branches"` + Tags Link `json:"tags"` + Commits Link `json:"commits"` + Clone []Link `json:"clone"` + Self Link `json:"self"` + Source Link `json:"source"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + Hooks Link `json:"hooks"` + Forks Link `json:"forks"` + Downloads Link `json:"downloads"` + Issues Link `json:"issues"` + Pullrequests Link `json:"pullrequests"` + } `json:"links"` + ForkPolicy string `json:"fork_policy"` + UUID string `json:"uuid"` + Language string `json:"language"` + CreatedOn time.Time `json:"created_on"` + Mainbranch struct { + Type string `json:"type"` + Name string `json:"name"` + } `json:"mainbranch"` + FullName string `json:"full_name"` + HasIssues bool `json:"has_issues"` + Owner User `json:"owner"` + UpdatedOn time.Time `json:"updated_on"` + Size int `json:"size"` + Type string `json:"type"` + Slug string `json:"slug"` + IsPrivate bool `json:"is_private"` + Description string `json:"description"` +} + +type Status struct { + UUID string `json:"uuid"` + Key string `json:"key"` + RefName string `json:"refname"` //optional + URL string `json:"url"` + State string `json:"state"` // SUCCESSFUL / FAILED / INPROGRESS / STOPPED + Name string `json:"name"` + Description string `json:"description"` + CreatedOn time.Time `json:"created_on"` + UpdatedOn time.Time `json:"updated_on"` + Links struct { + Self Link `json:"self"` + Commit Link `json:"commit"` + } `json:"links"` +} + +type Statuses struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []Status `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +type Branches struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []Branch `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +type Branch struct { + Heads []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links Link `json:"links"` + } `json:"heads"` + Name string `json:"name"` + Links struct { + Commits Link `json:"commits"` + Self Link `json:"self"` + HTML Link `json:"html"` + } `json:"links"` + DefaultMergeStrategy string `json:"default_merge_strategy"` + MergeStrategies []string `json:"merge_strategies"` + Type string `json:"type"` + Target struct { + Hash string `json:"hash"` + Repository struct { + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + } `json:"links"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + } `json:"repository"` + Links struct { + Self Link `json:"self"` + Comments Link `json:"comments"` + Patch Link `json:"patch"` + HTML Link `json:"html"` + Diff Link `json:"diff"` + Approve Link `json:"approve"` + Statuses Link `json:"statuses"` + } `json:"links"` + Author struct { + Raw string `json:"raw"` + Type string `json:"type"` + } `json:"author"` + Parents []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + } `json:"links"` + } `json:"parents"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` + } `json:"target"` +} + +type Commits struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []Commit `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +type Commit struct { + Rendered struct { + Message struct { + Raw string `json:"raw"` + Markup string `json:"markup"` + HTML string `json:"html"` + Type string `json:"type"` + } `json:"message"` + } `json:"rendered"` + Hash string `json:"hash"` + Repository struct { + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + } `json:"links"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + } `json:"repository"` + Links struct { + Self Link `json:"self"` + Comments Link `json:"comments"` + Patch Link `json:"patch"` + HTML Link `json:"html"` + Diff Link `json:"diff"` + Approve Link `json:"approve"` + Statuses Link `json:"statuses"` + } `json:"links"` + Author struct { + Raw string `json:"raw"` + Type string `json:"type"` + User User `json:"user"` + } `json:"author,omitempty"` + Summary struct { + Raw string `json:"raw"` + Markup string `json:"markup"` + HTML string `json:"html"` + Type string `json:"type"` + } `json:"summary"` + Parents []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links Link `json:"links"` + } `json:"parents"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` +} + +type Tags struct { + Pagelen int `json:"pagelen"` + Page int `json:"page"` + Size int64 `json:"size"` + Values []Tag `json:"values"` + Next string `json:"next"` + Previous string `json:"previous,omitempty"` +} + +type Tag struct { + Name string `json:"name"` + Links struct { + Commits Link `json:"commits"` + Self Link `json:"self"` + HTML Link `json:"html"` + } `json:"links"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` + Target struct { + Hash string `json:"hash"` + Repository struct { + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + Avatar Link `json:"avatar"` + } `json:"links"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + UUID string `json:"uuid"` + } `json:"repository"` + Links struct { + Self Link `json:"self"` + Comments Link `json:"comments"` + Patch Link `json:"patch"` + HTML Link `json:"html"` + Diff Link `json:"diff"` + Approve Link `json:"approve"` + Statuses Link `json:"statuses"` + } `json:"links"` + Author struct { + Raw string `json:"raw"` + Type string `json:"type"` + User struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + UUID string `json:"uuid"` + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + HTML struct { + Href string `json:"href"` + } `json:"html"` + Avatar struct { + Href string `json:"href"` + } `json:"avatar"` + } `json:"links"` + Nickname string `json:"nickname"` + Type string `json:"type"` + AccountID string `json:"account_id"` + } `json:"user"` + } `json:"author"` + Parents []struct { + Hash string `json:"hash"` + Type string `json:"type"` + Links struct { + Self Link `json:"self"` + HTML Link `json:"html"` + } `json:"links"` + } `json:"parents"` + Date time.Time `json:"date"` + Message string `json:"message"` + Type string `json:"type"` + } `json:"target"` +} diff --git a/engine/vcs/bitbucket/bitbucket.go b/engine/vcs/bitbucketserver/bitbucketserver.go similarity index 93% rename from engine/vcs/bitbucket/bitbucket.go rename to engine/vcs/bitbucketserver/bitbucketserver.go index 35374c4773..8c74ce092e 100644 --- a/engine/vcs/bitbucket/bitbucket.go +++ b/engine/vcs/bitbucketserver/bitbucketserver.go @@ -1,6 +1,7 @@ -package bitbucket +package bitbucketserver import ( + "context" "fmt" "strings" @@ -65,3 +66,7 @@ func getRepo(fullname string) (string, string, error) { slug := t[1] return project, slug, nil } + +func (c *bitbucketClient) GetAccessToken(_ context.Context) string { + return c.accessToken +} diff --git a/engine/vcs/bitbucket/bitbucket_test.go b/engine/vcs/bitbucketserver/bitbucketserver_test.go similarity index 98% rename from engine/vcs/bitbucket/bitbucket_test.go rename to engine/vcs/bitbucketserver/bitbucketserver_test.go index 76ea4defb6..0a2df801e7 100644 --- a/engine/vcs/bitbucket/bitbucket_test.go +++ b/engine/vcs/bitbucketserver/bitbucketserver_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" @@ -97,7 +97,7 @@ func getAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { } consumer := New(consumerKey, []byte(privateKey), url, "", "", "", username, password, cache, true) - cli, err := consumer.GetAuthorizedClient(context.Background(), token, secret) + cli, err := consumer.GetAuthorizedClient(context.Background(), token, secret, 0) test.NoError(t, err) return cli } diff --git a/engine/vcs/bitbucket/client_branch.go b/engine/vcs/bitbucketserver/client_branch.go similarity index 98% rename from engine/vcs/bitbucket/client_branch.go rename to engine/vcs/bitbucketserver/client_branch.go index c68864187a..21f37a7a18 100644 --- a/engine/vcs/bitbucket/client_branch.go +++ b/engine/vcs/bitbucketserver/client_branch.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_branch_test.go b/engine/vcs/bitbucketserver/client_branch_test.go similarity index 95% rename from engine/vcs/bitbucket/client_branch_test.go rename to engine/vcs/bitbucketserver/client_branch_test.go index 21c0055d84..0a48334c20 100644 --- a/engine/vcs/bitbucket/client_branch_test.go +++ b/engine/vcs/bitbucketserver/client_branch_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_commit.go b/engine/vcs/bitbucketserver/client_commit.go similarity index 99% rename from engine/vcs/bitbucket/client_commit.go rename to engine/vcs/bitbucketserver/client_commit.go index 173176faf1..d13916fb52 100644 --- a/engine/vcs/bitbucket/client_commit.go +++ b/engine/vcs/bitbucketserver/client_commit.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_commit_test.go b/engine/vcs/bitbucketserver/client_commit_test.go similarity index 95% rename from engine/vcs/bitbucket/client_commit_test.go rename to engine/vcs/bitbucketserver/client_commit_test.go index be07a18b16..97b7a84647 100644 --- a/engine/vcs/bitbucket/client_commit_test.go +++ b/engine/vcs/bitbucketserver/client_commit_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_event.go b/engine/vcs/bitbucketserver/client_event.go similarity index 97% rename from engine/vcs/bitbucket/client_event.go rename to engine/vcs/bitbucketserver/client_event.go index 928c7c2cb4..85a80fd1b0 100644 --- a/engine/vcs/bitbucket/client_event.go +++ b/engine/vcs/bitbucketserver/client_event.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_forks.go b/engine/vcs/bitbucketserver/client_forks.go similarity index 98% rename from engine/vcs/bitbucket/client_forks.go rename to engine/vcs/bitbucketserver/client_forks.go index 0a765c3744..72da0aefbe 100644 --- a/engine/vcs/bitbucket/client_forks.go +++ b/engine/vcs/bitbucketserver/client_forks.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_hook.go b/engine/vcs/bitbucketserver/client_hook.go similarity index 99% rename from engine/vcs/bitbucket/client_hook.go rename to engine/vcs/bitbucketserver/client_hook.go index da9d2435c4..b2ac53cae2 100644 --- a/engine/vcs/bitbucket/client_hook.go +++ b/engine/vcs/bitbucketserver/client_hook.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_hook_test.go b/engine/vcs/bitbucketserver/client_hook_test.go similarity index 96% rename from engine/vcs/bitbucket/client_hook_test.go rename to engine/vcs/bitbucketserver/client_hook_test.go index 9720674dd5..a7b3b762c6 100644 --- a/engine/vcs/bitbucket/client_hook_test.go +++ b/engine/vcs/bitbucketserver/client_hook_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_pull_request.go b/engine/vcs/bitbucketserver/client_pull_request.go similarity index 99% rename from engine/vcs/bitbucket/client_pull_request.go rename to engine/vcs/bitbucketserver/client_pull_request.go index 04dd66fa29..518d6266f7 100644 --- a/engine/vcs/bitbucket/client_pull_request.go +++ b/engine/vcs/bitbucketserver/client_pull_request.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_pull_request_test.go b/engine/vcs/bitbucketserver/client_pull_request_test.go similarity index 96% rename from engine/vcs/bitbucket/client_pull_request_test.go rename to engine/vcs/bitbucketserver/client_pull_request_test.go index 1c0b062391..3d7772be19 100644 --- a/engine/vcs/bitbucket/client_pull_request_test.go +++ b/engine/vcs/bitbucketserver/client_pull_request_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_release.go b/engine/vcs/bitbucketserver/client_release.go similarity index 95% rename from engine/vcs/bitbucket/client_release.go rename to engine/vcs/bitbucketserver/client_release.go index 32492a8585..ce3f92bd36 100644 --- a/engine/vcs/bitbucket/client_release.go +++ b/engine/vcs/bitbucketserver/client_release.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_repos.go b/engine/vcs/bitbucketserver/client_repos.go similarity index 99% rename from engine/vcs/bitbucket/client_repos.go rename to engine/vcs/bitbucketserver/client_repos.go index df2364ad7d..1759275040 100644 --- a/engine/vcs/bitbucket/client_repos.go +++ b/engine/vcs/bitbucketserver/client_repos.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_repos_test.go b/engine/vcs/bitbucketserver/client_repos_test.go similarity index 96% rename from engine/vcs/bitbucket/client_repos_test.go rename to engine/vcs/bitbucketserver/client_repos_test.go index 6f1b10b295..228cd2e79c 100644 --- a/engine/vcs/bitbucket/client_repos_test.go +++ b/engine/vcs/bitbucketserver/client_repos_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_status.go b/engine/vcs/bitbucketserver/client_status.go similarity index 99% rename from engine/vcs/bitbucket/client_status.go rename to engine/vcs/bitbucketserver/client_status.go index 308b4cea18..c481dfe3ea 100644 --- a/engine/vcs/bitbucket/client_status.go +++ b/engine/vcs/bitbucketserver/client_status.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_status_test.go b/engine/vcs/bitbucketserver/client_status_test.go similarity index 92% rename from engine/vcs/bitbucket/client_status_test.go rename to engine/vcs/bitbucketserver/client_status_test.go index 904a93f014..5799975e2a 100644 --- a/engine/vcs/bitbucket/client_status_test.go +++ b/engine/vcs/bitbucketserver/client_status_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_tags.go b/engine/vcs/bitbucketserver/client_tags.go similarity index 97% rename from engine/vcs/bitbucket/client_tags.go rename to engine/vcs/bitbucketserver/client_tags.go index 97f8f15ed0..5f7863b387 100644 --- a/engine/vcs/bitbucket/client_tags.go +++ b/engine/vcs/bitbucketserver/client_tags.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/client_user.go b/engine/vcs/bitbucketserver/client_user.go similarity index 95% rename from engine/vcs/bitbucket/client_user.go rename to engine/vcs/bitbucketserver/client_user.go index 03378be0f4..32a426c24c 100644 --- a/engine/vcs/bitbucket/client_user.go +++ b/engine/vcs/bitbucketserver/client_user.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" diff --git a/engine/vcs/bitbucket/http.go b/engine/vcs/bitbucketserver/http.go similarity index 99% rename from engine/vcs/bitbucket/http.go rename to engine/vcs/bitbucketserver/http.go index c1a4a81ac0..201b2cbaaf 100644 --- a/engine/vcs/bitbucket/http.go +++ b/engine/vcs/bitbucketserver/http.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "bytes" diff --git a/engine/vcs/bitbucket/oauth_consumer.go b/engine/vcs/bitbucketserver/oauth_consumer.go similarity index 95% rename from engine/vcs/bitbucket/oauth_consumer.go rename to engine/vcs/bitbucketserver/oauth_consumer.go index 9ecc5e38fe..9bdfaa2b89 100644 --- a/engine/vcs/bitbucket/oauth_consumer.go +++ b/engine/vcs/bitbucketserver/oauth_consumer.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "context" @@ -69,7 +69,7 @@ func (g *bitbucketConsumer) AuthorizeToken(ctx context.Context, token, verifier var instancesAuthorizedClient = map[string]*bitbucketClient{} //GetAuthorized returns an authorized client -func (g *bitbucketConsumer) GetAuthorizedClient(ctx context.Context, accessToken, accessTokenSecret string) (sdk.VCSAuthorizedClient, error) { +func (g *bitbucketConsumer) GetAuthorizedClient(ctx context.Context, accessToken, accessTokenSecret string, _ int64) (sdk.VCSAuthorizedClient, error) { c, ok := instancesAuthorizedClient[accessToken] if !ok { c = &bitbucketClient{ diff --git a/engine/vcs/bitbucket/oauth_consumer_helper.go b/engine/vcs/bitbucketserver/oauth_consumer_helper.go similarity index 98% rename from engine/vcs/bitbucket/oauth_consumer_helper.go rename to engine/vcs/bitbucketserver/oauth_consumer_helper.go index ec307bedde..212c27630d 100644 --- a/engine/vcs/bitbucket/oauth_consumer_helper.go +++ b/engine/vcs/bitbucketserver/oauth_consumer_helper.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "fmt" diff --git a/engine/vcs/bitbucket/oauth_consumer_sign.go b/engine/vcs/bitbucketserver/oauth_consumer_sign.go similarity index 99% rename from engine/vcs/bitbucket/oauth_consumer_sign.go rename to engine/vcs/bitbucketserver/oauth_consumer_sign.go index 7adc2b4c41..9d42e7d62a 100644 --- a/engine/vcs/bitbucket/oauth_consumer_sign.go +++ b/engine/vcs/bitbucketserver/oauth_consumer_sign.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "crypto" diff --git a/engine/vcs/bitbucket/oauth_consumer_token.go b/engine/vcs/bitbucketserver/oauth_consumer_token.go similarity index 99% rename from engine/vcs/bitbucket/oauth_consumer_token.go rename to engine/vcs/bitbucketserver/oauth_consumer_token.go index e89a0d6d4d..5c7f6f4d67 100644 --- a/engine/vcs/bitbucket/oauth_consumer_token.go +++ b/engine/vcs/bitbucketserver/oauth_consumer_token.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "errors" diff --git a/engine/vcs/bitbucket/types.go b/engine/vcs/bitbucketserver/types.go similarity index 99% rename from engine/vcs/bitbucket/types.go rename to engine/vcs/bitbucketserver/types.go index f5b678f4a2..0a1e908c2c 100644 --- a/engine/vcs/bitbucket/types.go +++ b/engine/vcs/bitbucketserver/types.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "encoding/json" diff --git a/engine/vcs/bitbucket/types_test.go b/engine/vcs/bitbucketserver/types_test.go similarity index 97% rename from engine/vcs/bitbucket/types_test.go rename to engine/vcs/bitbucketserver/types_test.go index fa9a4b5bb5..efbc8fb688 100644 --- a/engine/vcs/bitbucket/types_test.go +++ b/engine/vcs/bitbucketserver/types_test.go @@ -1,4 +1,4 @@ -package bitbucket +package bitbucketserver import ( "encoding/json" diff --git a/engine/vcs/gerrit/gerrit.go b/engine/vcs/gerrit/gerrit.go index 31abe42864..fcfc178a79 100644 --- a/engine/vcs/gerrit/gerrit.go +++ b/engine/vcs/gerrit/gerrit.go @@ -1,6 +1,8 @@ package gerrit import ( + "context" + g "github.com/andygrunwald/go-gerrit" "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/sdk" @@ -41,3 +43,7 @@ func New(URL string, store cache.Store, disableStatus bool, disableStatusDetail reviewerToken: reviewerToken, } } + +func (c *gerritClient) GetAccessToken(_ context.Context) string { + return c.reviewerToken +} diff --git a/engine/vcs/gerrit/gerrit_consumer.go b/engine/vcs/gerrit/gerrit_consumer.go index e1b1e8d4a0..911340daf7 100644 --- a/engine/vcs/gerrit/gerrit_consumer.go +++ b/engine/vcs/gerrit/gerrit_consumer.go @@ -21,7 +21,7 @@ func (g *gerritConsumer) AuthorizeToken(ctx context.Context, state, code string) } //GetAuthorized returns an authorized client -func (g *gerritConsumer) GetAuthorizedClient(ctx context.Context, username, password string) (sdk.VCSAuthorizedClient, error) { +func (g *gerritConsumer) GetAuthorizedClient(ctx context.Context, username, password string, _ int64) (sdk.VCSAuthorizedClient, error) { client, err := ger.NewClient(g.URL, nil) if err != nil { return nil, sdk.WrapError(err, "unable to create gerrit client") diff --git a/engine/vcs/github/github.go b/engine/vcs/github/github.go index 75686915d3..6d24ed0ca1 100644 --- a/engine/vcs/github/github.go +++ b/engine/vcs/github/github.go @@ -1,6 +1,8 @@ package github import ( + "context" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/sdk" ) @@ -67,3 +69,7 @@ func New(ClientID, ClientSecret, githubURL, githubAPIURL, apiURL, uiURL, proxyUR token: token, } } + +func (c *githubClient) GetAccessToken(_ context.Context) string { + return c.OAuthToken +} diff --git a/engine/vcs/github/github_test.go b/engine/vcs/github/github_test.go index cf5a24553e..2552d59093 100644 --- a/engine/vcs/github/github_test.go +++ b/engine/vcs/github/github_test.go @@ -64,7 +64,7 @@ func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { } ghConsummer := New(clientID, clientSecret, "", "", "http://localhost", "", "", "", "", cache, true, true) - cli, err := ghConsummer.GetAuthorizedClient(context.Background(), accessToken, "") + cli, err := ghConsummer.GetAuthorizedClient(context.Background(), accessToken, "", 0) if err != nil { t.Fatalf("Unable to init authorized client (%s): %v", redisHost, err) } @@ -117,7 +117,7 @@ func TestClientAuthorizeToken(t *testing.T) { t.Logf("Token is %s", accessToken) - ghClient, err := ghConsummer.GetAuthorizedClient(context.Background(), accessToken, accessTokenSecret) + ghClient, err := ghConsummer.GetAuthorizedClient(context.Background(), accessToken, accessTokenSecret, 0) test.NoError(t, err) assert.NotNil(t, ghClient) } diff --git a/engine/vcs/github/oauth_consumer.go b/engine/vcs/github/oauth_consumer.go index ff33dee55c..7b32a9203f 100644 --- a/engine/vcs/github/oauth_consumer.go +++ b/engine/vcs/github/oauth_consumer.go @@ -62,7 +62,7 @@ func (g *githubConsumer) AuthorizeToken(ctx context.Context, state, code string) return "", "", err } - if status < 200 && status >= 400 { + if status < 200 || status >= 400 { return "", "", fmt.Errorf("Github error (%d) %s ", status, string(res)) } @@ -81,7 +81,7 @@ func (g *githubConsumer) AuthorizeToken(ctx context.Context, state, code string) var instancesAuthorizedClient = map[string]*githubClient{} //GetAuthorized returns an authorized client -func (g *githubConsumer) GetAuthorizedClient(ctx context.Context, accessToken, accessTokenSecret string) (sdk.VCSAuthorizedClient, error) { +func (g *githubConsumer) GetAuthorizedClient(ctx context.Context, accessToken, accessTokenSecret string, _ int64) (sdk.VCSAuthorizedClient, error) { c, ok := instancesAuthorizedClient[accessToken] if !ok { c = &githubClient{ diff --git a/engine/vcs/gitlab/gitlab.go b/engine/vcs/gitlab/gitlab.go index 1326f67809..2d4d0eb925 100644 --- a/engine/vcs/gitlab/gitlab.go +++ b/engine/vcs/gitlab/gitlab.go @@ -1,6 +1,8 @@ package gitlab import ( + "context" + "github.com/xanzy/go-gitlab" "github.com/ovh/cds/engine/api/cache" @@ -15,6 +17,7 @@ var ( // gitlabClient implements VCSAuthorizedClient interface type gitlabClient struct { client *gitlab.Client + accessToken string uiURL string proxyURL string disableStatus bool @@ -37,14 +40,18 @@ type gitlabConsumer struct { // New instanciate a new gitlab consumer func New(appID, clientSecret, URL, callbackURL, uiURL, proxyURL string, store cache.Store, disableStatus bool, disableStatusDetail bool) sdk.VCSServer { return &gitlabConsumer{ - URL: URL, - secret: clientSecret, - cache: store, - appID: appID, + URL: URL, + secret: clientSecret, + cache: store, + appID: appID, AuthorizationCallbackURL: callbackURL, - uiURL: uiURL, - proxyURL: proxyURL, - disableStatus: disableStatus, - disableStatusDetail: disableStatusDetail, + uiURL: uiURL, + proxyURL: proxyURL, + disableStatus: disableStatus, + disableStatusDetail: disableStatusDetail, } } + +func (c *gitlabClient) GetAccessToken(_ context.Context) string { + return c.accessToken +} diff --git a/engine/vcs/gitlab/gitlab_test.go b/engine/vcs/gitlab/gitlab_test.go index 174a9ee79b..74adc8fa97 100644 --- a/engine/vcs/gitlab/gitlab_test.go +++ b/engine/vcs/gitlab/gitlab_test.go @@ -63,7 +63,7 @@ func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { } glConsummer := New(appID, secret, "https://gitlab.com", "http://localhost:8081", "", "", cache, true, true) - cli, err := glConsummer.GetAuthorizedClient(context.Background(), accessToken, "") + cli, err := glConsummer.GetAuthorizedClient(context.Background(), accessToken, "", 0) if err != nil { t.Fatalf("Unable to init authorized client (%s): %v", redisHost, err) } @@ -116,7 +116,7 @@ func TestClientAuthorizeToken(t *testing.T) { t.Logf("Token is %s", accessToken) - ghClient, err := glConsumer.GetAuthorizedClient(context.Background(), accessToken, accessTokenSecret) + ghClient, err := glConsumer.GetAuthorizedClient(context.Background(), accessToken, accessTokenSecret, 0) test.NoError(t, err) assert.NotNil(t, ghClient) } diff --git a/engine/vcs/gitlab/oauth_consumer.go b/engine/vcs/gitlab/oauth_consumer.go index 0ad55fd869..61eaaa318f 100644 --- a/engine/vcs/gitlab/oauth_consumer.go +++ b/engine/vcs/gitlab/oauth_consumer.go @@ -119,7 +119,7 @@ func (g *gitlabConsumer) AuthorizeToken(ctx context.Context, state, code string) var instancesAuthorizedClient = map[string]*gitlabClient{} //GetAuthorized returns an authorized client -func (g *gitlabConsumer) GetAuthorizedClient(ctx context.Context, accessToken, accessTokenSecret string) (sdk.VCSAuthorizedClient, error) { +func (g *gitlabConsumer) GetAuthorizedClient(ctx context.Context, accessToken, accessTokenSecret string, _created int64) (sdk.VCSAuthorizedClient, error) { c, ok := instancesAuthorizedClient[accessToken] httpClient := &http.Client{ Timeout: 60 * time.Second, diff --git a/engine/vcs/types.go b/engine/vcs/types.go index eac7f82ce8..0694899de6 100644 --- a/engine/vcs/types.go +++ b/engine/vcs/types.go @@ -45,18 +45,19 @@ type Configuration struct { // ServerConfiguration is the configuration for a VCS server type ServerConfiguration struct { - URL string `toml:"url" comment:"URL of this VCS Server" json:"url" json:"url"` - Github *GithubServerConfiguration `toml:"github" json:"github,omitempty" json:"github"` - Gitlab *GitlabServerConfiguration `toml:"gitlab" json:"gitlab,omitempty" json:"gitlab"` - Bitbucket *BitbucketServerConfiguration `toml:"bitbucket" json:"bitbucket,omitempty" json:"bitbucket"` - Gerrit *GerritServerConfiguration `toml:"gerrit" json:"gerrit,omitempty" json:"gerrit"` + URL string `toml:"url" comment:"URL of this VCS Server" json:"url"` + Github *GithubServerConfiguration `toml:"github" json:"github,omitempty"` + Gitlab *GitlabServerConfiguration `toml:"gitlab" json:"gitlab,omitempty"` + Bitbucket *BitbucketServerConfiguration `toml:"bitbucket" json:"bitbucket,omitempty"` + BitbucketCloud *BitbucketCloudConfiguration `toml:"bitbucket_cloud" json:"bitbucket_cloud,omitempty"` + Gerrit *GerritServerConfiguration `toml:"gerrit" json:"gerrit,omitempty"` } // GithubServerConfiguration represents the github configuration type GithubServerConfiguration struct { ClientID string `toml:"clientId" json:"-" default:"xxxxx" comment:"#######\n CDS <-> Github. Documentation on https://ovh.github.io/cds/hosting/repositories-manager/github/ \n#######\n Github OAuth Application Client ID"` ClientSecret string `toml:"clientSecret" json:"-" default:"xxxxx" comment:"Github OAuth Application Client Secret"` - APIURL string `toml:"apiUrl" json:"-" comment:"The URL for the GitHub API."` + APIURL string `toml:"apiUrl" json:"-" default:"https://api.github.com" comment:"The URL for the GitHub API."` Status struct { Disable bool `toml:"disable" default:"false" commented:"true" comment:"Set to true if you don't want CDS to push statuses on the VCS server" json:"disable"` ShowDetail bool `toml:"showDetail" default:"false" commented:"true" comment:"Set to true if you don't want CDS to push CDS URL in statuses on the VCS server" json:"show_detail"` @@ -125,6 +126,28 @@ func (s BitbucketServerConfiguration) check() error { return nil } +// BitbucketCloudConfiguration represents the bitbucket configuration +type BitbucketCloudConfiguration struct { + ClientID string `toml:"clientId" json:"-" default:"xxxxx" comment:"#######\n CDS <-> Bitbucket cloud. Documentation on https://ovh.github.io/cds/hosting/repositories-manager/bitbucketcloud/ \n#######\n Bitbucket cloud OAuth Application Client ID"` + ClientSecret string `toml:"clientSecret" json:"-" default:"xxxxx" comment:"Bitbucket Cloud OAuth Application Client Secret"` + Status struct { + Disable bool `toml:"disable" default:"false" commented:"true" comment:"Set to true if you don't want CDS to push statuses on the VCS server" json:"disable"` + ShowDetail bool `toml:"showDetail" default:"false" commented:"true" comment:"Set to true if you don't want CDS to push CDS URL in statuses on the VCS server" json:"show_detail"` + } + DisableWebHooks bool `toml:"disableWebHooks" comment:"Does webhooks are supported by VCS Server" json:"disable_web_hook"` + // DisablePolling bool `toml:"disablePolling" comment:"Does polling is supported by VCS Server" json:"disable_polling"` + ProxyWebhook string `toml:"proxyWebhook" default:"https://myproxy.com" commented:"true" comment:"If you want to have a reverse proxy url for your repository webhook, for example if you put https://myproxy.com it will generate a webhook URL like this https://myproxy.com/UUID_OF_YOUR_WEBHOOK" json:"proxy_webhook"` + // Username string `toml:"username" comment:"optional. Github username, used to add comment on Pull Request on failed build." json:"username"` + // Token string `toml:"token" comment:"optional, Bitbucket Cloud Token associated to username, used to add comment on Pull Request" json:"-"` +} + +func (s BitbucketCloudConfiguration) check() error { + if s.ProxyWebhook != "" && !strings.Contains(s.ProxyWebhook, "://") { + return fmt.Errorf("Bitbucket proxy webhook must have the HTTP scheme") + } + return nil +} + func (s *Service) addServerConfiguration(name string, c ServerConfiguration) error { if name == "" { return fmt.Errorf("Invalid VCS server name") diff --git a/engine/vcs/vcs.go b/engine/vcs/vcs.go index 47a9f21ef3..9ad45e7501 100644 --- a/engine/vcs/vcs.go +++ b/engine/vcs/vcs.go @@ -11,7 +11,8 @@ import ( "github.com/ovh/cds/engine/api" "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/services" - "github.com/ovh/cds/engine/vcs/bitbucket" + "github.com/ovh/cds/engine/vcs/bitbucketcloud" + "github.com/ovh/cds/engine/vcs/bitbucketserver" "github.com/ovh/cds/engine/vcs/gerrit" "github.com/ovh/cds/engine/vcs/github" "github.com/ovh/cds/engine/vcs/gitlab" @@ -92,7 +93,7 @@ func (s *Service) getConsumer(name string) (sdk.VCSServer, error) { ), nil } if serverCfg.Bitbucket != nil { - return bitbucket.New(serverCfg.Bitbucket.ConsumerKey, + return bitbucketserver.New(serverCfg.Bitbucket.ConsumerKey, []byte(serverCfg.Bitbucket.PrivateKey), serverCfg.URL, s.Cfg.API.HTTP.URL, @@ -104,6 +105,17 @@ func (s *Service) getConsumer(name string) (sdk.VCSServer, error) { serverCfg.Bitbucket.Status.Disable, ), nil } + if serverCfg.BitbucketCloud != nil { + return bitbucketcloud.New(serverCfg.BitbucketCloud.ClientID, + serverCfg.BitbucketCloud.ClientSecret, + serverCfg.URL, + s.Cfg.UI.HTTP.URL, + serverCfg.BitbucketCloud.ProxyWebhook, + s.Cache, + serverCfg.BitbucketCloud.Status.Disable, + !serverCfg.BitbucketCloud.Status.ShowDetail, + ), nil + } if serverCfg.Gitlab != nil { return gitlab.New(serverCfg.Gitlab.AppID, serverCfg.Gitlab.Secret, diff --git a/engine/vcs/vcs_auth.go b/engine/vcs/vcs_auth.go index 0fe863a454..a5c2f5f15e 100644 --- a/engine/vcs/vcs_auth.go +++ b/engine/vcs/vcs_auth.go @@ -5,23 +5,19 @@ import ( "encoding/base64" "fmt" "net/http" + "strconv" "github.com/ovh/cds/engine/service" "github.com/ovh/cds/sdk" ) -// HTTP Headers -const ( - HeaderXAccessToken = "X-CDS-ACCESS-TOKEN" - HeaderXAccessTokenSecret = "X-CDS-ACCESS-TOKEN-SECRET" -) - // Context type contextKey string var ( - contextKeyAccessToken contextKey = "access-token" - contextKeyAccessTokenSecret contextKey = "access-token-secret" + contextKeyAccessToken contextKey = "access-token" + contextKeyAccessTokenCreated contextKey = "access-token-created" + contextKeyAccessTokenSecret contextKey = "access-token-secret" ) func (s *Service) authMiddleware(ctx context.Context, w http.ResponseWriter, req *http.Request, rc *service.HandlerConfig) (context.Context, error) { @@ -34,24 +30,33 @@ func (s *Service) authMiddleware(ctx context.Context, w http.ResponseWriter, req return ctx, fmt.Errorf("bad header syntax: %s", err) } - encodedAccessToken := req.Header.Get(HeaderXAccessToken) + encodedAccessToken := req.Header.Get(sdk.HeaderXAccessToken) accessToken, err := base64.StdEncoding.DecodeString(encodedAccessToken) if err != nil { return ctx, fmt.Errorf("bad header syntax: %s", err) } - encodedAccessTokenSecret := req.Header.Get(HeaderXAccessTokenSecret) + encodedAccessTokenSecret := req.Header.Get(sdk.HeaderXAccessTokenSecret) accessTokenSecret, err := base64.StdEncoding.DecodeString(encodedAccessTokenSecret) if err != nil { return ctx, fmt.Errorf("bad header syntax: %s", err) } + encodedAccessTokenCreated := req.Header.Get(sdk.HeaderXAccessTokenCreated) + accessTokenCreated, err := base64.StdEncoding.DecodeString(encodedAccessTokenCreated) + if err != nil { + return ctx, fmt.Errorf("bad header syntax for access token created: %s", err) + } + if len(accessToken) != 0 { ctx = context.WithValue(ctx, contextKeyAccessToken, string(accessToken)) } if len(accessTokenSecret) != 0 { ctx = context.WithValue(ctx, contextKeyAccessTokenSecret, string(accessTokenSecret)) } + if len(accessTokenCreated) != 0 { + ctx = context.WithValue(ctx, contextKeyAccessTokenCreated, string(accessTokenCreated)) + } if s.Hash != string(hash) { return ctx, sdk.ErrUnauthorized @@ -60,14 +65,27 @@ func (s *Service) authMiddleware(ctx context.Context, w http.ResponseWriter, req return ctx, nil } -func getAccessTokens(ctx context.Context) (string, string, bool) { +func getAccessTokens(ctx context.Context) (string, string, int64, bool) { + var created int64 accessToken, ok := ctx.Value(contextKeyAccessToken).(string) if !ok { - return "", "", false + return "", "", created, false } accessTokenSecret, ok := ctx.Value(contextKeyAccessTokenSecret).(string) if !ok { - return "", "", false + return "", "", created, false + } + accessTokenCreated, ok := ctx.Value(contextKeyAccessTokenCreated).(string) + if !ok { + return "", "", created, false } - return string(accessToken), string(accessTokenSecret), len(accessToken) > 0 + if accessTokenCreated != "" { + var err error + created, err = strconv.ParseInt(accessTokenCreated, 10, 64) + if err != nil { + return "", "", created, false + } + } + + return string(accessToken), string(accessTokenSecret), created, len(accessToken) > 0 } diff --git a/engine/vcs/vcs_handlers.go b/engine/vcs/vcs_handlers.go index 54755f7d18..2e6327e5bd 100644 --- a/engine/vcs/vcs_handlers.go +++ b/engine/vcs/vcs_handlers.go @@ -115,6 +115,36 @@ func (s *Service) getVCSServersHooksHandler() service.Handler { "pr:comment:edited", "pr:comment:deleted", } + case cfg.BitbucketCloud != nil: + res.WebhooksSupported = true + res.WebhooksDisabled = cfg.BitbucketCloud.DisableWebHooks + res.WebhooksIcon = sdk.BitbucketIcon + // https://developer.atlassian.com/bitbucket/api/2/reference/resource/hook_events/%7Bsubject_type%7D + res.Events = []string{ + "pullrequest:unapproved", + "issue:comment_created", + "pullrequest:approved", + "repo:created", + "repo:deleted", + "repo:imported", + "pullrequest:comment_updated", + "issue:updated", + "project:updated", + "pullrequest:comment_created", + "repo:commit_status_updated", + "pullrequest:updated", + "issue:created", + "repo:fork", + "pullrequest:comment_deleted", + "repo:commit_status_created", + "repo:updated", + "pullrequest:rejected", + "pullrequest:fulfilled", + "repo:push", + "pullrequest:created", + "repo:transfer", + "repo:commit_comment_created", + } case cfg.Github != nil: res.WebhooksSupported = true res.WebhooksDisabled = cfg.Github.DisableWebHooks @@ -223,6 +253,8 @@ func (s *Service) getVCSServersPollingHandler() service.Handler { case cfg.Bitbucket != nil: res.PollingSupported = false res.PollingDisabled = cfg.Bitbucket.DisablePolling + case cfg.BitbucketCloud != nil: + res.PollingSupported = false case cfg.Github != nil: res.PollingSupported = true res.PollingDisabled = cfg.Github.DisablePolling @@ -284,7 +316,7 @@ func (s *Service) getReposHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { name := muxVar(r, "name") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getReposHandler> Unable to get access token headers") } @@ -294,10 +326,14 @@ func (s *Service) getReposHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable") } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client") } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } repos, err := client.Repos(ctx) if err != nil { @@ -314,7 +350,7 @@ func (s *Service) getRepoHandler() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getRepoHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -324,10 +360,14 @@ func (s *Service) getRepoHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } ghRepo, err := client.RepoByFullname(ctx, fmt.Sprintf("%s/%s", owner, repo)) if err != nil { @@ -344,7 +384,7 @@ func (s *Service) getBranchesHandler() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getBranchesHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -354,10 +394,14 @@ func (s *Service) getBranchesHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } branches, err := client.Branches(ctx, fmt.Sprintf("%s/%s", owner, repo)) if err != nil { @@ -374,7 +418,7 @@ func (s *Service) getBranchHandler() service.Handler { repo := muxVar(r, "repo") branch := r.URL.Query().Get("branch") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getBranchHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -384,10 +428,14 @@ func (s *Service) getBranchHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } ghBranch, err := client.Branch(ctx, fmt.Sprintf("%s/%s", owner, repo), branch) if err != nil { @@ -405,7 +453,7 @@ func (s *Service) getTagsHandler() service.Handler { log.Debug("getTagsHandler>") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getTagsHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -415,10 +463,14 @@ func (s *Service) getTagsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } tags, err := client.Tags(ctx, fmt.Sprintf("%s/%s", owner, repo)) if err != nil { @@ -439,7 +491,7 @@ func (s *Service) getCommitsHandler() service.Handler { log.Debug("getCommitsHandler>") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getCommitsHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -449,10 +501,14 @@ func (s *Service) getCommitsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } commits, err := client.Commits(ctx, fmt.Sprintf("%s/%s", owner, repo), branch, since, until) if err != nil { @@ -472,7 +528,7 @@ func (s *Service) getCommitsBetweenRefsHandler() service.Handler { log.Debug("getCommitsBetweenRefsHandler>") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getCommitsBetweenRefsHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -482,10 +538,14 @@ func (s *Service) getCommitsBetweenRefsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } commits, err := client.CommitsBetweenRefs(ctx, fmt.Sprintf("%s/%s", owner, repo), base, head) if err != nil { @@ -502,7 +562,7 @@ func (s *Service) getCommitHandler() service.Handler { repo := muxVar(r, "repo") commit := muxVar(r, "commit") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getCommitHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -512,10 +572,14 @@ func (s *Service) getCommitHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } c, err := client.Commit(ctx, fmt.Sprintf("%s/%s", owner, repo), commit) if err != nil { @@ -532,7 +596,7 @@ func (s *Service) getCommitStatusHandler() service.Handler { repo := muxVar(r, "repo") commit := muxVar(r, "commit") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getCommitStatusHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -542,10 +606,14 @@ func (s *Service) getCommitStatusHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } statuses, err := client.ListStatuses(ctx, fmt.Sprintf("%s/%s", owner, repo), commit) if err != nil { @@ -567,7 +635,7 @@ func (s *Service) getPullRequestHandler() service.Handler { return sdk.ErrWrongRequest } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -577,10 +645,14 @@ func (s *Service) getPullRequestHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } c, err := client.PullRequest(ctx, fmt.Sprintf("%s/%s", owner, repo), id) if err != nil { @@ -596,7 +668,7 @@ func (s *Service) getPullRequestsHandler() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getPullRequestsHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -606,10 +678,14 @@ func (s *Service) getPullRequestsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } c, err := client.PullRequests(ctx, fmt.Sprintf("%s/%s", owner, repo)) if err != nil { @@ -630,7 +706,7 @@ func (s *Service) postPullRequestsHandler() service.Handler { return sdk.WithStack(err) } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getPullRequestsHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -640,10 +716,14 @@ func (s *Service) postPullRequestsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } c, err := client.PullRequestCreate(ctx, fmt.Sprintf("%s/%s", owner, repo), prRequest) if err != nil { @@ -669,7 +749,7 @@ func (s *Service) postPullRequestCommentHandler() service.Handler { return sdk.WithStack(err) } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -679,10 +759,14 @@ func (s *Service) postPullRequestCommentHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } if err := client.PullRequestComment(ctx, fmt.Sprintf("%s/%s", owner, repo), id, body); err != nil { return sdk.WrapError(err, "Unable to create new PR comment %s %s/%s", name, owner, repo) @@ -708,7 +792,7 @@ func (s *Service) getEventsHandler() service.Handler { dateRef = time.Unix(int64(dateRefInt), 0) } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getEventsHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -718,10 +802,14 @@ func (s *Service) getEventsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } evts, delay, err := client.GetEvents(ctx, fmt.Sprintf("%s/%s", owner, repo), dateRef) if err != nil && err != github.ErrNoNewEvents { @@ -749,7 +837,7 @@ func (s *Service) postFilterEventsHandler() service.Handler { return sdk.WrapError(err, "Unable to read body") } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -759,10 +847,14 @@ func (s *Service) postFilterEventsHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } filter := r.URL.Query().Get("filter") @@ -806,7 +898,7 @@ func (s *Service) postStatusHandler() service.Handler { return sdk.WrapError(err, "Unable to read body") } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers") } @@ -816,10 +908,14 @@ func (s *Service) postStatusHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable") } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client") } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } if err := client.SetStatus(ctx, evt); err != nil { return sdk.WrapError(err, "Unable to set status on %s", name) @@ -835,7 +931,7 @@ func (s *Service) postReleaseHandler() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> postReleaseHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -845,10 +941,14 @@ func (s *Service) postReleaseHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } body := struct { Tag string `json:"tag"` @@ -886,7 +986,7 @@ func (s *Service) postUploadReleaseFileHandler() service.Handler { return err } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -896,10 +996,14 @@ func (s *Service) postUploadReleaseFileHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } if err := client.UploadReleaseFile(ctx, fmt.Sprintf("%s/%s", owner, repo), release, uploadURL, artifactName, r.Body); err != nil { return sdk.WrapError(err, "Unable to upload release file %s %s/%s", name, owner, repo) @@ -920,7 +1024,7 @@ func (s *Service) getHookHandler() service.Handler { return err } - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -930,10 +1034,14 @@ func (s *Service) getHookHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } hook, err := client.GetHook(ctx, fmt.Sprintf("%s/%s", owner, repo), hookURL) if err != nil { @@ -950,7 +1058,7 @@ func (s *Service) postHookHandler() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -960,10 +1068,14 @@ func (s *Service) postHookHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } body := sdk.VCSHook{} if err := service.UnmarshalBody(r, &body); err != nil { @@ -990,7 +1102,7 @@ func (s *Service) deleteHookHandler() service.Handler { hookID := r.URL.Query().Get("id") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> deleteHookHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -1000,10 +1112,14 @@ func (s *Service) deleteHookHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } var hook sdk.VCSHook if hookID == "" { @@ -1029,7 +1145,7 @@ func (s *Service) getListForks() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> getListForks> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -1039,10 +1155,14 @@ func (s *Service) getListForks() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } forks, err := client.ListForks(ctx, fmt.Sprintf("%s/%s", owner, repo)) if err != nil { @@ -1077,7 +1197,7 @@ func (s *Service) postRepoGrantHandler() service.Handler { owner := muxVar(r, "owner") repo := muxVar(r, "repo") - accessToken, accessTokenSecret, ok := getAccessTokens(ctx) + accessToken, accessTokenSecret, created, ok := getAccessTokens(ctx) if !ok { return sdk.WrapError(sdk.ErrUnauthorized, "VCS> postRepoGrantHandler> Unable to get access token headers %s %s/%s", name, owner, repo) } @@ -1087,10 +1207,14 @@ func (s *Service) postRepoGrantHandler() service.Handler { return sdk.WrapError(err, "VCS server unavailable %s %s/%s", name, owner, repo) } - client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret) + client, err := consumer.GetAuthorizedClient(ctx, accessToken, accessTokenSecret, created) if err != nil { return sdk.WrapError(err, "Unable to get authorized client %s %s/%s", name, owner, repo) } + // Check if access token has been refreshed + if accessToken != client.GetAccessToken(ctx) { + w.Header().Set(sdk.HeaderXAccessToken, client.GetAccessToken(ctx)) + } if err := client.GrantWritePermission(ctx, owner+"/"+repo); err != nil { return sdk.WrapError(err, "unable to grant %s/%s on %s", owner, repo, name) diff --git a/engine/vcs/vcs_handlers_test.go b/engine/vcs/vcs_handlers_test.go index 3b82a3933d..b41d4f6f74 100644 --- a/engine/vcs/vcs_handlers_test.go +++ b/engine/vcs/vcs_handlers_test.go @@ -6,6 +6,8 @@ import ( "net/http/httptest" "testing" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/engine/api/test" "github.com/ovh/cds/sdk/log" "github.com/stretchr/testify/assert" @@ -140,8 +142,8 @@ func Test_accessTokenAuth(t *testing.T) { req := newRequest(t, s, "GET", uri, nil) //Without any header, this should return 401 - req.Header.Set(HeaderXAccessToken, "") - req.Header.Set(HeaderXAccessTokenSecret, "") + req.Header.Set(sdk.HeaderXAccessToken, "") + req.Header.Set(sdk.HeaderXAccessTokenSecret, "") //Do the request rec := httptest.NewRecorder() @@ -178,9 +180,9 @@ func Test_getReposHandler(t *testing.T) { req := newRequest(t, s, "GET", uri, nil) token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -220,9 +222,9 @@ func Test_getRepoHandler(t *testing.T) { req := newRequest(t, s, "GET", uri, nil) token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -262,9 +264,9 @@ func Test_getBranchesHandler(t *testing.T) { req := newRequest(t, s, "GET", uri, nil) token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -307,9 +309,9 @@ func Test_getBranchHandler(t *testing.T) { req.URL.RawQuery = q.Encode() token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -354,9 +356,9 @@ func Test_getCommitsHandler(t *testing.T) { req.URL.RawQuery = q.Encode() token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -397,9 +399,9 @@ func Test_getCommitHandler(t *testing.T) { req := newRequest(t, s, "GET", uri, nil) token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -440,9 +442,9 @@ func Test_getCommitStatusHandler(t *testing.T) { req := newRequest(t, s, "GET", uri, nil) token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() @@ -526,9 +528,9 @@ func Test_postRepoGrantHandler(t *testing.T) { req := newRequest(t, s, "POST", uri, nil) token := base64.StdEncoding.EncodeToString([]byte(cfg["githubAccessToken"])) - req.Header.Set(HeaderXAccessToken, token) + req.Header.Set(sdk.HeaderXAccessToken, token) //accessTokenSecret is useless for github, let's give the same token - req.Header.Set(HeaderXAccessTokenSecret, token) + req.Header.Set(sdk.HeaderXAccessTokenSecret, token) //Do the request rec := httptest.NewRecorder() diff --git a/sdk/vcs.go b/sdk/vcs.go index 22f6d2098c..e9aa17e30a 100644 --- a/sdk/vcs.go +++ b/sdk/vcs.go @@ -7,6 +7,13 @@ import ( "time" ) +// HTTP Headers +const ( + HeaderXAccessToken = "X-CDS-ACCESS-TOKEN" + HeaderXAccessTokenCreated = "X-CDS-ACCESS-TOKEN-CREATED" + HeaderXAccessTokenSecret = "X-CDS-ACCESS-TOKEN-SECRET" +) + // BuildNumberAndHash represents BuildNumber, Commit Hash and Branch for a Pipeline Build or Node Run type BuildNumberAndHash struct { BuildNumber int64 @@ -30,7 +37,7 @@ type VCSConfiguration struct { type VCSServer interface { AuthorizeRedirect(context.Context) (string, string, error) AuthorizeToken(context.Context, string, string) (string, string, error) - GetAuthorizedClient(context.Context, string, string) (VCSAuthorizedClient, error) + GetAuthorizedClient(context.Context, string, string, int64) (VCSAuthorizedClient, error) } // VCSAuthorizedClient is an interface for a connected client on a VCS Server. @@ -82,6 +89,9 @@ type VCSAuthorizedClient interface { // Permissions GrantWritePermission(ctx context.Context, repo string) error + + // Access Token + GetAccessToken(ctx context.Context) string } // GetDefaultBranch return the default branch From 902743345d7fe8f7362248e857a0d91ae4ee4302 Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Wed, 12 Jun 2019 16:45:00 +0200 Subject: [PATCH 04/11] feat(api): add new conditions on stage level (#4298) * feat(api): add new conditions on stage level Signed-off-by: Benjamin Coenen --- engine/api/api.go | 5 +- engine/api/api_routes.go | 1 + engine/api/migrate/stage_conditions.go | 68 ++++++++ engine/api/pipeline/gorp_model.go | 15 ++ engine/api/pipeline/pipeline.go | 79 ++++----- engine/api/pipeline/pipeline_importer.go | 8 + engine/api/pipeline/pipeline_parser.go | 1 - engine/api/pipeline/pipeline_stage.go | 99 +++-------- engine/api/pipeline_stage_test.go | 157 ++++++++++-------- engine/api/stage.go | 67 ++++++++ engine/api/workflow/dao_run_test.go | 2 - engine/api/workflow/execute_node_run.go | 8 +- engine/api/workflow/process.go | 2 +- engine/api/workflow/process_node.go | 4 +- engine/api/workflow/process_outgoinghook.go | 2 +- engine/sql/167_stage_condition.sql | 5 + sdk/exportentities/pipeline.go | 22 +-- sdk/exportentities/pipeline_test.go | 15 +- sdk/pipeline.go | 12 +- sdk/stage.go | 5 +- sdk/workflow.go | 8 +- ui/resources/cds-hint.js | 30 ++-- ui/src/app/app.component.spec.ts | 4 +- ui/src/app/model/stage.model.ts | 4 +- .../app/service/pipeline/pipeline.service.ts | 11 ++ .../shared/conditions/conditions.component.ts | 148 +++++++++++++++++ ui/src/app/shared/conditions/conditions.html | 133 +++++++++++++++ ui/src/app/shared/conditions/conditions.scss | 18 ++ ui/src/app/shared/shared.module.ts | 3 + .../conditions/wizard.conditions.component.ts | 23 ++- .../wizard/conditions/wizard.conditions.html | 152 +---------------- .../show/workflow/pipeline.workflow.html | 5 +- .../pipeline.stage.form.component.spec.ts | 38 +---- .../form/pipeline.stage.form.component.ts | 56 ++----- .../stage/form/pipeline.stage.form.html | 25 ++- 35 files changed, 720 insertions(+), 515 deletions(-) create mode 100644 engine/api/migrate/stage_conditions.go create mode 100644 engine/sql/167_stage_condition.sql create mode 100644 ui/src/app/shared/conditions/conditions.component.ts create mode 100644 ui/src/app/shared/conditions/conditions.html create mode 100644 ui/src/app/shared/conditions/conditions.scss diff --git a/engine/api/api.go b/engine/api/api.go index 57f9e097bf..70b05c3220 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -786,7 +786,10 @@ func (a *API) Serve(ctx context.Context) error { migrate.Add(sdk.Migration{Name: "CleanArtifactBuiltinActions", Release: "0.38.1", Mandatory: true, ExecFunc: func(ctx context.Context) error { return migrate.CleanArtifactBuiltinActions(ctx, a.Cache, a.DBConnectionFactory.GetDBMap) }}) - // migrate.Add(sdk.Migration{Name: "GitClonePrivateKey", Release: "0.38.1", Mandatory: true, ExecFunc: func(ctx context.Context) error { + migrate.Add(sdk.Migration{Name: "StageConditions", Release: "0.39.3", Mandatory: true, ExecFunc: func(ctx context.Context) error { + return migrate.StageConditions(a.Cache, a.DBConnectionFactory.GetDBMap()) + }}) + // migrate.Add(sdk.Migration{Name: "GitClonePrivateKey", Release: "0.37.0", Mandatory: true, ExecFunc: func(ctx context.Context) error { // return migrate.GitClonePrivateKey(a.mustDB, a.Cache) // }}) migrate.Add(sdk.Migration{Name: "ActionModelRequirements", Release: "0.39.3", Mandatory: true, ExecFunc: func(ctx context.Context) error { diff --git a/engine/api/api_routes.go b/engine/api/api_routes.go index df906fd34e..795525a4ae 100644 --- a/engine/api/api_routes.go +++ b/engine/api/api_routes.go @@ -178,6 +178,7 @@ func (api *API) InitRouter() { r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/rollback/{auditID}", r.POST(api.postPipelineRollbackHandler)) r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/audits", r.GET(api.getPipelineAuditHandler)) r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/stage", r.POST(api.addStageHandler)) + r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/stage/condition", r.GET(api.getStageConditionsHandler)) r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/stage/move", r.POST(api.moveStageHandler)) r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/stage/{stageID}", r.GET(api.getStageHandler), r.PUT(api.updateStageHandler), r.DELETE(api.deleteStageHandler)) r.Handle("/project/{permProjectKey}/pipeline/{pipelineKey}/stage/{stageID}/job", r.POST(api.addJobToStageHandler)) diff --git a/engine/api/migrate/stage_conditions.go b/engine/api/migrate/stage_conditions.go new file mode 100644 index 0000000000..15dd353654 --- /dev/null +++ b/engine/api/migrate/stage_conditions.go @@ -0,0 +1,68 @@ +package migrate + +import ( + "encoding/json" + "strings" + + "github.com/go-gorp/gorp" + + "github.com/ovh/cds/engine/api/cache" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +type prerequisitesDB struct { + sdk.Prerequisite + PipelineStageID int64 `json:"pipeline_stage_id"` +} + +func StageConditions(store cache.Store, db gorp.SqlExecutor) error { + var prerequisites []prerequisitesDB + rows, err := db.Query("SELECT pipeline_stage_id, parameter, expected_value FROM pipeline_stage_prerequisite") + if err != nil { + return sdk.WrapError(err, "cannot load all pipeline stage prerequisites") + } + defer rows.Close() + + for rows.Next() { + var prereq prerequisitesDB + if err := rows.Scan(&prereq.PipelineStageID, &prereq.Parameter, &prereq.ExpectedValue); err != nil { + return sdk.WrapError(err, "cannot scan row") + } + prerequisites = append(prerequisites, prereq) + } + + conditionsMap := convertToNewConditions(prerequisites) + for stageID, condition := range conditionsMap { + conditionBts, err := json.Marshal(condition) + if err != nil { + log.Error("Cannot json to null string for condition %+v : %v", condition, err) + continue + } + + if _, err := db.Exec("UPDATE pipeline_stage SET conditions = $1 WHERE pipeline_stage.id = $2", conditionBts, stageID); err != nil { + log.Error("Cannot update pipeline_stage conditions for id %d : %v", stageID, err) + } + } + + return nil +} + +func convertToNewConditions(prerequisites []prerequisitesDB) map[int64]sdk.WorkflowNodeConditions { + conditionsMap := map[int64]sdk.WorkflowNodeConditions{} + for _, p := range prerequisites { + if !strings.HasPrefix(p.Parameter, "workflow.") && !strings.HasPrefix(p.Parameter, "git.") { + p.Parameter = "cds.pip." + p.Parameter + } + cond := sdk.WorkflowNodeCondition{ + Value: p.ExpectedValue, + Variable: p.Parameter, + Operator: sdk.WorkflowConditionsOperatorRegex, + } + plainConditions := append([]sdk.WorkflowNodeCondition{}, conditionsMap[p.PipelineStageID].PlainConditions...) + plainConditions = append(plainConditions, cond) + conditionsMap[p.PipelineStageID] = sdk.WorkflowNodeConditions{PlainConditions: plainConditions} + } + + return conditionsMap +} diff --git a/engine/api/pipeline/gorp_model.go b/engine/api/pipeline/gorp_model.go index 6169efca00..26a5c6c04a 100644 --- a/engine/api/pipeline/gorp_model.go +++ b/engine/api/pipeline/gorp_model.go @@ -3,6 +3,7 @@ package pipeline import ( "time" + "github.com/go-gorp/gorp" "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/ovh/cds/sdk" ) @@ -10,6 +11,9 @@ import ( // PipelineAudit is a gorp wrapper around sdk.PipelineAudit type PipelineAudit sdk.PipelineAudit +// Pipeline is a gorp wrapper around sdk.Pipeline +type Pipeline sdk.Pipeline + type pipelineAction struct { ID int64 `db:"id"` PipelineStageID int64 `db:"pipeline_stage_id"` @@ -37,7 +41,18 @@ func pipelineActionsToActionIDs(pas []pipelineAction) []int64 { func init() { gorpmapping.Register( + gorpmapping.New(Pipeline{}, "pipeline", true, "id"), gorpmapping.New(PipelineAudit{}, "pipeline_audit", true, "id"), gorpmapping.New(pipelineAction{}, "pipeline_action", true, "id"), ) } + +func (pip *Pipeline) PostGet(db gorp.SqlExecutor) error { + projectKey, err := db.SelectStr("SELECT project.projectkey FROM project WHERE id = $1", pip.ProjectID) + if err != nil { + return sdk.WrapError(err, "cannot fetch project key for project id %d", pip.ProjectID) + } + pip.ProjectKey = projectKey + + return nil +} diff --git a/engine/api/pipeline/pipeline.go b/engine/api/pipeline/pipeline.go index e235f766e0..8b07373d46 100644 --- a/engine/api/pipeline/pipeline.go +++ b/engine/api/pipeline/pipeline.go @@ -23,36 +23,33 @@ type structarg struct { // LoadPipeline loads a pipeline from database func LoadPipeline(db gorp.SqlExecutor, projectKey, name string, deep bool) (*sdk.Pipeline, error) { - var p sdk.Pipeline - - var lastModified time.Time + var p Pipeline query := `SELECT pipeline.id, pipeline.name, pipeline.description, pipeline.project_id, pipeline.last_modified, pipeline.from_repository FROM pipeline JOIN project on pipeline.project_id = project.id WHERE pipeline.name = $1 AND project.projectKey = $2` - if err := db.QueryRow(query, name, projectKey).Scan(&p.ID, &p.Name, &p.Description, &p.ProjectID, &lastModified, &p.FromRepository); err != nil { + if err := db.SelectOne(&p, query, name, projectKey); err != nil { if err == sql.ErrNoRows { return nil, sdk.WithStack(sdk.ErrPipelineNotFound) } return nil, sdk.WithStack(err) } - p.LastModified = lastModified.Unix() - p.ProjectKey = projectKey + pip := sdk.Pipeline(p) if deep { - if err := loadPipelineDependencies(context.TODO(), db, &p); err != nil { + if err := loadPipelineDependencies(context.TODO(), db, &pip); err != nil { return nil, err } } else { - parameters, err := GetAllParametersInPipeline(context.TODO(), db, p.ID) + parameters, err := GetAllParametersInPipeline(context.TODO(), db, pip.ID) if err != nil { return nil, err } p.Parameter = parameters } - return &p, nil + return &pip, nil } // LoadPipelineByID loads a pipeline from database @@ -64,36 +61,32 @@ func LoadPipelineByID(ctx context.Context, db gorp.SqlExecutor, pipelineID int64 ) defer end() - var lastModified time.Time - var p sdk.Pipeline - query := `SELECT pipeline.name, pipeline.description, project.projectKey, pipeline.last_modified, pipeline.from_repository + var p Pipeline + query := `SELECT pipeline.* FROM pipeline - JOIN project on pipeline.project_id = project.id WHERE pipeline.id = $1` - err := db.QueryRow(query, pipelineID).Scan(&p.Name, &p.Description, &p.ProjectKey, &lastModified, &p.FromRepository) - if err != nil { + if err := db.SelectOne(&p, query, pipelineID); err != nil { if err == sql.ErrNoRows { return nil, sdk.ErrPipelineNotFound } - return nil, err + return nil, sdk.WithStack(err) } - p.LastModified = lastModified.Unix() - p.ID = pipelineID + pip := sdk.Pipeline(p) if deep { - if err := loadPipelineDependencies(ctx, db, &p); err != nil { + if err := loadPipelineDependencies(ctx, db, &pip); err != nil { return nil, err } } else { - parameters, err := GetAllParametersInPipeline(ctx, db, p.ID) + parameters, err := GetAllParametersInPipeline(ctx, db, pip.ID) if err != nil { return nil, err } - p.Parameter = parameters + pip.Parameter = parameters } - return &p, nil + return &pip, nil } // LoadByWorkerModel loads pipelines from database for a given worker model name @@ -179,7 +172,7 @@ func LoadByWorkerModel(ctx context.Context, db gorp.SqlExecutor, u *sdk.User, mo // LoadByWorkflowID loads pipelines from database for a given workflow id func LoadByWorkflowID(db gorp.SqlExecutor, workflowID int64) ([]sdk.Pipeline, error) { - pips := []sdk.Pipeline{} + pips := []Pipeline{} query := `SELECT DISTINCT pipeline.* FROM pipeline JOIN workflow_node ON pipeline.id = workflow_node.pipeline_id @@ -188,12 +181,16 @@ func LoadByWorkflowID(db gorp.SqlExecutor, workflowID int64) ([]sdk.Pipeline, er if _, err := db.Select(&pips, query, workflowID); err != nil { if err == sql.ErrNoRows { - return pips, nil + return []sdk.Pipeline{}, nil } return nil, sdk.WrapError(err, "Unable to load pipelines linked to workflow id %d", workflowID) } + pipsSdk := make([]sdk.Pipeline, len(pips)) + for i := range pips { + pipsSdk[i] = sdk.Pipeline(pips[i]) + } - return pips, nil + return pipsSdk, nil } func loadPipelineDependencies(ctx context.Context, db gorp.SqlExecutor, p *sdk.Pipeline) error { @@ -230,47 +227,31 @@ func DeletePipeline(ctx context.Context, db gorp.SqlExecutor, pipelineID int64, // LoadPipelines loads all pipelines in a project func LoadPipelines(db gorp.SqlExecutor, projectID int64, loadDependencies bool) ([]sdk.Pipeline, error) { - var pip []sdk.Pipeline + var pips []sdk.Pipeline query := `SELECT id, name, description, project_id, last_modified, from_repository FROM pipeline WHERE project_id = $1 ORDER BY pipeline.name` - rows, errquery := db.Query(query, projectID) - if errquery != nil { - return nil, errquery + if _, err := db.Select(&pips, query, projectID); err != nil { + return nil, sdk.WithStack(err) } - defer rows.Close() - - for rows.Next() { - var p sdk.Pipeline - var lastModified time.Time - - // scan pipeline id - if err := rows.Scan(&p.ID, &p.Name, &p.Description, &p.ProjectID, &lastModified, &p.FromRepository); err != nil { - return nil, err - } - p.LastModified = lastModified.Unix() + for i := range pips { if loadDependencies { // load pipeline stages - if err := LoadPipelineStage(context.TODO(), db, &p); err != nil { + if err := LoadPipelineStage(context.TODO(), db, &pips[i]); err != nil { return nil, err } } - - pip = append(pip, p) - } - - for i := range pip { - params, err := GetAllParametersInPipeline(context.TODO(), db, pip[i].ID) + params, err := GetAllParametersInPipeline(context.TODO(), db, pips[i].ID) if err != nil { return nil, err } - pip[i].Parameter = params + pips[i].Parameter = params } - return pip, nil + return pips, nil } // LoadAllNames returns all pipeline names diff --git a/engine/api/pipeline/pipeline_importer.go b/engine/api/pipeline/pipeline_importer.go index aa05c27c60..d1f0707cbd 100644 --- a/engine/api/pipeline/pipeline_importer.go +++ b/engine/api/pipeline/pipeline_importer.go @@ -137,9 +137,12 @@ func ImportUpdate(ctx context.Context, db gorp.SqlExecutor, proj *sdk.Project, p //Check if we have to delete stages for _, os := range oldPipeline.Stages { var stageFound bool + var currentStage sdk.Stage for _, s := range pip.Stages { if s.Name == os.Name { stageFound = true + currentStage = s + currentStage.ID = os.ID break } } @@ -159,6 +162,11 @@ func ImportUpdate(ctx context.Context, db gorp.SqlExecutor, proj *sdk.Project, p if msgChan != nil { msgChan <- sdk.NewMessage(sdk.MsgPipelineStageDeleted, os.Name) } + } else { + // Update stage + if err := UpdateStage(db, ¤tStage); err != nil { + return sdk.WrapError(err, "cannot update stage %s (id=%d) for conditions, build_order and name", currentStage.Name, currentStage.ID) + } } } diff --git a/engine/api/pipeline/pipeline_parser.go b/engine/api/pipeline/pipeline_parser.go index 1f02c347f5..052f2d5ae1 100644 --- a/engine/api/pipeline/pipeline_parser.go +++ b/engine/api/pipeline/pipeline_parser.go @@ -51,7 +51,6 @@ func ParseAndImport(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, }(&msgList) var globalError error - if exist && !opts.Force { return pip, nil, sdk.ErrPipelineAlreadyExists } else if exist { diff --git a/engine/api/pipeline/pipeline_stage.go b/engine/api/pipeline/pipeline_stage.go index 5c41435b04..43e04e8a5a 100644 --- a/engine/api/pipeline/pipeline_stage.go +++ b/engine/api/pipeline/pipeline_stage.go @@ -5,7 +5,8 @@ import ( "database/sql" "encoding/json" "fmt" - "strconv" + + "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/go-gorp/gorp" "github.com/lib/pq" @@ -13,7 +14,6 @@ import ( "github.com/ovh/cds/engine/api/action" "github.com/ovh/cds/engine/api/observability" "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" ) var ( @@ -24,16 +24,13 @@ var ( // LoadStage Get a stage from its ID and pipeline ID func LoadStage(db gorp.SqlExecutor, pipelineID int64, stageID int64) (*sdk.Stage, error) { query := ` - SELECT pipeline_stage.id, pipeline_stage.pipeline_id, pipeline_stage.name, pipeline_stage.build_order, pipeline_stage.enabled, pipeline_stage_prerequisite.parameter, pipeline_stage_prerequisite.expected_value + SELECT pipeline_stage.id, pipeline_stage.pipeline_id, pipeline_stage.name, pipeline_stage.build_order, pipeline_stage.enabled FROM pipeline_stage - LEFT OUTER JOIN pipeline_stage_prerequisite ON pipeline_stage_prerequisite.pipeline_stage_id = pipeline_stage.id WHERE pipeline_stage.pipeline_id = $1 AND pipeline_stage.id = $2; ` var stage sdk.Stage - stage.Prerequisites = []sdk.Prerequisite{} - rows, err := db.Query(query, pipelineID, stageID) if err == sql.ErrNoRows { return nil, ErrNoStage @@ -44,14 +41,8 @@ func LoadStage(db gorp.SqlExecutor, pipelineID int64, stageID int64) (*sdk.Stage defer rows.Close() for rows.Next() { - var parameter, expectedValue sql.NullString - rows.Scan(&stage.ID, &stage.PipelineID, &stage.Name, &stage.BuildOrder, &stage.Enabled, ¶meter, &expectedValue) - if parameter.Valid && expectedValue.Valid { - p := sdk.Prerequisite{ - Parameter: parameter.String, - ExpectedValue: expectedValue.String, - } - stage.Prerequisites = append(stage.Prerequisites, p) + if err := rows.Scan(&stage.ID, &stage.PipelineID, &stage.Name, &stage.BuildOrder, &stage.Enabled); err != nil { + return nil, sdk.WithStack(err) } } @@ -65,24 +56,19 @@ func InsertStage(db gorp.SqlExecutor, s *sdk.Stage) error { if err := db.QueryRow(query, s.PipelineID, s.Name, s.BuildOrder, s.Enabled).Scan(&s.ID); err != nil { return err } - return InsertStagePrequisites(db, s) + return InsertStageConditions(db, s) } -// InsertStagePrequisites insert prequisite for given stage in database -func InsertStagePrequisites(db gorp.SqlExecutor, s *sdk.Stage) error { - if len(s.Prerequisites) > 0 { - query := "INSERT INTO \"pipeline_stage_prerequisite\" (pipeline_stage_id, parameter, expected_value) VALUES " - args := []interface{}{s.ID} - for i, p := range s.Prerequisites { - if i > 0 { - query += "," - } - args = append(args, p.Parameter, p.ExpectedValue) - query += fmt.Sprintf("($1, $%d, $%d)", len(args)-1, len(args)) +// InsertStageConditions insert prequisite for given stage in database +func InsertStageConditions(db gorp.SqlExecutor, s *sdk.Stage) error { + if len(s.Conditions.PlainConditions) > 0 || s.Conditions.LuaScript != "" { + query := "UPDATE pipeline_stage SET conditions = $1 WHERE id = $2" + + conditionsBts, err := json.Marshal(s.Conditions) + if err != nil { + return sdk.WrapError(err, "cannot marshal condition for stage %d", s.ID) } - query += " RETURNING id" - var i int - if err := db.QueryRow(query, args...).Scan(&i); err != nil { + if _, err := db.Exec(query, conditionsBts, s.ID); err != nil { return err } } @@ -102,16 +88,15 @@ func LoadPipelineStage(ctx context.Context, db gorp.SqlExecutor, p *sdk.Pipeline query := ` SELECT pipeline_stage_R.id as stage_id, pipeline_stage_R.pipeline_id, pipeline_stage_R.name, pipeline_stage_R.last_modified, - pipeline_stage_R.build_order, pipeline_stage_R.enabled, pipeline_stage_R.parameter, - pipeline_stage_R.expected_value, pipeline_action_R.id as pipeline_action_id, pipeline_action_R.action_id, pipeline_action_R.action_last_modified, + pipeline_stage_R.build_order, pipeline_stage_R.enabled, pipeline_stage_R.conditions, + pipeline_action_R.id as pipeline_action_id, pipeline_action_R.action_id, pipeline_action_R.action_last_modified, pipeline_action_R.action_args, pipeline_action_R.action_enabled FROM ( SELECT pipeline_stage.id, pipeline_stage.pipeline_id, pipeline_stage.name, pipeline_stage.last_modified, pipeline_stage.build_order, pipeline_stage.enabled, - pipeline_stage_prerequisite.parameter, pipeline_stage_prerequisite.expected_value + pipeline_stage.conditions FROM pipeline_stage - LEFT OUTER JOIN pipeline_stage_prerequisite ON pipeline_stage.id = pipeline_stage_prerequisite.pipeline_stage_id WHERE pipeline_id = $1 ) as pipeline_stage_R LEFT OUTER JOIN ( @@ -140,14 +125,13 @@ func LoadPipelineStage(ctx context.Context, db gorp.SqlExecutor, p *sdk.Pipeline var stageBuildOrder int var pipelineActionID, actionID sql.NullInt64 var stageName string - var stagePrerequisiteParameter, stagePrerequisiteExpectedValue, actionArgs sql.NullString + var stageConditions, actionArgs sql.NullString var stageEnabled, actionEnabled sql.NullBool var stageLastModified, actionLastModified pq.NullTime err = rows.Scan( &stageID, &pipelineID, &stageName, &stageLastModified, - &stageBuildOrder, &stageEnabled, &stagePrerequisiteParameter, - &stagePrerequisiteExpectedValue, &pipelineActionID, &actionID, &actionLastModified, + &stageBuildOrder, &stageEnabled, &stageConditions, &pipelineActionID, &actionID, &actionLastModified, &actionArgs, &actionEnabled) if err != nil { return sdk.WithStack(err) @@ -168,23 +152,8 @@ func LoadPipelineStage(ctx context.Context, db gorp.SqlExecutor, p *sdk.Pipeline stagesPtr = append(stagesPtr, stageData) } - //Stage prequisites - if stagePrerequisiteParameter.Valid && stagePrerequisiteExpectedValue.Valid { - p := sdk.Prerequisite{ - Parameter: stagePrerequisiteParameter.String, - ExpectedValue: stagePrerequisiteExpectedValue.String, - } - var found bool - for i := range stageData.Prerequisites { - if stageData.Prerequisites[i].Parameter == p.Parameter && - stageData.Prerequisites[i].ExpectedValue == p.ExpectedValue { - found = true - break - } - } - if !found { - stageData.Prerequisites = append(stageData.Prerequisites, p) - } + if err := gorpmapping.JSONNullString(stageConditions, &stageData.Conditions); err != nil { + return sdk.WrapError(err, "cannot unmarshal stage conditions for stage id %d", stageID) } //Get actions @@ -267,13 +236,8 @@ func UpdateStage(db gorp.SqlExecutor, s *sdk.Stage) error { return err } - //Remove all prequisites - if err := deleteStagePrerequisites(db, s.ID); err != nil { - return sdk.WithStack(err) - } - //Insert all prequisites - return InsertStagePrequisites(db, s) + return InsertStageConditions(db, s) } // DeleteStageByID Delete stage with associated pipeline action @@ -295,11 +259,6 @@ func DeleteStageByID(ctx context.Context, tx gorp.SqlExecutor, s *sdk.Stage, use } func deleteStageByID(tx gorp.SqlExecutor, s *sdk.Stage) error { - //Delete stage prequisites - if err := deleteStagePrerequisites(tx, s.ID); err != nil { - return err - } - //Delete stage query := `DELETE FROM pipeline_stage WHERE id = $1` _, err := tx.Exec(query, s.ID) @@ -337,14 +296,6 @@ func seleteAllStageID(db gorp.SqlExecutor, pipelineID int64) ([]int64, error) { return stageIDs, nil } -func deleteStagePrerequisites(db gorp.SqlExecutor, stageID int64) error { - log.Debug("deleteStagePrerequisites> delete prequisites for stage %d ", stageID) - //Delete stage prequisites - queryDelete := `DELETE FROM pipeline_stage_prerequisite WHERE pipeline_stage_id = $1` - _, err := db.Exec(queryDelete, strconv.Itoa(int(stageID))) - return sdk.WithStack(err) -} - // DeleteAllStage Delete all stages from pipeline ID func DeleteAllStage(ctx context.Context, db gorp.SqlExecutor, pipelineID int64, userID int64) error { stageIDs, err := seleteAllStageID(db, pipelineID) @@ -356,10 +307,6 @@ func DeleteAllStage(ctx context.Context, db gorp.SqlExecutor, pipelineID int64, if err := DeletePipelineActionByStage(ctx, db, id); err != nil { return err } - // delete stage prequisites - if err := deleteStagePrerequisites(db, id); err != nil { - return err - } } queryDelete := `DELETE FROM pipeline_stage WHERE pipeline_id = $1` diff --git a/engine/api/pipeline_stage_test.go b/engine/api/pipeline_stage_test.go index 11bd10af56..ee46144e3c 100644 --- a/engine/api/pipeline_stage_test.go +++ b/engine/api/pipeline_stage_test.go @@ -70,7 +70,7 @@ func deleteAll(t *testing.T, api *API, key string) error { return nil } -func TestInsertAndLoadPipelineWith1StageAnd0ActionWithoutPrerequisite(t *testing.T) { +func TestInsertAndLoadPipelineWith1StageAnd0ActionWithoutCondition(t *testing.T) { api, db, _, end := newTestAPI(t, bootstrap.InitiliazeDB) defer end() deleteAll(t, api, "TESTPIPELINESTAGES") @@ -89,11 +89,10 @@ func TestInsertAndLoadPipelineWith1StageAnd0ActionWithoutPrerequisite(t *testing //Insert Stage stage := &sdk.Stage{ - Name: "stage_Test_0", - PipelineID: pip.ID, - BuildOrder: 1, - Enabled: true, - Prerequisites: []sdk.Prerequisite{}, + Name: "stage_Test_0", + PipelineID: pip.ID, + BuildOrder: 1, + Enabled: true, } pip.Stages = append(pip.Stages, *stage) @@ -110,7 +109,7 @@ func TestInsertAndLoadPipelineWith1StageAnd0ActionWithoutPrerequisite(t *testing assert.Equal(t, len(pip.Stages), len(loadedPip.Stages)) assert.Equal(t, pip.Stages[0].Name, loadedPip.Stages[0].Name) assert.Equal(t, pip.Stages[0].Enabled, loadedPip.Stages[0].Enabled) - assert.Equal(t, len(pip.Stages[0].Prerequisites), len(loadedPip.Stages[0].Prerequisites)) + assert.Equal(t, len(pip.Stages[0].Conditions.PlainConditions), len(loadedPip.Stages[0].Conditions.PlainConditions)) assert.Equal(t, len(pip.Stages[0].Jobs), len(loadedPip.Stages[0].Jobs)) //Delete pipeline @@ -123,7 +122,7 @@ func TestInsertAndLoadPipelineWith1StageAnd0ActionWithoutPrerequisite(t *testing test.NoError(t, err) } -func TestInsertAndLoadPipelineWith1StageAnd1ActionWithoutPrerequisite(t *testing.T) { +func TestInsertAndLoadPipelineWith1StageAnd1ActionWithoutCondition(t *testing.T) { api, db, _, end := newTestAPI(t, bootstrap.InitiliazeDB) defer end() @@ -143,11 +142,10 @@ func TestInsertAndLoadPipelineWith1StageAnd1ActionWithoutPrerequisite(t *testing //Insert Stage stage := &sdk.Stage{ - Name: "stage_Test_0", - PipelineID: pip.ID, - BuildOrder: 1, - Enabled: true, - Prerequisites: []sdk.Prerequisite{}, + Name: "stage_Test_0", + PipelineID: pip.ID, + BuildOrder: 1, + Enabled: true, } pip.Stages = append(pip.Stages, *stage) @@ -179,7 +177,7 @@ func TestInsertAndLoadPipelineWith1StageAnd1ActionWithoutPrerequisite(t *testing assert.Equal(t, 1, len(loadedPip.Stages)) assert.Equal(t, pip.Stages[0].Name, loadedPip.Stages[0].Name) assert.Equal(t, pip.Stages[0].Enabled, loadedPip.Stages[0].Enabled) - assert.Equal(t, 0, len(loadedPip.Stages[0].Prerequisites)) + assert.Equal(t, 0, len(loadedPip.Stages[0].Conditions.PlainConditions)) assert.Equal(t, 1, len(loadedPip.Stages[0].Jobs)) assert.Equal(t, job.Action.Name, loadedPip.Stages[0].Jobs[0].Action.Name) assert.Equal(t, true, loadedPip.Stages[0].Jobs[0].Enabled) @@ -194,7 +192,7 @@ func TestInsertAndLoadPipelineWith1StageAnd1ActionWithoutPrerequisite(t *testing test.NoError(t, err) } -func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2ActionsStageWithoutPrerequisite(t *testing.T) { +func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2ActionsStageWithoutCondition(t *testing.T) { api, db, _, end := newTestAPI(t, bootstrap.InitiliazeDB) defer end() @@ -214,11 +212,10 @@ func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2Actio //Insert Stage stage0 := &sdk.Stage{ - Name: "stage_Test_0", - PipelineID: pip.ID, - BuildOrder: 1, - Enabled: true, - Prerequisites: []sdk.Prerequisite{}, + Name: "stage_Test_0", + PipelineID: pip.ID, + BuildOrder: 1, + Enabled: true, } pip.Stages = append(pip.Stages, *stage0) @@ -227,11 +224,10 @@ func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2Actio //Insert Stage stage1 := &sdk.Stage{ - Name: "stage_Test_1", - PipelineID: pip.ID, - BuildOrder: 2, - Enabled: true, - Prerequisites: []sdk.Prerequisite{}, + Name: "stage_Test_1", + PipelineID: pip.ID, + BuildOrder: 2, + Enabled: true, } pip.Stages = append(pip.Stages, *stage1) @@ -276,12 +272,14 @@ func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2Actio assert.Equal(t, pip.Stages[0].Name, loadedPip.Stages[0].Name) assert.Equal(t, pip.Stages[0].Enabled, loadedPip.Stages[0].Enabled) - assert.Equal(t, 0, len(loadedPip.Stages[0].Prerequisites)) + assert.Equal(t, 0, len(loadedPip.Stages[0].Conditions.PlainConditions)) + assert.Equal(t, 0, len(loadedPip.Stages[0].Conditions.LuaScript)) assert.Equal(t, 0, len(loadedPip.Stages[0].Jobs)) assert.Equal(t, pip.Stages[1].Name, loadedPip.Stages[1].Name) assert.Equal(t, pip.Stages[1].Enabled, loadedPip.Stages[1].Enabled) - assert.Equal(t, 0, len(loadedPip.Stages[1].Prerequisites)) + assert.Equal(t, 0, len(loadedPip.Stages[1].Conditions.PlainConditions)) + assert.Equal(t, 0, len(loadedPip.Stages[1].Conditions.LuaScript)) assert.Equal(t, 2, len(loadedPip.Stages[1].Jobs)) assert.Equal(t, job.Action.Name, loadedPip.Stages[1].Jobs[0].Action.Name) @@ -300,7 +298,7 @@ func TestInsertAndLoadPipelineWith2StagesWithAnEmptyStageAtFirstFollowedBy2Actio test.NoError(t, err) } -func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequisites(t *testing.T) { +func TestInsertAndLoadPipelineWith1StageWithoutConditionAnd1StageWith2Conditions(t *testing.T) { api, db, _, end := newTestAPI(t, bootstrap.InitiliazeDB) defer end() @@ -320,11 +318,10 @@ func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequ //Insert Stage stage := &sdk.Stage{ - Name: "stage_Test_0", - PipelineID: pip.ID, - BuildOrder: 1, - Enabled: true, - Prerequisites: []sdk.Prerequisite{}, + Name: "stage_Test_0", + PipelineID: pip.ID, + BuildOrder: 1, + Enabled: true, } pip.Stages = append(pip.Stages, *stage) @@ -350,14 +347,18 @@ func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequ PipelineID: pip.ID, BuildOrder: 2, Enabled: true, - Prerequisites: []sdk.Prerequisite{ - { - Parameter: ".git.branch", - ExpectedValue: "master", - }, - { - Parameter: ".git.author", - ExpectedValue: "someone@somewhere.com", + Conditions: sdk.WorkflowNodeConditions{ + PlainConditions: []sdk.WorkflowNodeCondition{ + sdk.WorkflowNodeCondition{ + Variable: ".git.branch", + Operator: "regex", + Value: "master", + }, + sdk.WorkflowNodeCondition{ + Variable: ".git.author", + Operator: "regex", + Value: "someone@somewhere.com", + }, }, }, } @@ -391,13 +392,13 @@ func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequ assert.Equal(t, pip.Stages[0].Name, loadedPip.Stages[0].Name) assert.Equal(t, pip.Stages[0].Enabled, loadedPip.Stages[0].Enabled) - assert.Equal(t, 0, len(loadedPip.Stages[0].Prerequisites)) + assert.Equal(t, 0, len(loadedPip.Stages[0].Conditions.PlainConditions)) assert.Equal(t, 1, len(loadedPip.Stages[0].Jobs)) assert.Equal(t, pip.Stages[1].Name, loadedPip.Stages[1].Name) assert.Equal(t, pip.Stages[1].Enabled, loadedPip.Stages[1].Enabled) - assert.Equal(t, 2, len(loadedPip.Stages[1].Prerequisites)) + assert.Equal(t, 2, len(loadedPip.Stages[1].Conditions.PlainConditions)) assert.Equal(t, 1, len(loadedPip.Stages[1].Jobs)) assert.Equal(t, job.Action.Name, loadedPip.Stages[0].Jobs[0].Action.Name) @@ -406,16 +407,16 @@ func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequ assert.Equal(t, job1.Action.Name, loadedPip.Stages[1].Jobs[0].Action.Name) assert.Equal(t, true, loadedPip.Stages[1].Jobs[0].Enabled) - assert.Equal(t, 2, len(loadedPip.Stages[1].Prerequisites)) + assert.Equal(t, 2, len(loadedPip.Stages[1].Conditions.PlainConditions)) var foundGitBranch, foundGitAuthor bool - for _, p := range loadedPip.Stages[1].Prerequisites { - if p.Parameter == ".git.branch" { - assert.Equal(t, "master", p.ExpectedValue) + for _, p := range loadedPip.Stages[1].Conditions.PlainConditions { + if p.Variable == ".git.branch" { + assert.Equal(t, "master", p.Value) foundGitBranch = true } - if p.Parameter == ".git.author" { - assert.Equal(t, "someone@somewhere.com", p.ExpectedValue) + if p.Variable == ".git.author" { + assert.Equal(t, "someone@somewhere.com", p.Value) foundGitAuthor = true } } @@ -433,7 +434,7 @@ func TestInsertAndLoadPipelineWith1StageWithoutPrerequisiteAnd1StageWith2Prerequ test.NoError(t, err) } -func TestDeleteStageByIDShouldDeleteStagePrerequisites(t *testing.T) { +func TestDeleteStageByIDShouldDeleteStageConditions(t *testing.T) { api, db, _, end := newTestAPI(t, bootstrap.InitiliazeDB) defer end() @@ -457,10 +458,13 @@ func TestDeleteStageByIDShouldDeleteStagePrerequisites(t *testing.T) { PipelineID: pip.ID, BuildOrder: 1, Enabled: true, - Prerequisites: []sdk.Prerequisite{ - { - Parameter: ".git.branch", - ExpectedValue: "master", + Conditions: sdk.WorkflowNodeConditions{ + PlainConditions: []sdk.WorkflowNodeCondition{ + sdk.WorkflowNodeCondition{ + Variable: ".git.banch", + Operator: "regex", + Value: "master", + }, }, }, } @@ -491,7 +495,7 @@ func TestDeleteStageByIDShouldDeleteStagePrerequisites(t *testing.T) { test.NoError(t, err) } -func TestUpdateStageShouldUpdateStagePrerequisites(t *testing.T) { +func TestUpdateStageShouldUpdateStageConditions(t *testing.T) { api, db, _, end := newTestAPI(t, bootstrap.InitiliazeDB) defer end() @@ -515,10 +519,13 @@ func TestUpdateStageShouldUpdateStagePrerequisites(t *testing.T) { PipelineID: pip.ID, BuildOrder: 1, Enabled: true, - Prerequisites: []sdk.Prerequisite{ - { - Parameter: ".git.branch", - ExpectedValue: "master", + Conditions: sdk.WorkflowNodeConditions{ + PlainConditions: []sdk.WorkflowNodeCondition{ + sdk.WorkflowNodeCondition{ + Variable: ".git.banch", + Operator: "regex", + Value: "master", + }, }, }, } @@ -527,14 +534,18 @@ func TestUpdateStageShouldUpdateStagePrerequisites(t *testing.T) { t.Logf("Insert Stage %s for Pipeline %s of Project %s", stage.Name, pip.Name, proj.Name) test.NoError(t, pipeline.InsertStage(api.mustDB(), stage)) - stage.Prerequisites = []sdk.Prerequisite{ - { - Parameter: "param1", - ExpectedValue: "value1", - }, - { - Parameter: "param2", - ExpectedValue: "value2", + stage.Conditions = sdk.WorkflowNodeConditions{ + PlainConditions: []sdk.WorkflowNodeCondition{ + sdk.WorkflowNodeCondition{ + Variable: "param1", + Operator: "regex", + Value: "value1", + }, + sdk.WorkflowNodeCondition{ + Variable: "param2", + Operator: "regex", + Value: "value2", + }, }, } @@ -549,16 +560,16 @@ func TestUpdateStageShouldUpdateStagePrerequisites(t *testing.T) { //Check all the things assert.NotNil(t, loadedPip) assert.Equal(t, 1, len(loadedPip.Stages)) - assert.Equal(t, 2, len(loadedPip.Stages[0].Prerequisites)) + assert.Equal(t, 2, len(loadedPip.Stages[0].Conditions.PlainConditions)) var foundParam1, foundParam2 bool - for _, p := range loadedPip.Stages[0].Prerequisites { - if p.Parameter == "param1" { - assert.Equal(t, "value1", p.ExpectedValue) + for _, p := range loadedPip.Stages[0].Conditions.PlainConditions { + if p.Variable == "param1" { + assert.Equal(t, "value1", p.Value) foundParam1 = true } - if p.Parameter == "param2" { - assert.Equal(t, "value2", p.ExpectedValue) + if p.Variable == "param2" { + assert.Equal(t, "value2", p.Value) foundParam2 = true } } diff --git a/engine/api/stage.go b/engine/api/stage.go index 5d62c0668a..4df4ea12ae 100644 --- a/engine/api/stage.go +++ b/engine/api/stage.go @@ -297,3 +297,70 @@ func (api *API) deleteStageHandler() service.Handler { return service.WriteJSON(w, pipelineData, http.StatusOK) } } + +func (api *API) getStageConditionsHandler() service.Handler { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { + // Get project name in URL + vars := mux.Vars(r) + projectKey := vars[permProjectKey] + pipelineKey := vars["pipelineKey"] + + data := struct { + Operators map[string]string `json:"operators"` + ConditionNames []string `json:"names"` + }{ + Operators: sdk.WorkflowConditionsOperators, + ConditionNames: []string{ + "git.hash", + "git.hash.short", + "git.branch", + "git.tag", + "git.author", + "git.repository", + "git.url", + "git.http_url", + "git.server", + }, + } + + // Check if pipeline exist + pipelineData, err := pipeline.LoadPipeline(api.mustDB(), projectKey, pipelineKey, false) + if err != nil { + return sdk.WrapError(err, "Cannot load pipeline %s", pipelineKey) + } + + pipParams, err := pipeline.GetAllParametersInPipeline(ctx, api.mustDB(), pipelineData.ID) + if err != nil { + return sdk.WrapError(err, "Cannot get all parameters in pipeline") + } + + for _, pipParam := range pipParams { + data.ConditionNames = append(data.ConditionNames, pipParam.Name) + } + + // add cds variable + data.ConditionNames = append(data.ConditionNames, + "cds.version", + "cds.application", + "cds.environment", + "cds.job", + "cds.manual", + "cds.pipeline", + "cds.project", + "cds.run", + "cds.run.number", + "cds.run.subnumber", + "cds.stage", + "cds.triggered_by.email", + "cds.triggered_by.fullname", + "cds.triggered_by.username", + "cds.ui.pipeline.run", + "cds.worker", + "cds.workflow", + "cds.workspace", + "payload", + ) + + return service.WriteJSON(w, data, http.StatusOK) + } +} diff --git a/engine/api/workflow/dao_run_test.go b/engine/api/workflow/dao_run_test.go index 287a83b91c..3d41ffb4aa 100644 --- a/engine/api/workflow/dao_run_test.go +++ b/engine/api/workflow/dao_run_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "fmt" "io/ioutil" "net/http" "testing" @@ -246,7 +245,6 @@ vcs_ssh_key: proj-blabla toDeleteNb := 0 for _, wfRun := range wruns { - fmt.Printf("%+v\n=======\n", wfRun) if wfRun.ToDelete { toDeleteNb++ } diff --git a/engine/api/workflow/execute_node_run.go b/engine/api/workflow/execute_node_run.go index a4dabaa6f6..ce7d3c1535 100644 --- a/engine/api/workflow/execute_node_run.go +++ b/engine/api/workflow/execute_node_run.go @@ -331,13 +331,9 @@ func addJobsToQueue(ctx context.Context, db gorp.SqlExecutor, stage *sdk.Stage, report := new(ProcessorReport) - _, next := observability.Span(ctx, "sdk.WorkflowCheckConditions") - conditionsOK, err := sdk.WorkflowCheckConditions(stage.Conditions(), run.BuildParameters) + _, next := observability.Span(ctx, "checkCondition") + conditionsOK := checkCondition(wr, stage.Conditions, run.BuildParameters) next() - if err != nil { - return report, sdk.WrapError(err, "cannot compute prerequisites on stage %s(%d)", stage.Name, stage.ID) - } - if !conditionsOK { stage.Status = sdk.StatusSkipped } diff --git a/engine/api/workflow/process.go b/engine/api/workflow/process.go index d18139f1d1..4f139c764f 100644 --- a/engine/api/workflow/process.go +++ b/engine/api/workflow/process.go @@ -34,7 +34,7 @@ func setValuesGitInBuildParameters(run *sdk.WorkflowNodeRun, vcsInfos vcsInfos) sdk.ParameterAddOrSetValue(&run.BuildParameters, tagGitServer, sdk.StringParameter, vcsInfos.Server) } -func checkNodeRunCondition(wr *sdk.WorkflowRun, conditions sdk.WorkflowNodeConditions, params []sdk.Parameter) bool { +func checkCondition(wr *sdk.WorkflowRun, conditions sdk.WorkflowNodeConditions, params []sdk.Parameter) bool { var conditionsOK bool var errc error if conditions.LuaScript == "" { diff --git a/engine/api/workflow/process_node.go b/engine/api/workflow/process_node.go index 8f6f5ab0ca..3584e4e15d 100644 --- a/engine/api/workflow/process_node.go +++ b/engine/api/workflow/process_node.go @@ -264,12 +264,12 @@ func processNode(ctx context.Context, db gorp.SqlExecutor, store cache.Store, pr return nil, false, sdk.WrapError(sdk.ErrWorkflowNodeNotFound, "Unable to find node %d", hook.NodeID) } - if !checkNodeRunCondition(wr, dest.Context.Conditions, params) { + if !checkCondition(wr, dest.Context.Conditions, params) { log.Debug("Avoid trigger workflow from hook %s", hook.UUID) return nil, false, nil } } else { - if !checkNodeRunCondition(wr, n.Context.Conditions, run.BuildParameters) { + if !checkCondition(wr, n.Context.Conditions, run.BuildParameters) { log.Debug("Condition failed %d/%d %+v", wr.ID, n.ID, run.BuildParameters) return nil, false, nil } diff --git a/engine/api/workflow/process_outgoinghook.go b/engine/api/workflow/process_outgoinghook.go index 909632f799..74962c0875 100644 --- a/engine/api/workflow/process_outgoinghook.go +++ b/engine/api/workflow/process_outgoinghook.go @@ -101,7 +101,7 @@ func processNodeOutGoingHook(ctx context.Context, db gorp.SqlExecutor, store cac OutgoingHook: node.OutGoingHookContext, } - if !checkNodeRunCondition(wr, node.Context.Conditions, hookRun.BuildParameters) { + if !checkCondition(wr, node.Context.Conditions, hookRun.BuildParameters) { log.Debug("Condition failed %d/%d %+v", wr.ID, node.ID, hookRun.BuildParameters) return report, false, nil } diff --git a/engine/sql/167_stage_condition.sql b/engine/sql/167_stage_condition.sql new file mode 100644 index 0000000000..6c705f1899 --- /dev/null +++ b/engine/sql/167_stage_condition.sql @@ -0,0 +1,5 @@ +-- +migrate Up +ALTER TABLE pipeline_stage ADD COLUMN conditions JSONB; + +-- +migrate Down +ALTER TABLE pipeline_stage DROP COLUMN conditions; \ No newline at end of file diff --git a/sdk/exportentities/pipeline.go b/sdk/exportentities/pipeline.go index 85241a2779..4cef7c4870 100644 --- a/sdk/exportentities/pipeline.go +++ b/sdk/exportentities/pipeline.go @@ -36,9 +36,9 @@ const ( // Stage represents exported sdk.Stage type Stage struct { - Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` - Jobs map[string]Job `json:"jobs,omitempty" yaml:"jobs,omitempty"` - Conditions map[string]string `json:"conditions,omitempty" yaml:"conditions,omitempty"` + Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + Jobs map[string]Job `json:"jobs,omitempty" yaml:"jobs,omitempty"` + Conditions *sdk.WorkflowNodeConditions `json:"conditions,omitempty" yaml:"conditions,omitempty"` } // Job represents exported sdk.Job @@ -120,13 +120,11 @@ func newStagesForPipelineV1(stages []sdk.Stage) ([]string, map[string]Stage) { st.Enabled = &s.Enabled hasOptions = true } - if len(s.Prerequisites) > 0 { - st.Conditions = make(map[string]string) + if len(s.Conditions.PlainConditions) > 0 || s.Conditions.LuaScript != "" { + st.Conditions = &s.Conditions hasOptions = true } - for _, r := range s.Prerequisites { - st.Conditions[r.Parameter] = r.ExpectedValue - } + if hasOptions == true { opts[s.Name] = st } @@ -305,12 +303,8 @@ func (p PipelineV1) Pipeline() (pip *sdk.Pipeline, err error) { mapStages[s].Enabled = true } - //Compute stage Prerequisites - for n, c := range opt.Conditions { - mapStages[s].Prerequisites = append(mapStages[s].Prerequisites, sdk.Prerequisite{ - Parameter: n, - ExpectedValue: c, - }) + if opt.Conditions != nil { + mapStages[s].Conditions = *opt.Conditions } } diff --git a/sdk/exportentities/pipeline_test.go b/sdk/exportentities/pipeline_test.go index 4fb061d96d..02c7d92aef 100644 --- a/sdk/exportentities/pipeline_test.go +++ b/sdk/exportentities/pipeline_test.go @@ -319,10 +319,13 @@ var ( BuildOrder: 2, Name: "stage 2", Enabled: true, - Prerequisites: []sdk.Prerequisite{ - { - Parameter: "param1", - ExpectedValue: "value1", + Conditions: sdk.WorkflowNodeConditions{ + PlainConditions: []sdk.WorkflowNodeCondition{ + { + Variable: "param1", + Operator: "regex", + Value: "value1", + }, }, }, Jobs: []sdk.Job{{ @@ -465,7 +468,7 @@ func TestExportAndImportPipeline_YAML(t *testing.T) { assert.Equal(t, stage.BuildOrder, s1.BuildOrder, "Build order does not match") assert.Equal(t, stage.Enabled, s1.Enabled, "Enabled does not match") - test.EqualValuesWithoutOrder(t, stage.Prerequisites, s1.Prerequisites) + test.EqualValuesWithoutOrder(t, stage.Conditions.PlainConditions, s1.Conditions.PlainConditions) for _, j := range stage.Jobs { var jobFound bool @@ -643,7 +646,7 @@ func TestExportAndImportPipelineV1_YAML(t *testing.T) { assert.Equal(t, stage.BuildOrder, s1.BuildOrder, "Build order does not match") assert.Equal(t, stage.Enabled, s1.Enabled, "Enabled does not match") - test.EqualValuesWithoutOrder(t, stage.Prerequisites, s1.Prerequisites) + test.EqualValuesWithoutOrder(t, stage.Conditions.PlainConditions, s1.Conditions.PlainConditions) for _, j := range stage.Jobs { var jobFound bool diff --git a/sdk/pipeline.go b/sdk/pipeline.go index 86a5cde2d9..224029818e 100644 --- a/sdk/pipeline.go +++ b/sdk/pipeline.go @@ -6,17 +6,17 @@ import ( // Pipeline represents the complete behavior of CDS for each projects type Pipeline struct { - ID int64 `json:"id" yaml:"-"` - Name string `json:"name" cli:"name,key"` - Description string `json:"description" cli:"description"` - ProjectKey string `json:"projectKey"` - ProjectID int64 `json:"-"` + ID int64 `json:"id" yaml:"-" db:"id"` + Name string `json:"name" cli:"name,key" db:"name"` + Description string `json:"description" cli:"description" db:"description"` + ProjectKey string `json:"projectKey" db:"projectKey"` + ProjectID int64 `json:"-" db:"project_id"` Stages []Stage `json:"stages"` Parameter []Parameter `json:"parameters,omitempty"` Usage *Usage `json:"usage,omitempty"` Permission int `json:"permission"` LastModified int64 `json:"last_modified" cli:"modified"` - FromRepository string `json:"from_repository" cli:"from_repository"` + FromRepository string `json:"from_repository" cli:"from_repository" db:"from_repository"` } // PipelineAudit represents pipeline audit diff --git a/sdk/stage.go b/sdk/stage.go index 9f34ef810e..c4200b2909 100644 --- a/sdk/stage.go +++ b/sdk/stage.go @@ -12,7 +12,8 @@ type Stage struct { BuildOrder int `json:"build_order"` Enabled bool `json:"enabled"` RunJobs []WorkflowNodeJobRun `json:"run_jobs"` - Prerequisites []Prerequisite `json:"prerequisites"` + Prerequisites []Prerequisite `json:"prerequisites"` //TODO: to delete + Conditions WorkflowNodeConditions `json:"conditions"` LastModified int64 `json:"last_modified"` Jobs []Job `json:"jobs"` Status Status `json:"status"` @@ -48,7 +49,7 @@ func (s Stage) ToSummary() StageSummary { } // Conditions returns stage prerequisites as a set of WorkflowTriggerCondition regex -func (s *Stage) Conditions() []WorkflowNodeCondition { +func (s *Stage) PlainConditions() []WorkflowNodeCondition { res := make([]WorkflowNodeCondition, len(s.Prerequisites)) for i, p := range s.Prerequisites { if !strings.HasPrefix(p.Parameter, "workflow.") && !strings.HasPrefix(p.Parameter, "git.") { diff --git a/sdk/workflow.go b/sdk/workflow.go index 6aca55cc38..ed630ff796 100644 --- a/sdk/workflow.go +++ b/sdk/workflow.go @@ -823,7 +823,7 @@ func (n WorkflowNode) migrate(withID bool) Node { Conditions: n.Context.Conditions, DefaultPayload: n.Context.DefaultPayload, DefaultPipelineParameters: n.Context.DefaultPipelineParameters, - Mutex: n.Context.Mutex, + Mutex: n.Context.Mutex, }, Hooks: make([]NodeHook, 0, len(n.Hooks)), Triggers: make([]NodeTrigger, 0, len(n.Triggers)+len(n.Forks)+len(n.OutgoingHooks)), @@ -1489,9 +1489,9 @@ type WorkflowNodeConditions struct { //WorkflowNodeCondition represents a condition to trigger ot not a pipeline in a workflow. Operator can be =, !=, regex type WorkflowNodeCondition struct { - Variable string `json:"variable"` - Operator string `json:"operator"` - Value string `json:"value"` + Variable string `json:"variable" yaml:"variable"` + Operator string `json:"operator" yaml:"operator"` + Value string `json:"value" yaml:"value"` } //WorkflowNodeContext represents a context attached on a node diff --git a/ui/resources/cds-hint.js b/ui/resources/cds-hint.js index 69afaa27c1..e90a39e1cf 100644 --- a/ui/resources/cds-hint.js +++ b/ui/resources/cds-hint.js @@ -2,17 +2,17 @@ * Hint addon for codemirror */ -(function(mod) { +(function (mod) { if (typeof exports == "object" && typeof module == "object") // CommonJS mod(require("../../lib/codemirror"), require("../../mode/css/css")); else if (typeof define == "function" && define.amd) // AMD define(["../../lib/codemirror", "../../mode/css/css"], mod); else // Plain browser env mod(CodeMirror); -})(function(CodeMirror) { +})(function (CodeMirror) { "use strict"; - CodeMirror.registerHelper("hint", "cds", function(cm, options) { + CodeMirror.registerHelper("hint", "cds", function (cm, options) { // Suggest list var cdsCompletionList = options.cdsCompletionList; @@ -24,7 +24,7 @@ var text = ''; if (!line) { - return null; + return null; } text = line.text; @@ -49,12 +49,15 @@ list: cdsCompletionList.filter(function (l) { return l.indexOf(areaBefore.substring(areaBefore.lastIndexOf('{{.'))) !== -1; }), - from: { line: cur.line, ch: areaBefore.lastIndexOf('{{.')}, + from: { + line: cur.line, + ch: areaBefore.lastIndexOf('{{.') + }, to: CodeMirror.Pos(cur.line, cur.ch) }; }); - CodeMirror.registerHelper("hint", "condition", function(cm, options) { + CodeMirror.registerHelper("hint", "condition", function (cm, options) { var cdsPrefix = 'cds_'; var workflowPrefix = 'workflow_'; var gitPrefix = 'git_'; @@ -63,7 +66,6 @@ // Get cursor position var cur = cm.getCursor(0); - // Get current line var text = cm.doc.children[0].lines[cur.line].text; if (text.indexOf(cdsPrefix) === -1 && text.indexOf(workflowPrefix) === -1 && text.indexOf(gitPrefix) === -1) { @@ -82,12 +84,15 @@ l.indexOf(areaBefore.substring(areaBefore.lastIndexOf(workflowPrefix))) !== -1 || l.indexOf(areaBefore.substring(areaBefore.lastIndexOf(gitPrefix))) !== -1; }), - from: { line: cur.line, ch: ch}, + from: { + line: cur.line, + ch: ch + }, to: CodeMirror.Pos(cur.line, cur.ch) }; }); - CodeMirror.registerHelper("hint", "payload", function(cm, options) { + CodeMirror.registerHelper("hint", "payload", function (cm, options) { var branchPrefix = '"git.branch":'; var tagPrefix = '"git.tag":'; var repoPrefix = '"git.repository":'; @@ -106,7 +111,7 @@ return null; } - switch(true) { + switch (true) { case text.indexOf(branchPrefix) !== -1: payloadCompletionList = options.payloadCompletionList.branches; prefix = branchPrefix; @@ -148,7 +153,10 @@ return { list: payloadCompletionList, - from: { line: cur.line, ch: lastIndexOfPrefix + prefix.length + inc}, + from: { + line: cur.line, + ch: lastIndexOfPrefix + prefix.length + inc + }, to: CodeMirror.Pos(cur.line, lastIndexOfComma) }; }); diff --git a/ui/src/app/app.component.spec.ts b/ui/src/app/app.component.spec.ts index 3d6b26efa4..02309914fa 100644 --- a/ui/src/app/app.component.spec.ts +++ b/ui/src/app/app.component.spec.ts @@ -6,6 +6,8 @@ import { getTestBed, TestBed } from '@angular/core/testing'; import { ActivatedRoute } from '@angular/router'; import { RouterTestingModule } from '@angular/router/testing'; import { Store } from '@ngxs/store'; +import { WorkflowRunService } from 'app/service/workflow/run/workflow.run.service'; +import { WorkflowService } from 'app/service/workflow/workflow.service'; import { of } from 'rxjs'; import 'rxjs/add/observable/of'; import { AppComponent } from './app.component'; @@ -20,8 +22,6 @@ import { PipelineService } from './service/pipeline/pipeline.service'; import { ProjectService } from './service/project/project.service'; import { SharedModule } from './shared/shared.module'; import { NgxsStoreModule } from './store/store.module'; -import { WorkflowService } from 'app/service/workflow/workflow.service'; -import { WorkflowRunService } from 'app/service/workflow/run/workflow.run.service'; describe('App: CDS', () => { diff --git a/ui/src/app/model/stage.model.ts b/ui/src/app/model/stage.model.ts index 6b85f3c8c6..a5e5a130f0 100644 --- a/ui/src/app/model/stage.model.ts +++ b/ui/src/app/model/stage.model.ts @@ -1,6 +1,6 @@ import { ActionWarning } from './action.model'; import { Job } from './job.model'; -import { Prerequisite } from './prerequisite.model'; +import { WorkflowNodeConditions } from './workflow.model'; import { WorkflowNodeJobRun } from './workflow.run.model'; export class Stage { @@ -11,7 +11,7 @@ export class Stage { enabled: boolean; jobs: Array; run_jobs: Array; - prerequisites: Array; + conditions: WorkflowNodeConditions; last_modified: number; warnings: Array; // UI params diff --git a/ui/src/app/service/pipeline/pipeline.service.ts b/ui/src/app/service/pipeline/pipeline.service.ts index 73b4c30333..fef3c339c0 100644 --- a/ui/src/app/service/pipeline/pipeline.service.ts +++ b/ui/src/app/service/pipeline/pipeline.service.ts @@ -1,6 +1,7 @@ import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; import { Injectable } from '@angular/core'; +import { WorkflowTriggerConditionCache } from 'app/model/workflow.model'; import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; import { Application } from '../../model/application.model'; @@ -110,6 +111,16 @@ export class PipelineService { return this._http.get('/project/' + key + '/pipeline/' + pipName + '/application'); } + /** + * Get the list of condition names for a given pipeline + * @param key Project unique key + * @param pipName Pipeline name + * @returns {Observable} + */ + getStageConditionsName(key: string, pipName: string): Observable { + return this._http.get('/project/' + key + '/pipeline/' + pipName + '/stage/condition'); + } + /** * Insert a new Stage * @param key Project unique key diff --git a/ui/src/app/shared/conditions/conditions.component.ts b/ui/src/app/shared/conditions/conditions.component.ts new file mode 100644 index 0000000000..ca584bac87 --- /dev/null +++ b/ui/src/app/shared/conditions/conditions.component.ts @@ -0,0 +1,148 @@ +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core'; +import { PermissionValue } from 'app/model/permission.model'; +import { PipelineStatus } from 'app/model/pipeline.model'; +import { Project } from 'app/model/project.model'; +import { WorkflowNodeCondition, WorkflowNodeConditions, WorkflowTriggerConditionCache } from 'app/model/workflow.model'; +import { ThemeStore } from 'app/service/theme/theme.store'; +import { AutoUnsubscribe } from 'app/shared/decorator/autoUnsubscribe'; +import { CodemirrorComponent } from 'ng2-codemirror-typescript/Codemirror'; +import { Subscription } from 'rxjs/Subscription'; +import { Table } from '../table/table'; +declare var CodeMirror: any; + +@Component({ + selector: 'app-conditions', + templateUrl: './conditions.html', + styleUrls: ['./conditions.scss'] +}) +@AutoUnsubscribe() +export class ConditionsComponent extends Table implements OnInit { + @Input('triggerConditions') set triggerConditions(data: WorkflowTriggerConditionCache) { + this._triggerCondition = data; + if (data) { + this.operators = Object.keys(data.operators).map(k => { + return { key: k, value: data.operators[k] }; + }); + this.conditionNames = data.names; + if (this.conditionNames) { + this.suggest = this.conditionNames.map((d) => d.replace(/-|\./g, '_')); + } + } + } + get triggerConditions(): WorkflowTriggerConditionCache { + return this._triggerCondition; + } + @Input('conditions') set conditions(conditions: WorkflowNodeConditions) { + this._conditions = conditions; + if (this._conditions.lua_script && this._conditions.lua_script !== '') { + this.isAdvanced = true; + } else { + this.isAdvanced = false; + } + } + get conditions(): WorkflowNodeConditions { + return this._conditions; + } + + @Input() project: Project; + @Input() pipelineId: number; + + _conditions: WorkflowNodeConditions; + @Input() readonly = true; + + @Output() conditionsChange = new EventEmitter(); + + @ViewChild('textareaCodeMirror') codemirror: CodemirrorComponent; + codeMirrorConfig: any; + isAdvanced = false; + suggest: Array = []; + loadingConditions = false; + operators: Array; + conditionNames: Array; + permission = PermissionValue; + statuses = [PipelineStatus.SUCCESS, PipelineStatus.FAIL, PipelineStatus.SKIPPED]; + loading = false; + previousValue: string; + themeSubscription: Subscription; + + _triggerCondition: WorkflowTriggerConditionCache; + + constructor( + private _theme: ThemeStore + ) { + super(); + } + + getData(): Array { + return undefined; + } + + ngOnInit(): void { + this.codeMirrorConfig = { + matchBrackets: true, + autoCloseBrackets: true, + mode: 'lua', + lineWrapping: true, + lineNumbers: true, + autoRefresh: true, + readOnly: this.readonly, + }; + + this.themeSubscription = this._theme.get().subscribe(t => { + this.codeMirrorConfig.theme = t === 'night' ? 'darcula' : 'default'; + if (this.codemirror && this.codemirror.instance) { + this.codemirror.instance.setOption('theme', this.codeMirrorConfig.theme); + } + }); + + if (!this.conditions) { + this.conditions = new WorkflowNodeConditions(); + } + if (!this.conditions.plain) { + this.conditions.plain = new Array(); + } + + this.previousValue = this.conditions.lua_script; + let condition = this.conditions.plain.find(cc => cc.variable === 'cds.manual'); + if (condition) { + condition.value = (condition.value !== 'false'); + } + } + + removeCondition(index: number): void { + this.conditions.plain.splice(index, 1); + this.pushChange('remove'); + } + + addEmptyCondition(): void { + let emptyCond = new WorkflowNodeCondition(); + emptyCond.operator = 'eq'; + this.conditions.plain.push(emptyCond); + } + + pushChange(event: string, e?: string): void { + if (event !== 'codemirror') { + this.conditionsChange.emit(this.conditions); + return; + } + if (event === 'codemirror' && e && e !== this.previousValue) { + this.previousValue = e; + this.conditionsChange.emit(this.conditions); + } + return; + + } + + changeCodeMirror(): void { + this.codemirror.instance.on('keyup', (cm, event) => { + if (!cm.state.completionActive && (event.keyCode > 46 || event.keyCode === 32)) { + CodeMirror.showHint(cm, CodeMirror.hint.condition, { + completeSingle: true, + closeCharacters: / /, + cdsCompletionList: this.suggest || [], + specialChars: '' + }); + } + }); + } +} diff --git a/ui/src/app/shared/conditions/conditions.html b/ui/src/app/shared/conditions/conditions.html new file mode 100644 index 0000000000..be90557d1f --- /dev/null +++ b/ui/src/app/shared/conditions/conditions.html @@ -0,0 +1,133 @@ +
+
+ {{'workflow_node_condition_advanced' | translate }} + +
+
+

+ {{'workflow_node_condition_warning' | translate}} +

+
+ +
+
+ {{ 'workflow_node_trigger_condition_no' | translate }} + +
+
+ + + + + + + + + + + + + + + + + + + + + + + +
{{ 'workflow_node_trigger_condition_name' | translate }}{{ 'workflow_node_trigger_condition_operator' | translate }}{{ 'workflow_node_trigger_condition_value' | translate }} + +
+
+ + + + + + +
+ 		+
+
+ + + + + + +
{{c.operator}}
+ + 		+
+ +
+ + + + + + + + + + +
+ + +
{{c.value}}
+ + 		+ +
+ +
+ + + +
+
+

{{'workflow_node_condition_lua_title' | translate}}

+ {{'workflow_node_condition_lua_help' | translate}} +
+ + +
+
+ + +
+ +
diff --git a/ui/src/app/shared/conditions/conditions.scss b/ui/src/app/shared/conditions/conditions.scss new file mode 100644 index 0000000000..8c91059419 --- /dev/null +++ b/ui/src/app/shared/conditions/conditions.scss @@ -0,0 +1,18 @@ +@import '../../../common'; + +td { + &.middle-aligned { + vertical-align: middle !important; + } +} + +.mb { + margin-bottom: 20px; +} + +.empty { + display: flex; + align-items: center; + flex-direction: row; + justify-content: space-between +} diff --git a/ui/src/app/shared/shared.module.ts b/ui/src/app/shared/shared.module.ts index 659dc3ab7f..40005c408c 100644 --- a/ui/src/app/shared/shared.module.ts +++ b/ui/src/app/shared/shared.module.ts @@ -5,6 +5,7 @@ import { RouterModule } from '@angular/router'; import { NguiAutoCompleteModule } from '@ngui/auto-complete'; import { TranslateModule } from '@ngx-translate/core'; import { NgxChartsModule } from '@swimlane/ngx-charts'; +import { ConditionsComponent } from 'app/shared/conditions/conditions.component'; import { WorkflowHookMenuEditComponent } from 'app/shared/workflow/menu/edit-hook/menu.edit.hook.component'; import { WorkflowWizardNodeConditionComponent } from 'app/shared/workflow/wizard/conditions/wizard.conditions.component'; import { WorkflowWizardOutgoingHookComponent } from 'app/shared/workflow/wizard/outgoinghook/wizard.outgoinghook.component'; @@ -209,6 +210,7 @@ import { ZoneComponent } from './zone/zone.component'; WorkflowWizardNodeContextComponent, WorkflowWizardNodeInputComponent, WorkflowWizardNodeConditionComponent, + ConditionsComponent, ZoneComponent, ZoneContentComponent, UsageWorkflowsComponent, @@ -270,6 +272,7 @@ import { ZoneComponent } from './zone/zone.component'; TokenListComponent, NgSemanticModule, NgxAutoScroll, + ConditionsComponent, ParameterDescriptionComponent, ParameterListComponent, ParameterFormComponent, diff --git a/ui/src/app/shared/workflow/wizard/conditions/wizard.conditions.component.ts b/ui/src/app/shared/workflow/wizard/conditions/wizard.conditions.component.ts index dbf0967747..a577f175d5 100644 --- a/ui/src/app/shared/workflow/wizard/conditions/wizard.conditions.component.ts +++ b/ui/src/app/shared/workflow/wizard/conditions/wizard.conditions.component.ts @@ -4,7 +4,14 @@ import { Store } from '@ngxs/store'; import { PermissionValue } from 'app/model/permission.model'; import { PipelineStatus } from 'app/model/pipeline.model'; import { Project } from 'app/model/project.model'; -import { WNode, WNodeContext, Workflow, WorkflowNodeCondition, WorkflowNodeConditions } from 'app/model/workflow.model'; +import { + WNode, + WNodeContext, + Workflow, + WorkflowNodeCondition, + WorkflowNodeConditions, + WorkflowTriggerConditionCache +} from 'app/model/workflow.model'; import { ThemeStore } from 'app/service/theme/theme.store'; import { VariableService } from 'app/service/variable/variable.service'; import { WorkflowService } from 'app/service/workflow/workflow.service'; @@ -28,6 +35,7 @@ export class WorkflowWizardNodeConditionComponent extends Table this.loadingConditions = false) ) .subscribe(wtc => { + this.triggerConditions = wtc; this.operators = Object.keys(wtc.operators).map(k => { return { key: k, value: wtc.operators[k] }; }); @@ -121,17 +131,6 @@ export class WorkflowWizardNodeConditionComponent extends Table -
- {{'workflow_node_condition_advanced' | translate }} + + -
- -
-
- {{ 'workflow_node_trigger_condition_no' | translate }} - -
-
- - - - - - - - - - - - - - - - - - - - - - - -
{{ 'workflow_node_trigger_condition_name' | translate }}{{ 'workflow_node_trigger_condition_operator' | translate }}{{ 'workflow_node_trigger_condition_value' | translate }} - -
-
- - - - - - -
- 		-
- - - - - - -
{{c.operator}}
- - 		-
- -
- - - - - - - - - - -
- - -
{{c.value}}
- - 		- -
- -
- - - -
-
-

{{'workflow_node_condition_lua_title' | translate}}

- {{'workflow_node_condition_lua_help' | translate}} -
- - -
-
- - -
- - diff --git a/ui/src/app/views/pipeline/show/workflow/pipeline.workflow.html b/ui/src/app/views/pipeline/show/workflow/pipeline.workflow.html index 5241edef8f..45e4171986 100644 --- a/ui/src/app/views/pipeline/show/workflow/pipeline.workflow.html +++ b/ui/src/app/views/pipeline/show/workflow/pipeline.workflow.html @@ -86,11 +86,10 @@ - \ No newline at end of file + diff --git a/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.spec.ts b/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.spec.ts index 996113bf81..373ea15b51 100644 --- a/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.spec.ts +++ b/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.spec.ts @@ -1,21 +1,17 @@ /* tslint:disable:no-unused-variable */ import { HttpClientTestingModule } from '@angular/common/http/testing'; -import { fakeAsync, TestBed } from '@angular/core/testing'; +import { TestBed } from '@angular/core/testing'; import { ActivatedRoute } from '@angular/router'; import { RouterTestingModule } from '@angular/router/testing'; import { TranslateLoader, TranslateModule, TranslateParser, TranslateService } from '@ngx-translate/core'; import { NavbarService } from 'app/service/navbar/navbar.service'; +import { WorkflowRunService } from 'app/service/workflow/run/workflow.run.service'; +import { WorkflowService } from 'app/service/workflow/workflow.service'; import { NgxsStoreModule } from 'app/store/store.module'; import 'rxjs/add/observable/of'; import { Observable } from 'rxjs/Observable'; -import { Prerequisite } from '../../../../../../model/prerequisite.model'; -import { Stage } from '../../../../../../model/stage.model'; -import { PrerequisiteEvent } from '../../../../../../shared/prerequisites/prerequisite.event.model'; import { SharedModule } from '../../../../../../shared/shared.module'; import { PipelineModule } from '../../../../pipeline.module'; -import { PipelineStageFormComponent } from './pipeline.stage.form.component'; -import {WorkflowService} from 'app/service/workflow/workflow.service'; -import { WorkflowRunService } from 'app/service/workflow/run/workflow.run.service'; describe('CDS: Stage From component', () => { @@ -42,34 +38,6 @@ describe('CDS: Stage From component', () => { ] }); }); - - it('should add and delete prerequisite', fakeAsync(() => { - // Create component - let fixture = TestBed.createComponent(PipelineStageFormComponent); - let component = fixture.debugElement.componentInstance; - expect(component).toBeTruthy(); - - // Init stage - let s = new Stage(); - fixture.componentInstance.stage = s; - - let eventAdd = new PrerequisiteEvent('add', new Prerequisite()); - eventAdd.prerequisite.parameter = 'git.branch'; - eventAdd.prerequisite.expected_value = 'master'; - - fixture.componentInstance.prerequisiteEvent(eventAdd); - // add twice - fixture.componentInstance.prerequisiteEvent(eventAdd); - - expect(fixture.componentInstance.stage.prerequisites.length).toBe(1, 'Must have 1 prerequisite'); - expect(fixture.componentInstance.stage.prerequisites[0].parameter).toBe('git.branch'); - expect(fixture.componentInstance.stage.prerequisites[0].expected_value).toBe('master'); - - - let eventDelete = new PrerequisiteEvent('delete', eventAdd.prerequisite); - fixture.componentInstance.prerequisiteEvent(eventDelete); - expect(fixture.componentInstance.stage.prerequisites.length).toBe(0, 'Must have 0 prerequisite'); - })); }); class MockToast { diff --git a/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.ts b/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.ts index d03d7f85dd..a402ef1fee 100644 --- a/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.ts +++ b/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.component.ts @@ -1,11 +1,11 @@ import { Component, Input, OnInit } from '@angular/core'; -import cloneDeep from 'lodash-es/cloneDeep'; +import { WorkflowTriggerConditionCache } from 'app/model/workflow.model'; +import { PipelineService } from 'app/service/pipeline/pipeline.service'; +import { finalize } from 'rxjs/operators'; import { PermissionValue } from '../../../../../../model/permission.model'; import { Pipeline } from '../../../../../../model/pipeline.model'; -import { Prerequisite } from '../../../../../../model/prerequisite.model'; import { Project } from '../../../../../../model/project.model'; import { Stage } from '../../../../../../model/stage.model'; -import { PrerequisiteEvent } from '../../../../../../shared/prerequisites/prerequisite.event.model'; @Component({ selector: 'app-pipeline-stage-form', @@ -18,53 +18,17 @@ export class PipelineStageFormComponent implements OnInit { @Input() pipeline: Pipeline; @Input() stage: Stage; - availablePrerequisites: Array; permissionEnum = PermissionValue; + triggerConditions: WorkflowTriggerConditionCache; + loading = true; - constructor() { } + constructor(private pipelineService: PipelineService) { - ngOnInit(): void { - this.initPrerequisites(); - } - - private initPrerequisites() { - if (!this.availablePrerequisites) { - this.availablePrerequisites = new Array(); - } - this.availablePrerequisites.push({ - parameter: 'git.branch', - expected_value: '' - }); - - if (this.pipeline.parameters) { - this.pipeline.parameters.forEach(p => { - this.availablePrerequisites.push({ - parameter: p.name, - expected_value: '' - }); - }); - } } - prerequisiteEvent(event: PrerequisiteEvent): void { - this.stage.hasChanged = true; - switch (event.type) { - case 'add': - if (!this.stage.prerequisites) { - this.stage.prerequisites = new Array(); - } - - let indexAdd = this.stage.prerequisites.findIndex(p => p.parameter === event.prerequisite.parameter); - if (indexAdd === -1) { - this.stage.prerequisites.push(cloneDeep(event.prerequisite)); - } - break; - case 'delete': - let indexDelete = this.stage.prerequisites.findIndex(p => p.parameter === event.prerequisite.parameter); - if (indexDelete > -1) { - this.stage.prerequisites.splice(indexDelete, 1); - } - break; - } + ngOnInit(): void { + this.pipelineService.getStageConditionsName(this.project.key, this.pipeline.name) + .pipe(finalize(() => this.loading = false)) + .subscribe((conditions) => this.triggerConditions = conditions); } } diff --git a/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.html b/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.html index 24dfc07a94..9201eda993 100644 --- a/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.html +++ b/ui/src/app/views/pipeline/show/workflow/stage/form/pipeline.stage.form.html @@ -3,27 +3,26 @@
- -
{{stage.name}}
+ +
{{stage.name}} +
- +
- - -
-
- - + + +
From ef0bbcb96441b2fc33ed8d2c5aa59c25f424be87 Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Fri, 14 Jun 2019 09:52:52 +0200 Subject: [PATCH 05/11] fix(vcs): do not return error if no created field is available (#4375) Signed-off-by: Benjamin Coenen --- engine/vcs/vcs_auth.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/engine/vcs/vcs_auth.go b/engine/vcs/vcs_auth.go index a5c2f5f15e..fff31e2dea 100644 --- a/engine/vcs/vcs_auth.go +++ b/engine/vcs/vcs_auth.go @@ -76,10 +76,7 @@ func getAccessTokens(ctx context.Context) (string, string, int64, bool) { return "", "", created, false } accessTokenCreated, ok := ctx.Value(contextKeyAccessTokenCreated).(string) - if !ok { - return "", "", created, false - } - if accessTokenCreated != "" { + if ok && accessTokenCreated != "" { var err error created, err = strconv.ParseInt(accessTokenCreated, 10, 64) if err != nil { From 4b0ed37f3cea6f1144d5d8c851c79d6e49fd91ba Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 14 Jun 2019 11:44:30 +0200 Subject: [PATCH 06/11] fix(api): No hatchery can spawn a worker msg (#4370) * fix(api): No hatchery can spawn a worker msg close #3318 Signed-off-by: Yvonnick Esnault --- engine/api/api_routes.go | 4 - engine/api/hatchery.go | 26 -- engine/api/services/hatchery.go | 56 ---- engine/api/workflow/execute_node_job_run.go | 22 -- engine/api/workflow/gorp_model.go | 8 - engine/api/workflow_queue.go | 69 ---- engine/hatchery/local/local.go | 2 +- sdk/cdsclient/client_hatchery.go | 19 -- sdk/cdsclient/client_queue.go | 8 - sdk/cdsclient/interface.go | 7 - sdk/hatchery/hatchery.go | 42 +-- sdk/hatchery/starter.go | 52 +-- sdk/workflow_run.go | 1 - sdk/workflow_run_easyjson.go | 342 +++++++++----------- 14 files changed, 164 insertions(+), 494 deletions(-) delete mode 100644 engine/api/hatchery.go delete mode 100644 engine/api/services/hatchery.go delete mode 100644 sdk/cdsclient/client_hatchery.go diff --git a/engine/api/api_routes.go b/engine/api/api_routes.go index 795525a4ae..e62fd5f560 100644 --- a/engine/api/api_routes.go +++ b/engine/api/api_routes.go @@ -91,9 +91,6 @@ func (api *API) InitRouter() { r.Handle("/group/{permGroupName}/token", r.GET(api.getGroupTokenListHandler), r.POST(api.generateTokenHandler)) r.Handle("/group/{permGroupName}/token/{tokenid}", r.DELETE(api.deleteTokenHandler)) - // Hatchery - r.Handle("/hatchery/count/{workflowNodeRunID}", r.GET(api.hatcheryCountHandler)) - // Hooks r.Handle("/hook/{uuid}/workflow/{workflowID}/vcsevent/{vcsServer}", r.GET(api.getHookPollingVCSEvents)) @@ -286,7 +283,6 @@ func (api *API) InitRouter() { r.Handle("/queue/workflows/count", r.GET(api.countWorkflowJobQueueHandler, EnableTracing(), MaintenanceAware())) r.Handle("/queue/workflows/{id}/take", r.POST(api.postTakeWorkflowJobHandler, NeedWorker(), EnableTracing(), MaintenanceAware())) r.Handle("/queue/workflows/{id}/book", r.POST(api.postBookWorkflowJobHandler, NeedHatchery(), EnableTracing(), MaintenanceAware()), r.DELETE(api.deleteBookWorkflowJobHandler, NeedHatchery(), EnableTracing(), MaintenanceAware())) - r.Handle("/queue/workflows/{id}/attempt", r.POST(api.postIncWorkflowJobAttemptHandler, NeedHatchery(), EnableTracing(), MaintenanceAware())) r.Handle("/queue/workflows/{id}/infos", r.GET(api.getWorkflowJobHandler, NeedWorker(), NeedHatchery(), EnableTracing(), MaintenanceAware())) r.Handle("/queue/workflows/{permID}/vulnerability", r.POSTEXECUTE(api.postVulnerabilityReportHandler, NeedWorker(), EnableTracing(), MaintenanceAware())) r.Handle("/queue/workflows/{id}/spawn/infos", r.POST(r.Asynchronous(api.postSpawnInfosWorkflowJobHandler, 1), NeedHatchery(), EnableTracing(), MaintenanceAware())) diff --git a/engine/api/hatchery.go b/engine/api/hatchery.go deleted file mode 100644 index cb27a7f48c..0000000000 --- a/engine/api/hatchery.go +++ /dev/null @@ -1,26 +0,0 @@ -package api - -import ( - "context" - "net/http" - - "github.com/ovh/cds/engine/api/services" - "github.com/ovh/cds/engine/service" - "github.com/ovh/cds/sdk" -) - -func (api *API) hatcheryCountHandler() service.Handler { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { - wfNodeRunID, err := requestVarInt(r, "workflowNodeRunID") - if err != nil { - return sdk.WrapError(err, "cannot convert workflow node run ID") - } - - count, err := services.CountHatcheries(api.mustDB(), wfNodeRunID) - if err != nil { - return sdk.WrapError(err, "cannot get hatcheries count") - } - - return service.WriteJSON(w, count, http.StatusOK) - } -} diff --git a/engine/api/services/hatchery.go b/engine/api/services/hatchery.go deleted file mode 100644 index bf7820c733..0000000000 --- a/engine/api/services/hatchery.go +++ /dev/null @@ -1,56 +0,0 @@ -package services - -import ( - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/group" -) - -// CountHatcheries retrieves in database the number of hatcheries -func CountHatcheries(db gorp.SqlExecutor, wfNodeRunID int64) (int64, error) { - query := ` - SELECT COUNT(1) - FROM services - WHERE ( - services.type = $1 - AND services.group_id = ANY( - SELECT DISTINCT(project_group.group_id) - FROM workflow_node_run - JOIN workflow_run ON workflow_run.id = workflow_node_run.workflow_run_id - JOIN workflow ON workflow.id = workflow_run.workflow_id - JOIN project ON project.id = workflow.project_id - JOIN project_group ON project_group.project_id = project.id - WHERE workflow_node_run.id = $2 - AND project_group.role >= 5 - ) - OR - services.group_id = $3 - ) - ` - return db.SelectInt(query, TypeHatchery, wfNodeRunID, group.SharedInfraGroup.ID) -} - -// LoadHatcheriesCountByNodeJobRunID retrieves in database the number of hatcheries given the node job run id -func LoadHatcheriesCountByNodeJobRunID(db gorp.SqlExecutor, wfNodeJobRunID int64) (int64, error) { - query := ` - SELECT COUNT(1) - FROM services - WHERE ( - services.type = $1 - AND services.group_id = ANY( - SELECT DISTINCT(project_group.group_id) - FROM workflow_node_run_job - JOIN workflow_node_run ON workflow_node_run.id = workflow_node_run_job.workflow_node_run_id - JOIN workflow_run ON workflow_run.id = workflow_node_run.workflow_run_id - JOIN workflow ON workflow.id = workflow_run.workflow_id - JOIN project ON project.id = workflow.project_id - JOIN project_group ON project_group.project_id = project.id - WHERE workflow_node_run.id = $2 - AND project_group.role >= 5 - ) - OR - services.group_id = $3 - ) - ` - return db.SelectInt(query, TypeHatchery, wfNodeJobRunID, group.SharedInfraGroup.ID) -} diff --git a/engine/api/workflow/execute_node_job_run.go b/engine/api/workflow/execute_node_job_run.go index 12b3e73f59..fd1031a2c4 100644 --- a/engine/api/workflow/execute_node_job_run.go +++ b/engine/api/workflow/execute_node_job_run.go @@ -3,7 +3,6 @@ package workflow import ( "bytes" "context" - "database/sql" "encoding/base64" "fmt" "sync" @@ -570,27 +569,6 @@ func FreeNodeJobRun(store cache.Store, id int64) error { return sdk.WrapError(sdk.ErrJobNotBooked, "BookNodeJobRun> job %d already released", id) } -//AddNodeJobAttempt add an hatchery attempt to spawn a job -func AddNodeJobAttempt(db gorp.SqlExecutor, id, hatcheryID int64) ([]int64, error) { - var ids []int64 - query := "UPDATE workflow_node_run_job SET spawn_attempts = array_append(spawn_attempts, $1) WHERE id = $2" - if _, err := db.Exec(query, hatcheryID, id); err != nil && err != sql.ErrNoRows { - return ids, sdk.WrapError(err, "cannot update node run job") - } - - rows, err := db.Query("SELECT DISTINCT unnest(spawn_attempts) FROM workflow_node_run_job WHERE id = $1", id) - var hID int64 - defer rows.Close() - for rows.Next() { - if errS := rows.Scan(&hID); errS != nil { - return ids, sdk.WrapError(errS, "AddNodeJobAttempt> cannot scan") - } - ids = append(ids, hID) - } - - return ids, err -} - //AddLog adds a build log func AddLog(db gorp.SqlExecutor, job *sdk.WorkflowNodeJobRun, logs *sdk.Log, maxLogSize int64) error { if job != nil { diff --git a/engine/api/workflow/gorp_model.go b/engine/api/workflow/gorp_model.go index e68187f3bd..dce4171ae9 100644 --- a/engine/api/workflow/gorp_model.go +++ b/engine/api/workflow/gorp_model.go @@ -4,8 +4,6 @@ import ( "database/sql" "time" - "github.com/lib/pq" - "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/ovh/cds/sdk" @@ -88,7 +86,6 @@ type JobRun struct { Parameters sql.NullString `db:"variables"` Status string `db:"status"` Retry int `db:"retry"` - SpawnAttempts *pq.Int64Array `db:"spawn_attempts"` Queued time.Time `db:"queued"` Start time.Time `db:"start"` Done time.Time `db:"done"` @@ -116,8 +113,6 @@ func (j *JobRun) ToJobRun(jr *sdk.WorkflowNodeJobRun) (err error) { } j.Status = jr.Status j.Retry = jr.Retry - array := pq.Int64Array(jr.SpawnAttempts) - j.SpawnAttempts = &array j.Queued = jr.Queued j.Start = jr.Start j.Done = jr.Done @@ -154,9 +149,6 @@ func (j JobRun) WorkflowNodeRunJob() (sdk.WorkflowNodeJobRun, error) { BookedBy: j.BookedBy, ContainsService: j.ContainsService, } - if j.SpawnAttempts != nil { - jr.SpawnAttempts = *j.SpawnAttempts - } if err := gorpmapping.JSONNullString(j.Job, &jr.Job); err != nil { return jr, sdk.WrapError(err, "column job") } diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 0206b7b9d8..8ba2006116 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -20,7 +20,6 @@ import ( "github.com/ovh/cds/engine/api/permission" "github.com/ovh/cds/engine/api/project" "github.com/ovh/cds/engine/api/repositoriesmanager" - "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/worker" "github.com/ovh/cds/engine/api/workermodel" "github.com/ovh/cds/engine/api/workflow" @@ -223,74 +222,6 @@ func (api *API) deleteBookWorkflowJobHandler() service.Handler { } } -func (api *API) postIncWorkflowJobAttemptHandler() service.Handler { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { - id, errc := requestVarInt(r, "id") - if errc != nil { - return sdk.WrapError(errc, "Invalid id") - } - h := getHatchery(ctx) - if h == nil { - return service.WriteJSON(w, nil, http.StatusUnauthorized) - } - spawnAttempts, err := workflow.AddNodeJobAttempt(api.mustDB(), id, h.ID) - if err != nil { - return err - } - - hCount, err := services.LoadHatcheriesCountByNodeJobRunID(api.mustDB(), id) - if err != nil { - return sdk.WrapError(err, "Cannot get hatcheries count") - } - - if int64(len(spawnAttempts)) >= hCount { - infos := []sdk.SpawnInfo{ - { - RemoteTime: time.Now(), - Message: sdk.SpawnMsg{ - ID: sdk.MsgSpawnInfoHatcheryCannotStartJob.ID, - Args: []interface{}{}, - }, - }, - } - - tx, errBegin := api.mustDB().Begin() - if errBegin != nil { - return sdk.WrapError(errBegin, "Cannot start transaction") - } - defer tx.Rollback() - - if err := workflow.AddSpawnInfosNodeJobRun(tx, id, infos); err != nil { - return sdk.WrapError(err, "Cannot save spawn info on node job run %d", id) - } - - wfNodeJobRun, errLj := workflow.LoadNodeJobRun(tx, api.Cache, id) - if errLj != nil { - return sdk.WrapError(errLj, "Cannot load node job run") - } - - wfNodeRun, errLr := workflow.LoadAndLockNodeRunByID(ctx, tx, wfNodeJobRun.WorkflowNodeRunID) - if errLr != nil { - return sdk.WrapError(errLr, "cannot load node run: %d", wfNodeJobRun.WorkflowNodeRunID) - } - - if found, err := workflow.SyncNodeRunRunJob(ctx, tx, wfNodeRun, *wfNodeJobRun); err != nil || !found { - return sdk.WrapError(err, "Cannot sync run job (found=%v)", found) - } - - if err := workflow.UpdateNodeRun(tx, wfNodeRun); err != nil { - return sdk.WrapError(err, "Cannot update node job run") - } - - if err := tx.Commit(); err != nil { - return sdk.WrapError(err, "Cannot commit tx") - } - } - - return service.WriteJSON(w, spawnAttempts, http.StatusOK) - } -} - func (api *API) getWorkflowJobHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { id, errc := requestVarInt(r, "id") diff --git a/engine/hatchery/local/local.go b/engine/hatchery/local/local.go index 7f1ed6a5f2..8f889773a2 100644 --- a/engine/hatchery/local/local.go +++ b/engine/hatchery/local/local.go @@ -196,7 +196,7 @@ func (h *HatcheryLocal) CanSpawn(model *sdk.Model, jobID int64, requirements []s for _, r := range requirements { ok, err := h.checkRequirement(r) if err != nil || !ok { - log.Debug("CanSpawn false hatchery.checkRequirement ok:%v err:%v", ok, err) + log.Debug("CanSpawn false hatchery.checkRequirement ok:%v err:%v r:%v", ok, err, r) return false } } diff --git a/sdk/cdsclient/client_hatchery.go b/sdk/cdsclient/client_hatchery.go deleted file mode 100644 index 703038b72c..0000000000 --- a/sdk/cdsclient/client_hatchery.go +++ /dev/null @@ -1,19 +0,0 @@ -package cdsclient - -import ( - "context" - "fmt" - - "github.com/ovh/cds/sdk" -) - -func (c *client) HatcheryCount(ctx context.Context, workflowNodeRunID int64) (int64, error) { - var hatcheriesCount int64 - code, err := c.GetJSON(ctx, fmt.Sprintf("/hatchery/count/%d", workflowNodeRunID), &hatcheriesCount) - if code > 300 && err == nil { - return hatcheriesCount, fmt.Errorf("HatcheryCount> HTTP %d", code) - } else if err != nil { - return hatcheriesCount, sdk.WrapError(err, "Error") - } - return hatcheriesCount, nil -} diff --git a/sdk/cdsclient/client_queue.go b/sdk/cdsclient/client_queue.go index f360828054..d8a761ed6f 100644 --- a/sdk/cdsclient/client_queue.go +++ b/sdk/cdsclient/client_queue.go @@ -235,14 +235,6 @@ func (c *client) QueueJobSendSpawnInfo(ctx context.Context, id int64, in []sdk.S return err } -// QueueJobIncAttempts add hatcheryID that cannot run this job and return the spawn attempts list -func (c *client) QueueJobIncAttempts(ctx context.Context, jobID int64) ([]int64, error) { - var spawnAttempts []int64 - path := fmt.Sprintf("/queue/workflows/%d/attempt", jobID) - _, err := c.PostJSON(ctx, path, nil, &spawnAttempts) - return spawnAttempts, err -} - // QueueJobBook books a job for a Hatchery func (c *client) QueueJobBook(ctx context.Context, id int64) error { path := fmt.Sprintf("/queue/workflows/%d/book", id) diff --git a/sdk/cdsclient/interface.go b/sdk/cdsclient/interface.go index 78e904fb8b..0776c391a6 100644 --- a/sdk/cdsclient/interface.go +++ b/sdk/cdsclient/interface.go @@ -169,11 +169,6 @@ type GroupClient interface { GroupRename(oldGroupname, newGroupname string) error } -// HatcheryClient exposes hatcheries related functions -type HatcheryClient interface { - HatcheryCount(ctx context.Context, wfNodeRunID int64) (int64, error) -} - // BroadcastClient expose all function for CDS Broadcasts type BroadcastClient interface { Broadcasts() ([]sdk.Broadcast, error) @@ -242,7 +237,6 @@ type QueueClient interface { QueueArtifactUpload(ctx context.Context, projectKey, integrationName string, nodeJobRunID int64, tag, filePath string) (bool, time.Duration, error) QueueStaticFilesUpload(ctx context.Context, projectKey, integrationName string, nodeJobRunID int64, name, entrypoint, staticKey string, tarContent io.Reader) (string, bool, time.Duration, error) QueueJobTag(ctx context.Context, jobID int64, tags []sdk.WorkflowRunTag) error - QueueJobIncAttempts(ctx context.Context, jobID int64) ([]int64, error) QueueServiceLogs(ctx context.Context, logs []sdk.ServiceLog) error } @@ -346,7 +340,6 @@ type Interface interface { ExportImportInterface GroupClient GRPCPluginsClient - HatcheryClient BroadcastClient MaintenanceClient PipelineClient diff --git a/sdk/hatchery/hatchery.go b/sdk/hatchery/hatchery.go index 69d888ed53..267d49aeeb 100644 --- a/sdk/hatchery/hatchery.go +++ b/sdk/hatchery/hatchery.go @@ -111,46 +111,12 @@ func Create(ctx context.Context, h Interface) error { ) // run the starters pool - workersStartChan, workerStartResultChan := startWorkerStarters(ctx, h) + workersStartChan := startWorkerStarters(ctx, h) hostname, errh := os.Hostname() if errh != nil { return fmt.Errorf("Create> Cannot retrieve hostname: %s", errh) } - // read the result channel in another goroutine to let the main goroutine start new workers - sdk.GoRoutine(ctx, "checkStarterResult", func(ctx context.Context) { - for startWorkerRes := range workerStartResultChan { - if startWorkerRes.err != nil { - errs <- startWorkerRes.err - } - if startWorkerRes.temptToSpawn { - found := false - for _, hID := range startWorkerRes.request.spawnAttempts { - if hID == h.ID() { - found = true - break - } - } - if !found { - ctxHatcheryCount, cancelHatcheryCount := context.WithTimeout(ctx, 10*time.Second) - if hCount, err := h.CDSClient().HatcheryCount(ctxHatcheryCount, startWorkerRes.request.workflowNodeRunID); err == nil { - if int64(len(startWorkerRes.request.spawnAttempts)) < hCount { - ctxtJobIncAttemps, cancelJobIncAttemps := context.WithTimeout(ctx, 10*time.Second) - if _, errQ := h.CDSClient().QueueJobIncAttempts(ctxtJobIncAttemps, startWorkerRes.request.id); errQ != nil { - log.Warning("Hatchery> Create> cannot inc spawn attempts %v", errQ) - } - cancelJobIncAttemps() - } - } else { - log.Warning("Hatchery> Create> cannot get hatchery count: %v", err) - } - cancelHatcheryCount() - } - } - } - }, - PanicDump(h), - ) // read the errs channel in another goroutine too sdk.GoRoutine(ctx, "checkErrs", func(ctx context.Context) { @@ -257,7 +223,6 @@ func Create(ctx context.Context, h Interface) error { requirements: j.Job.Action.Requirements, hostname: hostname, timestamp: time.Now().Unix(), - spawnAttempts: j.SpawnAttempts, workflowNodeRunID: j.WorkflowNodeRunID, } @@ -272,11 +237,6 @@ func Create(ctx context.Context, h Interface) error { // No model has been found, let's send a failing result if chosenModel == nil { - workerStartResultChan <- workerStarterResult{ - request: workerRequest, - isRun: false, - temptToSpawn: true, - } log.Debug("hatchery> no model") endTrace("no model") continue diff --git a/sdk/hatchery/starter.go b/sdk/hatchery/starter.go index 49e280de75..7cff0830c1 100644 --- a/sdk/hatchery/starter.go +++ b/sdk/hatchery/starter.go @@ -25,18 +25,10 @@ type workerStarterRequest struct { requirements []sdk.Requirement hostname string timestamp int64 - spawnAttempts []int64 workflowNodeRunID int64 registerWorkerModel *sdk.Model } -type workerStarterResult struct { - request workerStarterRequest - isRun bool - temptToSpawn bool - err error -} - func PanicDump(h Interface) func(s string) (io.WriteCloser, error) { return func(s string) (io.WriteCloser, error) { dir, err := h.PanicDumpDirectory() @@ -49,9 +41,8 @@ func PanicDump(h Interface) func(s string) (io.WriteCloser, error) { // Start all goroutines which manage the hatchery worker spawning routine. // the purpose is to avoid go routines leak when there is a bunch of worker to start -func startWorkerStarters(ctx context.Context, h Interface) (chan<- workerStarterRequest, chan workerStarterResult) { +func startWorkerStarters(ctx context.Context, h Interface) chan<- workerStarterRequest { jobs := make(chan workerStarterRequest, 1) - results := make(chan workerStarterResult, 1) maxProv := h.Configuration().Provision.MaxConcurrentProvisioning if maxProv < 1 { @@ -59,37 +50,18 @@ func startWorkerStarters(ctx context.Context, h Interface) (chan<- workerStarter } for workerNum := 0; workerNum < maxProv; workerNum++ { sdk.GoRoutine(ctx, "workerStarter", func(ctx context.Context) { - workerStarter(ctx, h, fmt.Sprintf("%d", workerNum), jobs, results) + workerStarter(ctx, h, fmt.Sprintf("%d", workerNum), jobs) }, PanicDump(h)) } - return jobs, results + return jobs } -func workerStarter(ctx context.Context, h Interface, workerNum string, jobs <-chan workerStarterRequest, results chan<- workerStarterResult) { +func workerStarter(ctx context.Context, h Interface, workerNum string, jobs <-chan workerStarterRequest) { for j := range jobs { // Start a worker for a job if m := j.registerWorkerModel; m == nil { - ctx2, end := observability.Span(j.ctx, "hatchery.workerStarter") - isRun, err := spawnWorkerForJob(h, j) - //Check the result - res := workerStarterResult{ - request: j, - err: err, - isRun: isRun, - temptToSpawn: true, - } - - _, cend := observability.Span(ctx2, "sendResult") - //Send the result back - results <- res - cend() - - if err != nil { - j.cancel(err.Error()) - } else { - j.cancel("") - } - end() + _ = spawnWorkerForJob(h, j) + j.cancel("") } else { // Start a worker for registering log.Debug("Spawning worker for register model %s", m.Name) if atomic.LoadInt64(&nbWorkerToStart) > int64(h.Configuration().Provision.MaxConcurrentProvisioning) { @@ -114,7 +86,7 @@ func workerStarter(ctx context.Context, h Interface, workerNum string, jobs <-ch } } -func spawnWorkerForJob(h Interface, j workerStarterRequest) (bool, error) { +func spawnWorkerForJob(h Interface, j workerStarterRequest) bool { ctx, end := observability.Span(j.ctx, "hatchery.spawnWorkerForJob") defer end() @@ -129,7 +101,7 @@ func spawnWorkerForJob(h Interface, j workerStarterRequest) (bool, error) { } if atomic.LoadInt64(&nbWorkerToStart) >= int64(maxProv) { log.Debug("hatchery> spawnWorkerForJob> max concurrent provisioning reached") - return false, nil + return false } atomic.AddInt64(&nbWorkerToStart, 1) @@ -139,7 +111,7 @@ func spawnWorkerForJob(h Interface, j workerStarterRequest) (bool, error) { if h.CDSClient().GetService() == nil || h.ID() == 0 { log.Warning("hatchery> spawnWorkerForJob> %d - job %d %s- hatchery not registered - srv:%t id:%d", j.timestamp, j.id, j.model.Name, h.CDSClient().GetService() == nil, h.ID()) - return false, nil + return false } ctxQueueJobBook, next := observability.Span(ctx, "hatchery.QueueJobBook") @@ -149,7 +121,7 @@ func spawnWorkerForJob(h Interface, j workerStarterRequest) (bool, error) { // perhaps already booked by another hatchery log.Info("hatchery> spawnWorkerForJob> %d - cannot book job %d %s: %s", j.timestamp, j.id, j.model.Name, err) cancel() - return false, nil + return false } next() cancel() @@ -179,7 +151,7 @@ func spawnWorkerForJob(h Interface, j workerStarterRequest) (bool, error) { }) log.Error("hatchery %s cannot spawn worker %s for job %d: %v", h.Service().Name, j.model.Name, j.id, errSpawn) next() - return false, nil + return false } ctxSendSpawnInfo, next = observability.Span(ctx, "hatchery.SendSpawnInfo", observability.Tag("msg", sdk.MsgSpawnInfoHatcheryStartsSuccessfully.ID)) @@ -201,5 +173,5 @@ func spawnWorkerForJob(h Interface, j workerStarterRequest) (bool, error) { }) next() } - return true, nil // ok for this job + return true // ok for this job } diff --git a/sdk/workflow_run.go b/sdk/workflow_run.go index d275fc3a7f..74e4957824 100644 --- a/sdk/workflow_run.go +++ b/sdk/workflow_run.go @@ -358,7 +358,6 @@ type WorkflowNodeJobRun struct { Parameters []Parameter `json:"parameters,omitempty"` Status string `json:"status"` Retry int `json:"retry"` - SpawnAttempts []int64 `json:"spawn_attempts,omitempty"` Queued time.Time `json:"queued,omitempty"` QueuedSeconds int64 `json:"queued_seconds,omitempty"` Start time.Time `json:"start,omitempty"` diff --git a/sdk/workflow_run_easyjson.go b/sdk/workflow_run_easyjson.go index e039b9aea1..2015e9d227 100644 --- a/sdk/workflow_run_easyjson.go +++ b/sdk/workflow_run_easyjson.go @@ -71,29 +71,6 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk(in *jlexer.Lexer, out *WorkflowNod out.Status = string(in.String()) case "retry": out.Retry = int(in.Int()) - case "spawn_attempts": - if in.IsNull() { - in.Skip() - out.SpawnAttempts = nil - } else { - in.Delim('[') - if out.SpawnAttempts == nil { - if !in.IsDelim(']') { - out.SpawnAttempts = make([]int64, 0, 8) - } else { - out.SpawnAttempts = []int64{} - } - } else { - out.SpawnAttempts = (out.SpawnAttempts)[:0] - } - for !in.IsDelim(']') { - var v2 int64 - v2 = int64(in.Int64()) - out.SpawnAttempts = append(out.SpawnAttempts, v2) - in.WantComma() - } - in.Delim(']') - } case "queued": if data := in.Raw(); in.Ok() { in.AddError((out.Queued).UnmarshalJSON(data)) @@ -130,9 +107,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk(in *jlexer.Lexer, out *WorkflowNod out.SpawnInfos = (out.SpawnInfos)[:0] } for !in.IsDelim(']') { - var v3 SpawnInfo - easyjsonD7860c2dDecodeGithubComOvhCdsSdk4(in, &v3) - out.SpawnInfos = append(out.SpawnInfos, v3) + var v2 SpawnInfo + easyjsonD7860c2dDecodeGithubComOvhCdsSdk4(in, &v2) + out.SpawnInfos = append(out.SpawnInfos, v2) in.WantComma() } in.Delim(']') @@ -153,9 +130,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk(in *jlexer.Lexer, out *WorkflowNod out.ExecGroups = (out.ExecGroups)[:0] } for !in.IsDelim(']') { - var v4 Group - easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in, &v4) - out.ExecGroups = append(out.ExecGroups, v4) + var v3 Group + easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in, &v3) + out.ExecGroups = append(out.ExecGroups, v3) in.WantComma() } in.Delim(']') @@ -176,9 +153,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk(in *jlexer.Lexer, out *WorkflowNod out.IntegrationPluginBinaries = (out.IntegrationPluginBinaries)[:0] } for !in.IsDelim(']') { - var v5 GRPCPluginBinary - easyjsonD7860c2dDecodeGithubComOvhCdsSdk6(in, &v5) - out.IntegrationPluginBinaries = append(out.IntegrationPluginBinaries, v5) + var v4 GRPCPluginBinary + easyjsonD7860c2dDecodeGithubComOvhCdsSdk6(in, &v4) + out.IntegrationPluginBinaries = append(out.IntegrationPluginBinaries, v4) in.WantComma() } in.Delim(']') @@ -196,9 +173,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk(in *jlexer.Lexer, out *WorkflowNod for !in.IsDelim('}') { key := string(in.String()) in.WantColon() - var v6 string - v6 = string(in.String()) - (out.Header)[key] = v6 + var v5 string + v5 = string(in.String()) + (out.Header)[key] = v5 in.WantComma() } in.Delim('}') @@ -269,11 +246,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo } { out.RawByte('[') - for v7, v8 := range in.Parameters { - if v7 > 0 { + for v6, v7 := range in.Parameters { + if v6 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk2(out, v8) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk2(out, v7) } out.RawByte(']') } @@ -298,25 +275,6 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo } out.Int(int(in.Retry)) } - if len(in.SpawnAttempts) != 0 { - const prefix string = ",\"spawn_attempts\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - { - out.RawByte('[') - for v9, v10 := range in.SpawnAttempts { - if v9 > 0 { - out.RawByte(',') - } - out.Int64(int64(v10)) - } - out.RawByte(']') - } - } if true { const prefix string = ",\"queued\":" if first { @@ -377,7 +335,7 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo } out.String(string(in.ModelType)) } - { + if true { const prefix string = ",\"bookedby\":" if first { first = false @@ -399,11 +357,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo out.RawString("null") } else { out.RawByte('[') - for v11, v12 := range in.SpawnInfos { - if v11 > 0 { + for v8, v9 := range in.SpawnInfos { + if v8 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk4(out, v12) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk4(out, v9) } out.RawByte(']') } @@ -420,11 +378,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo out.RawString("null") } else { out.RawByte('[') - for v13, v14 := range in.ExecGroups { - if v13 > 0 { + for v10, v11 := range in.ExecGroups { + if v10 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out, v14) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out, v11) } out.RawByte(']') } @@ -439,11 +397,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo } { out.RawByte('[') - for v15, v16 := range in.IntegrationPluginBinaries { - if v15 > 0 { + for v12, v13 := range in.IntegrationPluginBinaries { + if v12 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk6(out, v16) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk6(out, v13) } out.RawByte(']') } @@ -458,16 +416,16 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk(out *jwriter.Writer, in WorkflowNo } { out.RawByte('{') - v17First := true - for v17Name, v17Value := range in.Header { - if v17First { - v17First = false + v14First := true + for v14Name, v14Value := range in.Header { + if v14First { + v14First = false } else { out.RawByte(',') } - out.String(string(v17Name)) + out.String(string(v14Name)) out.RawByte(':') - out.String(string(v17Value)) + out.String(string(v14Value)) } out.RawByte('}') } @@ -561,9 +519,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk6(in *jlexer.Lexer, out *GRPCPlugin out.Entrypoints = (out.Entrypoints)[:0] } for !in.IsDelim(']') { - var v18 string - v18 = string(in.String()) - out.Entrypoints = append(out.Entrypoints, v18) + var v15 string + v15 = string(in.String()) + out.Entrypoints = append(out.Entrypoints, v15) in.WantComma() } in.Delim(']') @@ -586,9 +544,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk6(in *jlexer.Lexer, out *GRPCPlugin out.Args = (out.Args)[:0] } for !in.IsDelim(']') { - var v19 string - v19 = string(in.String()) - out.Args = append(out.Args, v19) + var v16 string + v16 = string(in.String()) + out.Args = append(out.Args, v16) in.WantComma() } in.Delim(']') @@ -609,11 +567,11 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk6(in *jlexer.Lexer, out *GRPCPlugin out.Requirements = (out.Requirements)[:0] } for !in.IsDelim(']') { - var v20 Requirement + var v17 Requirement if data := in.Raw(); in.Ok() { - in.AddError((v20).UnmarshalJSON(data)) + in.AddError((v17).UnmarshalJSON(data)) } - out.Requirements = append(out.Requirements, v20) + out.Requirements = append(out.Requirements, v17) in.WantComma() } in.Delim(']') @@ -741,11 +699,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk6(out *jwriter.Writer, in GRPCPlugi } { out.RawByte('[') - for v22, v23 := range in.Entrypoints { - if v22 > 0 { + for v19, v20 := range in.Entrypoints { + if v19 > 0 { out.RawByte(',') } - out.String(string(v23)) + out.String(string(v20)) } out.RawByte(']') } @@ -770,11 +728,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk6(out *jwriter.Writer, in GRPCPlugi } { out.RawByte('[') - for v24, v25 := range in.Args { - if v24 > 0 { + for v21, v22 := range in.Args { + if v21 > 0 { out.RawByte(',') } - out.String(string(v25)) + out.String(string(v22)) } out.RawByte(']') } @@ -789,11 +747,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk6(out *jwriter.Writer, in GRPCPlugi } { out.RawByte('[') - for v26, v27 := range in.Requirements { - if v26 > 0 { + for v23, v24 := range in.Requirements { + if v23 > 0 { out.RawByte(',') } - out.Raw((v27).MarshalJSON()) + out.Raw((v24).MarshalJSON()) } out.RawByte(']') } @@ -859,9 +817,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in *jlexer.Lexer, out *Group) { out.Admins = (out.Admins)[:0] } for !in.IsDelim(']') { - var v30 User - easyjsonD7860c2dDecodeGithubComOvhCdsSdk7(in, &v30) - out.Admins = append(out.Admins, v30) + var v27 User + easyjsonD7860c2dDecodeGithubComOvhCdsSdk7(in, &v27) + out.Admins = append(out.Admins, v27) in.WantComma() } in.Delim(']') @@ -882,9 +840,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in *jlexer.Lexer, out *Group) { out.Users = (out.Users)[:0] } for !in.IsDelim(']') { - var v31 User - easyjsonD7860c2dDecodeGithubComOvhCdsSdk7(in, &v31) - out.Users = append(out.Users, v31) + var v28 User + easyjsonD7860c2dDecodeGithubComOvhCdsSdk7(in, &v28) + out.Users = append(out.Users, v28) in.WantComma() } in.Delim(']') @@ -905,9 +863,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in *jlexer.Lexer, out *Group) { out.Tokens = (out.Tokens)[:0] } for !in.IsDelim(']') { - var v32 Token - easyjsonD7860c2dDecodeGithubComOvhCdsSdk8(in, &v32) - out.Tokens = append(out.Tokens, v32) + var v29 Token + easyjsonD7860c2dDecodeGithubComOvhCdsSdk8(in, &v29) + out.Tokens = append(out.Tokens, v29) in.WantComma() } in.Delim(']') @@ -956,11 +914,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out *jwriter.Writer, in Group) { } { out.RawByte('[') - for v33, v34 := range in.Admins { - if v33 > 0 { + for v30, v31 := range in.Admins { + if v30 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk7(out, v34) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk7(out, v31) } out.RawByte(']') } @@ -975,11 +933,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out *jwriter.Writer, in Group) { } { out.RawByte('[') - for v35, v36 := range in.Users { - if v35 > 0 { + for v32, v33 := range in.Users { + if v32 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk7(out, v36) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk7(out, v33) } out.RawByte(']') } @@ -994,11 +952,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out *jwriter.Writer, in Group) { } { out.RawByte('[') - for v37, v38 := range in.Tokens { - if v37 > 0 { + for v34, v35 := range in.Tokens { + if v34 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk8(out, v38) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk8(out, v35) } out.RawByte(']') } @@ -1183,9 +1141,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk7(in *jlexer.Lexer, out *User) { out.Groups = (out.Groups)[:0] } for !in.IsDelim(']') { - var v39 Group - easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in, &v39) - out.Groups = append(out.Groups, v39) + var v36 Group + easyjsonD7860c2dDecodeGithubComOvhCdsSdk5(in, &v36) + out.Groups = append(out.Groups, v36) in.WantComma() } in.Delim(']') @@ -1208,9 +1166,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk7(in *jlexer.Lexer, out *User) { out.Favorites = (out.Favorites)[:0] } for !in.IsDelim(']') { - var v40 Favorite - easyjsonD7860c2dDecodeGithubComOvhCdsSdk9(in, &v40) - out.Favorites = append(out.Favorites, v40) + var v37 Favorite + easyjsonD7860c2dDecodeGithubComOvhCdsSdk9(in, &v37) + out.Favorites = append(out.Favorites, v37) in.WantComma() } in.Delim(']') @@ -1293,11 +1251,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk7(out *jwriter.Writer, in User) { } { out.RawByte('[') - for v41, v42 := range in.Groups { - if v41 > 0 { + for v38, v39 := range in.Groups { + if v38 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out, v42) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk5(out, v39) } out.RawByte(']') } @@ -1324,11 +1282,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk7(out *jwriter.Writer, in User) { out.RawString("null") } else { out.RawByte('[') - for v43, v44 := range in.Favorites { - if v43 > 0 { + for v40, v41 := range in.Favorites { + if v40 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk9(out, v44) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk9(out, v41) } out.RawByte(']') } @@ -1380,9 +1338,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk9(in *jlexer.Lexer, out *Favorite) out.ProjectIDs = (out.ProjectIDs)[:0] } for !in.IsDelim(']') { - var v45 int64 - v45 = int64(in.Int64()) - out.ProjectIDs = append(out.ProjectIDs, v45) + var v42 int64 + v42 = int64(in.Int64()) + out.ProjectIDs = append(out.ProjectIDs, v42) in.WantComma() } in.Delim(']') @@ -1403,9 +1361,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk9(in *jlexer.Lexer, out *Favorite) out.WorkflowIDs = (out.WorkflowIDs)[:0] } for !in.IsDelim(']') { - var v46 int64 - v46 = int64(in.Int64()) - out.WorkflowIDs = append(out.WorkflowIDs, v46) + var v43 int64 + v43 = int64(in.Int64()) + out.WorkflowIDs = append(out.WorkflowIDs, v43) in.WantComma() } in.Delim(']') @@ -1436,11 +1394,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk9(out *jwriter.Writer, in Favorite) out.RawString("null") } else { out.RawByte('[') - for v47, v48 := range in.ProjectIDs { - if v47 > 0 { + for v44, v45 := range in.ProjectIDs { + if v44 > 0 { out.RawByte(',') } - out.Int64(int64(v48)) + out.Int64(int64(v45)) } out.RawByte(']') } @@ -1457,11 +1415,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk9(out *jwriter.Writer, in Favorite) out.RawString("null") } else { out.RawByte('[') - for v49, v50 := range in.WorkflowIDs { - if v49 > 0 { + for v46, v47 := range in.WorkflowIDs { + if v46 > 0 { out.RawByte(',') } - out.Int64(int64(v50)) + out.Int64(int64(v47)) } out.RawByte(']') } @@ -1592,15 +1550,15 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk10(in *jlexer.Lexer, out *SpawnMsg) out.Args = (out.Args)[:0] } for !in.IsDelim(']') { - var v51 interface{} - if m, ok := v51.(easyjson.Unmarshaler); ok { + var v48 interface{} + if m, ok := v48.(easyjson.Unmarshaler); ok { m.UnmarshalEasyJSON(in) - } else if m, ok := v51.(json.Unmarshaler); ok { + } else if m, ok := v48.(json.Unmarshaler); ok { _ = m.UnmarshalJSON(in.Raw()) } else { - v51 = in.Interface() + v48 = in.Interface() } - out.Args = append(out.Args, v51) + out.Args = append(out.Args, v48) in.WantComma() } in.Delim(']') @@ -1641,16 +1599,16 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk10(out *jwriter.Writer, in SpawnMsg out.RawString("null") } else { out.RawByte('[') - for v52, v53 := range in.Args { - if v52 > 0 { + for v49, v50 := range in.Args { + if v49 > 0 { out.RawByte(',') } - if m, ok := v53.(easyjson.Marshaler); ok { + if m, ok := v50.(easyjson.Marshaler); ok { m.MarshalEasyJSON(out) - } else if m, ok := v53.(json.Marshaler); ok { + } else if m, ok := v50.(json.Marshaler); ok { out.Raw(m.MarshalJSON()) } else { - out.Raw(json.Marshal(v53)) + out.Raw(json.Marshal(v50)) } } out.RawByte(']') @@ -1728,15 +1686,15 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk3(in *jlexer.Lexer, out *Service) { for !in.IsDelim('}') { key := string(in.String()) in.WantColon() - var v54 interface{} - if m, ok := v54.(easyjson.Unmarshaler); ok { + var v51 interface{} + if m, ok := v51.(easyjson.Unmarshaler); ok { m.UnmarshalEasyJSON(in) - } else if m, ok := v54.(json.Unmarshaler); ok { + } else if m, ok := v51.(json.Unmarshaler); ok { _ = m.UnmarshalJSON(in.Raw()) } else { - v54 = in.Interface() + v51 = in.Interface() } - (out.Config)[key] = v54 + (out.Config)[key] = v51 in.WantComma() } in.Delim('}') @@ -1881,21 +1839,21 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk3(out *jwriter.Writer, in Service) out.RawString(`null`) } else { out.RawByte('{') - v55First := true - for v55Name, v55Value := range in.Config { - if v55First { - v55First = false + v52First := true + for v52Name, v52Value := range in.Config { + if v52First { + v52First = false } else { out.RawByte(',') } - out.String(string(v55Name)) + out.String(string(v52Name)) out.RawByte(':') - if m, ok := v55Value.(easyjson.Marshaler); ok { + if m, ok := v52Value.(easyjson.Marshaler); ok { m.MarshalEasyJSON(out) - } else if m, ok := v55Value.(json.Marshaler); ok { + } else if m, ok := v52Value.(json.Marshaler); ok { out.Raw(m.MarshalJSON()) } else { - out.Raw(json.Marshal(v55Value)) + out.Raw(json.Marshal(v52Value)) } } out.RawByte('}') @@ -1972,9 +1930,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk11(in *jlexer.Lexer, out *Monitorin out.Lines = (out.Lines)[:0] } for !in.IsDelim(']') { - var v56 MonitoringStatusLine - easyjsonD7860c2dDecodeGithubComOvhCdsSdk12(in, &v56) - out.Lines = append(out.Lines, v56) + var v53 MonitoringStatusLine + easyjsonD7860c2dDecodeGithubComOvhCdsSdk12(in, &v53) + out.Lines = append(out.Lines, v53) in.WantComma() } in.Delim(']') @@ -2015,11 +1973,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk11(out *jwriter.Writer, in Monitori out.RawString("null") } else { out.RawByte('[') - for v57, v58 := range in.Lines { - if v57 > 0 { + for v54, v55 := range in.Lines { + if v54 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk12(out, v58) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk12(out, v55) } out.RawByte(']') } @@ -2251,9 +2209,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk1(in *jlexer.Lexer, out *ExecutedJo out.StepStatus = (out.StepStatus)[:0] } for !in.IsDelim(']') { - var v59 StepStatus - easyjsonD7860c2dDecodeGithubComOvhCdsSdk13(in, &v59) - out.StepStatus = append(out.StepStatus, v59) + var v56 StepStatus + easyjsonD7860c2dDecodeGithubComOvhCdsSdk13(in, &v56) + out.StepStatus = append(out.StepStatus, v56) in.WantComma() } in.Delim(']') @@ -2290,9 +2248,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk1(in *jlexer.Lexer, out *ExecutedJo out.Warnings = (out.Warnings)[:0] } for !in.IsDelim(']') { - var v60 PipelineBuildWarning - easyjsonD7860c2dDecodeGithubComOvhCdsSdk15(in, &v60) - out.Warnings = append(out.Warnings, v60) + var v57 PipelineBuildWarning + easyjsonD7860c2dDecodeGithubComOvhCdsSdk15(in, &v57) + out.Warnings = append(out.Warnings, v57) in.WantComma() } in.Delim(']') @@ -2323,11 +2281,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk1(out *jwriter.Writer, in ExecutedJ out.RawString("null") } else { out.RawByte('[') - for v61, v62 := range in.StepStatus { - if v61 > 0 { + for v58, v59 := range in.StepStatus { + if v58 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk13(out, v62) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk13(out, v59) } out.RawByte(']') } @@ -2424,11 +2382,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk1(out *jwriter.Writer, in ExecutedJ out.RawString("null") } else { out.RawByte('[') - for v63, v64 := range in.Warnings { - if v63 > 0 { + for v60, v61 := range in.Warnings { + if v60 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk15(out, v64) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk15(out, v61) } out.RawByte(']') } @@ -2557,11 +2515,11 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk14(in *jlexer.Lexer, out *Action) { out.Requirements = (out.Requirements)[:0] } for !in.IsDelim(']') { - var v65 Requirement + var v62 Requirement if data := in.Raw(); in.Ok() { - in.AddError((v65).UnmarshalJSON(data)) + in.AddError((v62).UnmarshalJSON(data)) } - out.Requirements = append(out.Requirements, v65) + out.Requirements = append(out.Requirements, v62) in.WantComma() } in.Delim(']') @@ -2582,9 +2540,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk14(in *jlexer.Lexer, out *Action) { out.Parameters = (out.Parameters)[:0] } for !in.IsDelim(']') { - var v66 Parameter - easyjsonD7860c2dDecodeGithubComOvhCdsSdk2(in, &v66) - out.Parameters = append(out.Parameters, v66) + var v63 Parameter + easyjsonD7860c2dDecodeGithubComOvhCdsSdk2(in, &v63) + out.Parameters = append(out.Parameters, v63) in.WantComma() } in.Delim(']') @@ -2605,9 +2563,9 @@ func easyjsonD7860c2dDecodeGithubComOvhCdsSdk14(in *jlexer.Lexer, out *Action) { out.Actions = (out.Actions)[:0] } for !in.IsDelim(']') { - var v67 Action - easyjsonD7860c2dDecodeGithubComOvhCdsSdk14(in, &v67) - out.Actions = append(out.Actions, v67) + var v64 Action + easyjsonD7860c2dDecodeGithubComOvhCdsSdk14(in, &v64) + out.Actions = append(out.Actions, v64) in.WantComma() } in.Delim(']') @@ -2770,11 +2728,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk14(out *jwriter.Writer, in Action) out.RawString("null") } else { out.RawByte('[') - for v68, v69 := range in.Requirements { - if v68 > 0 { + for v65, v66 := range in.Requirements { + if v65 > 0 { out.RawByte(',') } - out.Raw((v69).MarshalJSON()) + out.Raw((v66).MarshalJSON()) } out.RawByte(']') } @@ -2791,11 +2749,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk14(out *jwriter.Writer, in Action) out.RawString("null") } else { out.RawByte('[') - for v70, v71 := range in.Parameters { - if v70 > 0 { + for v67, v68 := range in.Parameters { + if v67 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk2(out, v71) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk2(out, v68) } out.RawByte(']') } @@ -2810,11 +2768,11 @@ func easyjsonD7860c2dEncodeGithubComOvhCdsSdk14(out *jwriter.Writer, in Action) } { out.RawByte('[') - for v72, v73 := range in.Actions { - if v72 > 0 { + for v69, v70 := range in.Actions { + if v69 > 0 { out.RawByte(',') } - easyjsonD7860c2dEncodeGithubComOvhCdsSdk14(out, v73) + easyjsonD7860c2dEncodeGithubComOvhCdsSdk14(out, v70) } out.RawByte(']') } From 006e783c470a342aea1a9d0468cd6b7fe58b2c01 Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Fri, 14 Jun 2019 14:49:56 +0200 Subject: [PATCH 07/11] refactor(hatchery): get only queue for the right modeltype of hatchery (#4373) * refactor(hatchery): get only queue for the right modeltype of hatchery Signed-off-by: Benjamin Coenen --- .gitignore | 1 + sdk/cdsclient/client_queue.go | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index db48a0127b..83356b6cdb 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ debug.test myOpenstackIntegration.yml myAWSS3Integration.yml test_results.yml +test_results.xml *.dump .tmp contrib/**/dist diff --git a/sdk/cdsclient/client_queue.go b/sdk/cdsclient/client_queue.go index d8a761ed6f..4466d9eaea 100644 --- a/sdk/cdsclient/client_queue.go +++ b/sdk/cdsclient/client_queue.go @@ -120,19 +120,18 @@ func (c *client) QueuePolling(ctx context.Context, jobs chan<- sdk.WorkflowNodeJ continue } - reqMods := []RequestModifier{} + urlValues := url.Values{} if ratioService != nil { - reqMods = append(reqMods, SetHeader("ratioService", strconv.Itoa(*ratioService))) + urlValues.Set("ratioService", strconv.Itoa(*ratioService)) } if modelType != "" { - reqMods = append(reqMods, SetHeader("modelType", modelType)) + urlValues.Set("modelType", modelType) } ctxt, cancel := context.WithTimeout(ctx, 10*time.Second) - queue := sdk.WorkflowQueue{} - if _, err := c.GetJSON(ctxt, "/queue/workflows", &queue, reqMods...); err != nil { + if _, err := c.GetJSON(ctxt, "/queue/workflows?"+urlValues.Encode(), &queue, nil); err != nil { errs <- sdk.WrapError(err, "Unable to load jobs") cancel() continue From 434410fe580b69e9e8a96a429031b19d8196ebff Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Fri, 14 Jun 2019 14:51:11 +0200 Subject: [PATCH 08/11] fix(hatchery/local): check requirement os/arch (#4376) This is must have if: - osDefault != osHatchery - provision 0 on hatchery local Signed-off-by: Yvonnick Esnault --- engine/hatchery/local/local.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/engine/hatchery/local/local.go b/engine/hatchery/local/local.go index 8f889773a2..d0549da1d5 100644 --- a/engine/hatchery/local/local.go +++ b/engine/hatchery/local/local.go @@ -460,6 +460,12 @@ func (h *HatcheryLocal) checkRequirement(r sdk.Requirement) (bool, error) { return true, nil case sdk.PluginRequirement: return true, nil + case sdk.OSArchRequirement: + osarch := strings.Split(r.Value, "/") + if len(osarch) != 2 { + return false, fmt.Errorf("invalid requirement %s", r.Value) + } + return osarch[0] == strings.ToLower(sdk.GOOS) && osarch[1] == strings.ToLower(sdk.GOARCH), nil default: return false, nil } From 932fcd30fe28023244418260936dcc26cb69c053 Mon Sep 17 00:00:00 2001 From: Coenen Benjamin Date: Mon, 17 Jun 2019 09:31:23 +0200 Subject: [PATCH 09/11] fix(ui): create empty array when add first stage conditions (#4381) Signed-off-by: Benjamin Coenen --- ui/src/app/shared/conditions/conditions.component.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/src/app/shared/conditions/conditions.component.ts b/ui/src/app/shared/conditions/conditions.component.ts index ca584bac87..779b57809b 100644 --- a/ui/src/app/shared/conditions/conditions.component.ts +++ b/ui/src/app/shared/conditions/conditions.component.ts @@ -117,7 +117,13 @@ export class ConditionsComponent extends Table implements addEmptyCondition(): void { let emptyCond = new WorkflowNodeCondition(); emptyCond.operator = 'eq'; - this.conditions.plain.push(emptyCond); + + if (!this.conditions.plain) { + this.conditions.plain = [emptyCond]; + } else { + this.conditions.plain.push(emptyCond); + } + this.conditionsChange.emit(this.conditions); } pushChange(event: string, e?: string): void { From 974762d324e77653c13a0e8ea290c7241f05dbdf Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 17 Jun 2019 14:42:55 +0200 Subject: [PATCH 10/11] fix(cdsctl): linux amd64 nokeychain support (#4380) * fix(cdsctl): linux amd64 nokeychain support Signed-off-by: Yvonnick Esnault --- cli/cdsctl/Makefile | 8 ++++---- cli/cdsctl/configstore.go | 2 +- cli/cdsctl/configstore_linux_amd64.go | 2 ++ engine/api/pipeline_parameter.go | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cli/cdsctl/Makefile b/cli/cdsctl/Makefile index 8d2daae892..b7960a1aca 100644 --- a/cli/cdsctl/Makefile +++ b/cli/cdsctl/Makefile @@ -63,15 +63,15 @@ clean: $(TARGET_BINARIES): $(info *** building binary $@) - @$(MAKE) --no-print-directory gobuild GOOS=$(call get_os_from_binary_file,$@) GOARCH=$(call get_arch_from_binary_file,$@) OUTPUT=$@ + @$(MAKE) --no-print-directory gobuild TAGS='' GOOS=$(call get_os_from_binary_file,$@) GOARCH=$(call get_arch_from_binary_file,$@) OUTPUT=$@ $(TARGET_BINARIES_VARIANT): - $(info *** building binary $@) - @$(MAKE) --no-print-directory gobuild GOOS=$(call get_os_from_binary_file,$(subst -nokeychain,,$@)) GOARCH=$(call get_arch_from_binary_file,$(subst -nokeychain,,$@)) OUTPUT=$@ + $(info *** building binary variant $@) + @$(MAKE) --no-print-directory gobuild TAGS="--tags='nokeychain'" GOOS=$(call get_os_from_binary_file,$(subst -nokeychain,,$@)) GOARCH=$(call get_arch_from_binary_file,$(subst -nokeychain,,$@)) OUTPUT=$@ gobuild: $(info ... OS:$(GOOS) ARCH:$(GOARCH) -> $(OUTPUT)) - @GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD) $(TARGET_LDFLAGS) -o $(OUTPUT) + @GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD) $(TAGS) $(TARGET_LDFLAGS) -o $(OUTPUT) build: $(TARGET_DIR) $(TARGET_BINARIES_VARIANT) $(TARGET_BINARIES) diff --git a/cli/cdsctl/configstore.go b/cli/cdsctl/configstore.go index 25514ae595..a10a9f37b0 100644 --- a/cli/cdsctl/configstore.go +++ b/cli/cdsctl/configstore.go @@ -1,4 +1,4 @@ -// +build freebsd openbsd linux,386 linux,arm windows,386 windows,arm nokeychain +// +build nokeychain freebsd openbsd 386 package main diff --git a/cli/cdsctl/configstore_linux_amd64.go b/cli/cdsctl/configstore_linux_amd64.go index f55f7164eb..1cd4cae741 100644 --- a/cli/cdsctl/configstore_linux_amd64.go +++ b/cli/cdsctl/configstore_linux_amd64.go @@ -1,3 +1,5 @@ +// +build !nokeychain + package main import ( diff --git a/engine/api/pipeline_parameter.go b/engine/api/pipeline_parameter.go index 458b06a5e5..27e7eb8521 100644 --- a/engine/api/pipeline_parameter.go +++ b/engine/api/pipeline_parameter.go @@ -83,7 +83,7 @@ func (api *API) updateParameterInPipelineHandler() service.Handler { return err } - p, err := pipeline.LoadPipeline(api.mustDB(), key, pipelineName, false) + p, err := pipeline.LoadPipeline(api.mustDB(), key, pipelineName, true) if err != nil { return sdk.WrapError(err, "updateParameterInPipelineHandler: Cannot load %s", pipelineName) } From 6977b7902ecd27a403bd116a06ad9508dbd0e354 Mon Sep 17 00:00:00 2001 From: Guiheux Steven Date: Mon, 17 Jun 2019 16:42:44 +0200 Subject: [PATCH 11/11] refactor(api): remove old workflow structure (#4296) --- engine/api/api.go | 4 - engine/api/application/dao.go | 6 +- engine/api/application/dao_test.go | 1 - engine/api/ascode_test.go | 2 +- engine/api/environment/environment.go | 6 +- engine/api/event/publish_workflow_run.go | 45 +- engine/api/hook.go | 11 +- engine/api/migrate/gitclone_key_migration.go | 6 +- engine/api/migrate/migration.go | 2 +- engine/api/migrate/workflow_run_old_model.go | 62 - engine/api/pipeline/pipeline.go | 5 +- engine/api/pipeline/pipeline_test.go | 1 - engine/api/project.go | 3 +- engine/api/project_group_test.go | 2 - engine/api/repositories_manager.go | 93 +- engine/api/user_test.go | 2 - .../warning_project_permission_test.go | 2 - .../warning/warning_project_variable_test.go | 2 - engine/api/worker_model_test.go | 1 - engine/api/workflow.go | 94 +- engine/api/workflow/as_code.go | 40 +- engine/api/workflow/dao.go | 387 ++--- engine/api/workflow/dao_data.go | 8 + engine/api/workflow/dao_data_context.go | 6 + engine/api/workflow/dao_data_hook.go | 115 +- engine/api/workflow/dao_fork.go | 134 -- engine/api/workflow/dao_hook.go | 415 ----- engine/api/workflow/dao_join.go | 189 --- engine/api/workflow/dao_node.go | 570 ------- engine/api/workflow/dao_node_run.go | 8 +- engine/api/workflow/dao_notification.go | 46 +- engine/api/workflow/dao_run.go | 51 +- engine/api/workflow/dao_run_test.go | 21 +- engine/api/workflow/dao_staticfiles_test.go | 2 - engine/api/workflow/dao_test.go | 503 +++++- engine/api/workflow/dao_trigger.go | 70 - engine/api/workflow/execute_node_run.go | 4 +- engine/api/workflow/gorp_model.go | 37 - engine/api/workflow/hook.go | 422 +++-- engine/api/workflow/hook_test.go | 62 - engine/api/workflow/process_node_test.go | 172 ++- engine/api/workflow/repository.go | 9 +- engine/api/workflow/run_workflow.go | 2 +- engine/api/workflow/run_workflow_test.go | 6 +- engine/api/workflow/sort.go | 38 +- engine/api/workflow/workflow_exporter.go | 24 +- engine/api/workflow/workflow_exporter_test.go | 1 - engine/api/workflow/workflow_importer.go | 43 +- engine/api/workflow/workflow_importer_test.go | 1 - engine/api/workflow/workflow_parser.go | 5 - engine/api/workflow/workflow_parser_test.go | 15 +- engine/api/workflow_ascode.go | 26 + engine/api/workflow_ascode_test.go | 12 +- engine/api/workflow_export_test.go | 4 +- engine/api/workflow_group.go | 12 +- engine/api/workflow_group_test.go | 13 - engine/api/workflow_hook_test.go | 2 - engine/api/workflow_import.go | 8 +- engine/api/workflow_import_test.go | 14 +- engine/api/workflow_queue_test.go | 5 - engine/api/workflow_run.go | 21 +- engine/api/workflow_run_test.go | 21 +- engine/api/workflow_test.go | 30 +- engine/hooks/hooks_handlers.go | 26 +- engine/hooks/scheduler.go | 2 +- engine/hooks/tasks.go | 15 +- engine/sql/168_disabledConstraint.sql | 13 + sdk/cdsclient/client_workflow_hooks.go | 4 +- sdk/cdsclient/interface.go | 2 +- sdk/error.go | 6 + sdk/exportentities/workflow.go | 4 +- sdk/workflow.go | 1374 +---------------- sdk/workflow_data.go | 7 +- sdk/workflow_hook.go | 183 +-- sdk/workflow_node.go | 29 +- sdk/workflow_test.go | 151 -- tests/clictl_template_bulk.yml | 4 +- tests/fixtures/template/bulk_request.yml | 6 +- ui/src/app/model/workflow.model.ts | 4 +- .../app/shared/table/data-table.component.ts | 7 +- .../context/wizard.context.component.ts | 19 + .../workflow.notification.list.component.ts | 2 +- 82 files changed, 1309 insertions(+), 4473 deletions(-) delete mode 100644 engine/api/migrate/workflow_run_old_model.go delete mode 100644 engine/api/workflow/dao_fork.go delete mode 100644 engine/api/workflow/dao_hook.go delete mode 100644 engine/api/workflow/dao_join.go delete mode 100644 engine/api/workflow/dao_trigger.go delete mode 100644 engine/api/workflow/hook_test.go create mode 100644 engine/sql/168_disabledConstraint.sql delete mode 100644 sdk/workflow_test.go diff --git a/engine/api/api.go b/engine/api/api.go index 70b05c3220..850cf70fb2 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -776,10 +776,6 @@ func (a *API) Serve(ctx context.Context) error { sdk.GoRoutine(ctx, "migrate.KeyMigration", func(ctx context.Context) { migrate.KeyMigration(a.Cache, a.DBConnectionFactory.GetDBMap, &sdk.User{Admin: true}) }, a.PanicDump()) - - migrate.Add(sdk.Migration{Name: "WorkflowOldStruct", Release: "0.38.1", Mandatory: true, ExecFunc: func(ctx context.Context) error { - return migrate.WorkflowRunOldModel(ctx, a.DBConnectionFactory.GetDBMap) - }}) migrate.Add(sdk.Migration{Name: "WorkflowNotification", Release: "0.38.1", Mandatory: true, ExecFunc: func(ctx context.Context) error { return migrate.WorkflowNotifications(a.Cache, a.DBConnectionFactory.GetDBMap) }}) diff --git a/engine/api/application/dao.go b/engine/api/application/dao.go index 1d55528577..9f1901c31a 100644 --- a/engine/api/application/dao.go +++ b/engine/api/application/dao.go @@ -106,9 +106,9 @@ func LoadByWorkflowID(db gorp.SqlExecutor, workflowID int64) ([]sdk.Application, apps := []sdk.Application{} query := fmt.Sprintf(`SELECT DISTINCT %s FROM application - JOIN workflow_node_context ON workflow_node_context.application_id = application.id - JOIN workflow_node ON workflow_node.id = workflow_node_context.workflow_node_id - JOIN workflow ON workflow.id = workflow_node.workflow_id + JOIN w_node_context ON w_node_context.application_id = application.id + JOIN w_node ON w_node.id = w_node_context.node_id + JOIN workflow ON workflow.id = w_node.workflow_id WHERE workflow.id = $1`, appRows) if _, err := db.Select(&apps, query, workflowID); err != nil { diff --git a/engine/api/application/dao_test.go b/engine/api/application/dao_test.go index 6075f753d4..da58da9ba1 100644 --- a/engine/api/application/dao_test.go +++ b/engine/api/application/dao_test.go @@ -193,7 +193,6 @@ func TestLoadByWorkflowID(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() proj, _ = project.LoadByID(db, cache, proj.ID, u, project.LoadOptions.WithApplications, project.LoadOptions.WithPipelines, project.LoadOptions.WithEnvironments, project.LoadOptions.WithGroups) diff --git a/engine/api/ascode_test.go b/engine/api/ascode_test.go index 6f64975702..acbf9e53a2 100644 --- a/engine/api/ascode_test.go +++ b/engine/api/ascode_test.go @@ -212,7 +212,7 @@ func Test_postPerformImportAsCodeHandler(t *testing.T) { t.Logf("RequestURI: %s", r.URL.Path) switch r.URL.Path { case "/task/bulk": - hooks := map[string]sdk.WorkflowNodeHook{} + hooks := map[string]sdk.NodeHook{} if err := service.UnmarshalBody(r, &hooks); err != nil { return nil, sdk.WithStack(err) } diff --git a/engine/api/environment/environment.go b/engine/api/environment/environment.go index d404300219..ff601b144b 100644 --- a/engine/api/environment/environment.go +++ b/engine/api/environment/environment.go @@ -129,9 +129,9 @@ func LoadEnvironmentByName(db gorp.SqlExecutor, projectKey, envName string) (*sd func LoadByWorkflowID(db gorp.SqlExecutor, workflowID int64) ([]sdk.Environment, error) { envs := []sdk.Environment{} query := `SELECT DISTINCT environment.* FROM environment - JOIN workflow_node_context ON workflow_node_context.environment_id = environment.id - JOIN workflow_node ON workflow_node.id = workflow_node_context.workflow_node_id - JOIN workflow ON workflow.id = workflow_node.workflow_id + JOIN w_node_context ON w_node_context.environment_id = environment.id + JOIN w_node ON w_node.id = w_node_context.node_id + JOIN workflow ON workflow.id = w_node.workflow_id WHERE workflow.id = $1` if _, err := db.Select(&envs, query, workflowID); err != nil { diff --git a/engine/api/event/publish_workflow_run.go b/engine/api/event/publish_workflow_run.go index fc8b0fd6c6..986a792288 100644 --- a/engine/api/event/publish_workflow_run.go +++ b/engine/api/event/publish_workflow_run.go @@ -84,36 +84,25 @@ func PublishWorkflowNodeRun(db gorp.SqlExecutor, nr sdk.WorkflowNodeRun, w sdk.W var nodeName string var app sdk.Application var env sdk.Environment - n := w.GetNode(nr.WorkflowNodeID) - if n == nil { - // check on workflow data - wnode := w.WorkflowData.NodeByID(nr.WorkflowNodeID) - if wnode == nil { - log.Warning("PublishWorkflowNodeRun> Unable to publish event on node %d", nr.WorkflowNodeID) - return - } - nodeName = wnode.Name - if wnode.Context != nil && wnode.Context.PipelineID != 0 { - pipName = w.Pipelines[wnode.Context.PipelineID].Name - } - if wnode.Context != nil && wnode.Context.ApplicationID != 0 { - app = w.Applications[wnode.Context.ApplicationID] - } - if wnode.Context != nil && wnode.Context.EnvironmentID != 0 { - env = w.Environments[wnode.Context.EnvironmentID] - } - e.NodeType = wnode.Type - } else { - nodeName = n.Name - pipName = w.Pipelines[n.PipelineID].Name - if n.Context != nil && n.Context.Application != nil { - app = *n.Context.Application - } - if n.Context != nil && n.Context.Environment != nil { - env = *n.Context.Environment - } + // check on workflow data + wnode := w.WorkflowData.NodeByID(nr.WorkflowNodeID) + if wnode == nil { + log.Warning("PublishWorkflowNodeRun> Unable to publish event on node %d", nr.WorkflowNodeID) + return + } + nodeName = wnode.Name + if wnode.Context != nil && wnode.Context.PipelineID != 0 { + pipName = w.Pipelines[wnode.Context.PipelineID].Name + } + + if wnode.Context != nil && wnode.Context.ApplicationID != 0 { + app = w.Applications[wnode.Context.ApplicationID] + } + if wnode.Context != nil && wnode.Context.EnvironmentID != 0 { + env = w.Environments[wnode.Context.EnvironmentID] } + e.NodeType = wnode.Type // Try to get gerrit variable var project, changeID, branch, revision, url string diff --git a/engine/api/hook.go b/engine/api/hook.go index 1b03f9382d..722dd0efaa 100644 --- a/engine/api/hook.go +++ b/engine/api/hook.go @@ -25,7 +25,7 @@ func (api *API) getHookPollingVCSEvents() service.Handler { lastExec := time.Now() workflowID, errV := requestVarInt(r, "workflowID") if errV != nil { - return sdk.WrapError(sdk.ErrWrongRequest, "getHookPollingVCSEvents> cannot convert workflowID to int %s", errV) + return errV } if r.Header.Get("X-CDS-Last-Execution") != "" { @@ -36,22 +36,19 @@ func (api *API) getHookPollingVCSEvents() service.Handler { h, errL := workflow.LoadHookByUUID(api.mustDB(), uuid) if errL != nil { - return sdk.WrapError(errL, "getHookPollingVCSEvents> cannot load hook") - } - if h == nil { - return sdk.ErrNotFound + return errL } proj, errProj := project.Load(api.mustDB(), api.Cache, h.Config[sdk.HookConfigProject].Value, nil) if errProj != nil { - return sdk.WrapError(errProj, "getHookPollingVCSEvents> cannot load project") + return errProj } //get the client for the repositories manager vcsServer := repositoriesmanager.GetProjectVCSServer(proj, vcsServerParam) client, errR := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, proj.Key, vcsServer) if errR != nil { - return sdk.WrapError(errR, "getHookPollingVCSEvents> Unable to get client for %s %s", proj.Key, vcsServerParam) + return errR } //Check if the polling if disabled diff --git a/engine/api/migrate/gitclone_key_migration.go b/engine/api/migrate/gitclone_key_migration.go index 54dcefc759..d8d84f4e4a 100644 --- a/engine/api/migrate/gitclone_key_migration.go +++ b/engine/api/migrate/gitclone_key_migration.go @@ -182,12 +182,12 @@ func migrateActionGitClonePipeline(db gorp.SqlExecutor, store cache.Store, p pip if err != nil { return err } - node := w.GetNodeByName(p.WorkflowNodeName) + node := w.WorkflowData.NodeByName(p.WorkflowNodeName) if node == nil { return sdk.ErrWorkflowNodeNotFound } - if node.Context != nil && node.Context.Application != nil { - p.AppName = node.Context.Application.Name + if node.Context != nil && node.Context.ApplicationID != 0 { + p.AppName = w.Applications[node.Context.ApplicationID].Name } } diff --git a/engine/api/migrate/migration.go b/engine/api/migrate/migration.go index 2756ffb908..501d3eba0f 100644 --- a/engine/api/migrate/migration.go +++ b/engine/api/migrate/migration.go @@ -15,7 +15,7 @@ import ( ) // MinCompatibleRelease represent the minimum release which is working with these migrations, need to update when we delete migration in our codebase -const MinCompatibleRelease = "0.38.1" +const MinCompatibleRelease = "0.39.3" var migrations = []sdk.Migration{} diff --git a/engine/api/migrate/workflow_run_old_model.go b/engine/api/migrate/workflow_run_old_model.go deleted file mode 100644 index b0edea4d85..0000000000 --- a/engine/api/migrate/workflow_run_old_model.go +++ /dev/null @@ -1,62 +0,0 @@ -package migrate - -import ( - "context" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/workflow" - "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" -) - -// WorkflowRunOldModel migrates workflow run with old workflow struct to the new one -func WorkflowRunOldModel(ctx context.Context, DBFunc func() *gorp.DbMap) error { - db := DBFunc() - - log.Info("migrate>WorkflowRunOldModel> Start migration") - - for { - ids, err := workflow.LoadRunIDsWithOldModel(db) - if err != nil { - return err - } - - if len(ids) == 0 { - break - } - - log.Info("migrate>WorkflowRunOldModel> %d run to migrate", len(ids)) - for _, id := range ids { - if err := migrateRun(ctx, db, id); err != nil && !sdk.ErrorIs(err, sdk.ErrLocked) { - return err - } - } - } - - log.Info("End WorkflowRunOldModel migration") - return nil -} - -func migrateRun(ctx context.Context, db *gorp.DbMap, id int64) error { - tx, err := db.Begin() - if err != nil { - return sdk.WithStack(err) - } - defer tx.Rollback() // nolint - - run, err := workflow.LockRun(tx, id) - if err != nil { - if sdk.ErrorIs(err, sdk.ErrLocked) { - // Already lock, go to next run - return nil - } - return err - } - - if err := workflow.MigrateWorkflowRun(ctx, tx, run); err != nil { - return err - } - - return sdk.WithStack(tx.Commit()) -} diff --git a/engine/api/pipeline/pipeline.go b/engine/api/pipeline/pipeline.go index 8b07373d46..35ee46a114 100644 --- a/engine/api/pipeline/pipeline.go +++ b/engine/api/pipeline/pipeline.go @@ -175,8 +175,9 @@ func LoadByWorkflowID(db gorp.SqlExecutor, workflowID int64) ([]sdk.Pipeline, er pips := []Pipeline{} query := `SELECT DISTINCT pipeline.* FROM pipeline - JOIN workflow_node ON pipeline.id = workflow_node.pipeline_id - JOIN workflow ON workflow_node.workflow_id = workflow.id + JOIN w_node_context ON pipeline.id = w_node_context.pipeline_id + JOIN w_node ON w_node.id = w_node_context.node_id + JOIN workflow ON w_node.workflow_id = workflow.id WHERE workflow.id = $1` if _, err := db.Select(&pips, query, workflowID); err != nil { diff --git a/engine/api/pipeline/pipeline_test.go b/engine/api/pipeline/pipeline_test.go index 4bdac005d0..03dd29dd41 100644 --- a/engine/api/pipeline/pipeline_test.go +++ b/engine/api/pipeline/pipeline_test.go @@ -180,7 +180,6 @@ func TestLoadByWorkflowID(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() proj, _ = project.LoadByID(db, cache, proj.ID, u, project.LoadOptions.WithApplications, project.LoadOptions.WithPipelines, project.LoadOptions.WithEnvironments, project.LoadOptions.WithGroups) diff --git a/engine/api/project.go b/engine/api/project.go index 0b0a82a745..b6ff11bd58 100644 --- a/engine/api/project.go +++ b/engine/api/project.go @@ -102,10 +102,9 @@ func (api *API) getProjectsHandler() service.Handler { return err } - wapps := w.GetApplications() //Checks the workflow use one of the applications wapps: - for _, a := range wapps { + for _, a := range w.Applications { for _, b := range apps { if a.Name == b.Name { ws = append(ws, p.Workflows[i]) diff --git a/engine/api/project_group_test.go b/engine/api/project_group_test.go index 3be50f3ebf..fb0aaa3ccc 100644 --- a/engine/api/project_group_test.go +++ b/engine/api/project_group_test.go @@ -45,8 +45,6 @@ func Test_ProjectPerms(t *testing.T) { ProjectKey: proj.Key, } - (&newWf).RetroMigrate() - //Prepare request vars := map[string]string{ "permProjectKey": proj.Key, diff --git a/engine/api/repositories_manager.go b/engine/api/repositories_manager.go index e476b96ad5..674aa37d8d 100644 --- a/engine/api/repositories_manager.go +++ b/engine/api/repositories_manager.go @@ -506,19 +506,24 @@ func (api *API) attachRepositoriesManagerHandler() service.Handler { } for _, wf := range usage.Workflows { - rootCtx, errNc := workflow.LoadNodeContext(db, api.Cache, proj, wf.RootID, u, workflow.LoadOptions{}) - if errNc != nil { - return sdk.WrapError(errNc, "attachRepositoriesManager> Cannot DefaultPayloadToMap") + wfDB, errWL := workflow.LoadByID(db, api.Cache, proj, wf.ID, u, workflow.LoadOptions{}) + if errWL != nil { + return errWL } - if rootCtx.ApplicationID != app.ID { - continue + wfOld, errWL := workflow.LoadByID(db, api.Cache, proj, wf.ID, u, workflow.LoadOptions{}) + if errWL != nil { + return errWL } - wf.Root = &sdk.WorkflowNode{ - Context: rootCtx, + if wfDB.WorkflowData.Node.Context == nil { + wfDB.WorkflowData.Node.Context = &sdk.NodeContext{} + } + if wfDB.WorkflowData.Node.Context.ApplicationID != app.ID { + continue } - payload, errD := rootCtx.DefaultPayloadToMap() + + payload, errD := wfDB.WorkflowData.Node.Context.DefaultPayloadToMap() if errD != nil { return sdk.WrapError(errP, "attachRepositoriesManager> Cannot DefaultPayloadToMap") } @@ -527,14 +532,18 @@ func (api *API) attachRepositoriesManagerHandler() service.Handler { continue } - defaultPayload, errPay := workflow.DefaultPayload(ctx, db, api.Cache, proj, &wf) + defaultPayload, errPay := workflow.DefaultPayload(ctx, db, api.Cache, proj, wfDB) if errPay != nil { return sdk.WrapError(errPay, "attachRepositoriesManager> Cannot get defaultPayload") } - wf.Root.Context.DefaultPayload = defaultPayload - if err := workflow.UpdateNodeContext(db, wf.Root.Context); err != nil { - return sdk.WrapError(err, "Cannot update node context %d", wf.Root.Context.ID) + wfDB.WorkflowData.Node.Context.DefaultPayload = defaultPayload + + if err := workflow.Update(ctx, db, api.Cache, wfDB, proj, u, workflow.UpdateOptions{DisableHookManagement: true}); err != nil { + return sdk.WrapError(err, "Cannot update node context %d", wf.WorkflowData.Node.Context.ID) } + + event.PublishWorkflowUpdate(proj.Key, *wfDB, *wfOld, u) + } } @@ -557,6 +566,15 @@ func (api *API) detachRepositoriesManagerHandler() service.Handler { return sdk.WrapError(errl, "detachRepositoriesManager> error on load project %s", projectKey) } + // Check if there is hooks on this application + hooksCount, err := workflow.CountHooksByApplication(db, app.ID) + if err != nil { + return err + } + if hooksCount > 0 { + return sdk.WithStack(sdk.ErrRepositoryUsedByHook) + } + //Remove all the things in a transaction tx, errT := db.Begin() if errT != nil { @@ -572,57 +590,6 @@ func (api *API) detachRepositoriesManagerHandler() service.Handler { return sdk.WrapError(err, "Cannot commit transaction") } - usage, errU := loadApplicationUsage(db, projectKey, appName) - if errU != nil { - return sdk.WrapError(errU, "detachRepositoriesManager> Cannot load application usage") - } - - // Update default payload of linked workflow root - if len(usage.Workflows) > 0 { - proj, errP := project.Load(db, api.Cache, projectKey, u) - if errP != nil { - return sdk.WrapError(errP, "detachRepositoriesManager> Cannot load project") - } - - hookToDelete := map[string]sdk.WorkflowNodeHook{} - for _, wf := range usage.Workflows { - nodeHooks, err := workflow.LoadHooksByNodeID(db, wf.RootID) - if err != nil { - return sdk.WrapError(err, "Cannot load node hook by nodeID %d", wf.RootID) - } - - for _, nodeHook := range nodeHooks { - if nodeHook.WorkflowHookModel.Name != sdk.RepositoryWebHookModelName && nodeHook.WorkflowHookModel.Name != sdk.GitPollerModelName { - continue - } - hookToDelete[nodeHook.UUID] = nodeHook - } - } - - if len(hookToDelete) > 0 { - txDel, errTx := db.Begin() - if errTx != nil { - return sdk.WrapError(errTx, "detachRepositoriesManager> Cannot create delete transaction") - } - defer func() { - _ = txDel.Rollback() - }() - - for _, nodeHook := range hookToDelete { - if err := workflow.DeleteHook(txDel, &nodeHook); err != nil { - return sdk.WrapError(err, "Cannot delete hooks") - } - } - if err := workflow.DeleteHookConfiguration(ctx, txDel, api.Cache, proj, hookToDelete); err != nil { - return sdk.WrapError(err, "Cannot delete hooks vcs configuration") - } - - if err := txDel.Commit(); err != nil { - return sdk.WrapError(err, "Cannot commit delete transaction") - } - } - } - event.PublishApplicationRepositoryDelete(projectKey, appName, app.VCSServer, app.RepositoryFullname, u) return service.WriteJSON(w, app, http.StatusOK) diff --git a/engine/api/user_test.go b/engine/api/user_test.go index 81ced356ac..bc10300fff 100644 --- a/engine/api/user_test.go +++ b/engine/api/user_test.go @@ -496,8 +496,6 @@ func Test_postUserFavoriteHandler(t *testing.T) { }, } - (&wf).RetroMigrate() - test.NoError(t, workflow.Insert(db, api.Cache, &wf, proj, u1)) uri := api.Router.GetRoute("POST", api.postUserFavoriteHandler, nil) diff --git a/engine/api/warning/warning_project_permission_test.go b/engine/api/warning/warning_project_permission_test.go index 07f06e67e5..c0d226b333 100644 --- a/engine/api/warning/warning_project_permission_test.go +++ b/engine/api/warning/warning_project_permission_test.go @@ -58,8 +58,6 @@ func TestMissingProjectPermissionWorkflowWarning(t *testing.T) { }, } - (&w).RetroMigrate() - projUpdate, err := project.Load(db, cache, proj.Key, u, project.LoadOptions.WithPipelines) assert.NoError(t, err) test.NoError(t, workflow.Insert(db, cache, &w, projUpdate, u)) diff --git a/engine/api/warning/warning_project_variable_test.go b/engine/api/warning/warning_project_variable_test.go index 39ddc7d7f0..e8f726d306 100644 --- a/engine/api/warning/warning_project_variable_test.go +++ b/engine/api/warning/warning_project_variable_test.go @@ -358,8 +358,6 @@ func TestMissingProjectVariableWorkflow(t *testing.T) { }, } - (&w).RetroMigrate() - projUpdate, err := project.Load(db, cache, proj.Key, u, project.LoadOptions.WithPipelines) assert.NoError(t, err) test.NoError(t, workflow.Insert(db, cache, &w, projUpdate, u)) diff --git a/engine/api/worker_model_test.go b/engine/api/worker_model_test.go index 55a9c33dfa..6ccdf866f6 100644 --- a/engine/api/worker_model_test.go +++ b/engine/api/worker_model_test.go @@ -265,7 +265,6 @@ func Test_WorkerModelUsage(t *testing.T) { }, }, } - (&wf).RetroMigrate() test.NoError(t, workflow.Insert(db, api.Cache, &wf, proj, u)) diff --git a/engine/api/workflow.go b/engine/api/workflow.go index 2d33ca3f2d..3910b80c31 100644 --- a/engine/api/workflow.go +++ b/engine/api/workflow.go @@ -106,20 +106,6 @@ func (api *API) getWorkflowHandler() service.Handler { w1.Permission = permission.WorkflowPermission(key, w1.Name, deprecatedGetUser(ctx)) - // FIXME Sync hooks on new model. To delete when hooks will be compute on new model - hooks := w1.GetHooks() - w1.WorkflowData.Node.Hooks = make([]sdk.NodeHook, 0, len(hooks)) - for _, h := range hooks { - w1.WorkflowData.Node.Hooks = append(w1.WorkflowData.Node.Hooks, sdk.NodeHook{ - Ref: h.Ref, - HookModelID: h.WorkflowHookModelID, - Config: h.Config, - UUID: h.UUID, - HookModelName: h.WorkflowHookModel.Name, - NodeID: w1.WorkflowData.Node.ID, - }) - } - w1.URLs.APIURL = api.Config.URL.API + api.Router.GetRoute("GET", api.getWorkflowHandler, map[string]string{"key": key, "permWorkflowName": w1.Name}) w1.URLs.UIURL = api.Config.URL.UI + "/project/" + key + "/workflow/" + w1.Name @@ -260,7 +246,7 @@ func (api *API) postWorkflowLabelHandler() service.Handler { } } - wf, errW := workflow.Load(ctx, db, api.Cache, proj, workflowName, u, workflow.LoadOptions{WithoutNode: true, WithLabels: true}) + wf, errW := workflow.Load(ctx, db, api.Cache, proj, workflowName, u, workflow.LoadOptions{WithLabels: true}) if errW != nil { return sdk.WrapError(errW, "postWorkflowLabelHandler> cannot load workflow %s/%s", key, workflowName) } @@ -299,7 +285,7 @@ func (api *API) deleteWorkflowLabelHandler() service.Handler { return sdk.WrapError(errP, "deleteWorkflowLabelHandler> cannot load project %s", key) } - wf, errW := workflow.Load(ctx, db, api.Cache, proj, workflowName, u, workflow.LoadOptions{WithoutNode: true}) + wf, errW := workflow.Load(ctx, db, api.Cache, proj, workflowName, u, workflow.LoadOptions{}) if errW != nil { return sdk.WrapError(errW, "deleteWorkflowLabelHandler> cannot load workflow %s/%s", key, workflowName) } @@ -326,48 +312,34 @@ func (api *API) postWorkflowHandler() service.Handler { project.LoadOptions.WithIntegrations, ) if errP != nil { - return sdk.WrapError(errP, "Cannot load Project %s", key) + return errP } var wf sdk.Workflow if err := service.UnmarshalBody(r, &wf); err != nil { - return sdk.WrapError(err, "Cannot read body") + return err } if wf.WorkflowData == nil { - return sdk.WrapError(sdk.ErrWrongRequest, "No node found") + return sdk.WrapError(sdk.ErrWrongRequest, "no node found") } if err := workflow.RenameNode(api.mustDB(), &wf); err != nil { - return sdk.WrapError(err, "postWorkflowHandler> Cannot rename node") + return err } - (&wf).RetroMigrate() - wf.ProjectID = p.ID wf.ProjectKey = key tx, errT := api.mustDB().Begin() if errT != nil { - return sdk.WrapError(errT, "Cannot start transaction") + return sdk.WithStack(errT) } defer tx.Rollback() - if wf.Root != nil && wf.Root.Context != nil && (wf.Root.Context.Application != nil || wf.Root.Context.ApplicationID != 0) { - var err error - if wf.Root.Context.DefaultPayload, err = workflow.DefaultPayload(ctx, tx, api.Cache, p, &wf); err != nil { - log.Warning("postWorkflowHandler> Cannot set default payload : %v", err) - } - wf.WorkflowData.Node.Context.DefaultPayload = wf.Root.Context.DefaultPayload - } - if err := workflow.Insert(tx, api.Cache, &wf, p, deprecatedGetUser(ctx)); err != nil { return sdk.WrapError(err, "Cannot insert workflow") } - if errHr := workflow.HookRegistration(ctx, tx, api.Cache, nil, wf, p); errHr != nil { - return sdk.WrapError(errHr, "postWorkflowHandler>Hook registration failed") - } - if err := tx.Commit(); err != nil { return sdk.WrapError(err, "Cannot commit transaction") } @@ -376,14 +348,14 @@ func (api *API) postWorkflowHandler() service.Handler { if errl != nil { return sdk.WrapError(errl, "Cannot load workflow") } + + event.PublishWorkflowAdd(p.Key, *wf1, deprecatedGetUser(ctx)) + wf1.Permission = permission.PermissionReadWriteExecute //We filter project and workflow configurtaion key, because they are always set on insertHooks wf1.FilterHooksConfig(sdk.HookConfigProject, sdk.HookConfigWorkflow) - // TODO REMOVE WHEN WE WILL DELETE OLD NODE STRUCT - wf1.Root = nil - wf1.Joins = nil return service.WriteJSON(w, wf1, http.StatusCreated) } } @@ -423,13 +395,7 @@ func (api *API) putWorkflowHandler() service.Handler { return sdk.WrapError(err, "Update> cannot check pipeline name") } - // TODO : Delete in migration step 3 - // Retro migrate workflow - (&wf).RetroMigrate() - wf.ID = oldW.ID - wf.RootID = oldW.RootID - wf.Root.ID = oldW.RootID wf.ProjectID = p.ID wf.ProjectKey = key @@ -439,25 +405,11 @@ func (api *API) putWorkflowHandler() service.Handler { } defer tx.Rollback() - // TODO Remove old struct - if wf.Root.Context != nil && (wf.Root.Context.Application != nil || wf.Root.Context.ApplicationID != 0) { - var err error - if wf.Root.Context.DefaultPayload, err = workflow.DefaultPayload(ctx, tx, api.Cache, p, &wf); err != nil { - log.Warning("putWorkflowHandler> Cannot set default payload : %v", err) - } - wf.WorkflowData.Node.Context.DefaultPayload = wf.Root.Context.DefaultPayload - } - - if err := workflow.Update(ctx, tx, api.Cache, &wf, oldW, p, deprecatedGetUser(ctx)); err != nil { + if err := workflow.Update(ctx, tx, api.Cache, &wf, p, deprecatedGetUser(ctx), workflow.UpdateOptions{OldWorkflow: oldW}); err != nil { return sdk.WrapError(err, "Cannot update workflow") } - // HookRegistration after workflow.Update. It needs hooks to be created on DB - if errHr := workflow.HookRegistration(ctx, tx, api.Cache, oldW, wf, p); errHr != nil { - return sdk.WrapError(errHr, "putWorkflowHandler> HookRegistration") - } - - if defaultTags, ok := wf.Metadata["default_tags"]; wf.Root.IsLinkedToRepo() && (!ok || defaultTags == "") { + if defaultTags, ok := wf.Metadata["default_tags"]; wf.WorkflowData.Node.IsLinkedToRepo(&wf) && (!ok || defaultTags == "") { if wf.Metadata == nil { wf.Metadata = sdk.Metadata{} } @@ -475,6 +427,9 @@ func (api *API) putWorkflowHandler() service.Handler { if errl != nil { return sdk.WrapError(errl, "putWorkflowHandler> Cannot load workflow") } + + event.PublishWorkflowUpdate(p.Key, *wf1, *oldW, deprecatedGetUser(ctx)) + wf1.Permission = permission.PermissionReadWriteExecute usage, errU := loadWorkflowUsage(api.mustDB(), wf1.ID) @@ -483,25 +438,8 @@ func (api *API) putWorkflowHandler() service.Handler { } wf1.Usage = &usage - // FIXME Sync hooks on new model. To delete when hooks will be compute on new model - hooks := wf1.GetHooks() - wf1.WorkflowData.Node.Hooks = make([]sdk.NodeHook, 0, len(hooks)) - for _, h := range hooks { - wf1.WorkflowData.Node.Hooks = append(wf1.WorkflowData.Node.Hooks, sdk.NodeHook{ - Ref: h.Ref, - HookModelID: h.WorkflowHookModelID, - Config: h.Config, - UUID: h.UUID, - HookModelName: h.WorkflowHookModel.Name, - NodeID: wf1.WorkflowData.Node.ID, - }) - } - //We filter project and workflow configuration key, because they are always set on insertHooks wf1.FilterHooksConfig(sdk.HookConfigProject, sdk.HookConfigWorkflow) - // TODO REMOVE - wf1.Root = nil - wf1.Joins = nil return service.WriteJSON(w, wf1, http.StatusOK) } } @@ -644,7 +582,7 @@ func (api *API) getWorkflowHookHandler() service.Handler { return sdk.WrapError(errW, "getWorkflowHookHandler> Cannot load Workflow %s/%s", key, name) } - whooks := wf.GetHooks() + whooks := wf.WorkflowData.GetHooks() _, has := whooks[uuid] if !has { return sdk.WrapError(sdk.ErrNotFound, "getWorkflowHookHandler> Cannot load Workflow %s/%s hook %s", key, name, uuid) diff --git a/engine/api/workflow/as_code.go b/engine/api/workflow/as_code.go index ba1222cd88..1850c38a44 100644 --- a/engine/api/workflow/as_code.go +++ b/engine/api/workflow/as_code.go @@ -236,27 +236,9 @@ func UpdateWorkflowAsCodeResult(ctx context.Context, db *gorp.DbMap, store cache CreationDate: time.Now(), } - // add repo webhook - found := false - for _, h := range wf.WorkflowData.GetHooks() { - if h.HookModelName == sdk.RepositoryWebHookModelName { - found = true - break - } - } - - if !found { - h := sdk.WorkflowNodeHook{ - Config: sdk.RepositoryWebHookModel.DefaultConfig, - WorkflowHookModel: sdk.RepositoryWebHookModel, - } - wf.Root.Hooks = append(wf.Root.Hooks, h) - } - - // Update the workflow - oldW, err := LoadByID(db, store, p, wf.ID, u, LoadOptions{}) - if err != nil { - log.Error("postWorkflowAsCodeHandler> unable to load workflow %s: %v", wf.Name, err) + oldW, errOld := LoadByID(db, store, p, wf.ID, u, LoadOptions{}) + if errOld != nil { + log.Error("postWorkflowAsCodeHandler> unable to load workflow: %v", err) ope.Status = sdk.OperationStatusError ope.Error = "unable to load workflow" return @@ -278,26 +260,14 @@ func UpdateWorkflowAsCodeResult(ctx context.Context, db *gorp.DbMap, store cache return } - if err := Update(ctx, tx, store, wf, oldW, p, u); err != nil { - log.Error("postWorkflowAsCodeHandler> unable to update workflow: %v", err) - ope.Status = sdk.OperationStatusError - ope.Error = "unable to update workflow" - return - } - - if errHr := HookRegistration(ctx, tx, store, oldW, *wf, p); errHr != nil { - log.Error("postWorkflowAsCodeHandler> unable to update hook registration: %v", errHr) - ope.Status = sdk.OperationStatusError - ope.Error = "unable to update hook" - return - } - if err := tx.Commit(); err != nil { log.Error("postWorkflowAsCodeHandler> unable to commit workflow: %v", err) ope.Status = sdk.OperationStatusError ope.Error = "unable to commit transaction" return } + + event.PublishWorkflowUpdate(p.Key, *wf, *oldW, u) return } diff --git a/engine/api/workflow/dao.go b/engine/api/workflow/dao.go index 18ae6abd93..a757edec2a 100644 --- a/engine/api/workflow/dao.go +++ b/engine/api/workflow/dao.go @@ -44,7 +44,6 @@ func GetAllByIDs(db gorp.SqlExecutor, ids []int64) ([]sdk.Workflow, error) { // LoadOptions custom option for loading workflow type LoadOptions struct { DeepPipeline bool - WithoutNode bool Base64Keys bool OnlyRootNode bool WithFavorites bool @@ -53,6 +52,12 @@ type LoadOptions struct { WithAsCodeUpdateEvent bool } +// UpdateOptions is option to parse a workflow +type UpdateOptions struct { + DisableHookManagement bool + OldWorkflow *sdk.Workflow +} + // CountVarInWorkflowData represents the result of CountVariableInWorkflow function type CountVarInWorkflowData struct { WorkflowName string `db:"workflow_name"` @@ -77,16 +82,16 @@ func Exists(db gorp.SqlExecutor, key string, name string) (bool, error) { // CountVariableInWorkflow counts how many time the given variable is used on all workflows of the given project func CountVariableInWorkflow(db gorp.SqlExecutor, projectKey string, varName string) ([]CountVarInWorkflowData, error) { query := ` - SELECT DISTINCT workflow.name as workflow_name, workflow_node.name as node_name + SELECT DISTINCT workflow.name as workflow_name, w_node.name as node_name FROM workflow JOIN project ON project.id = workflow.project_id - JOIN workflow_node ON workflow_node.workflow_id = workflow.id - JOIN workflow_node_context ON workflow_node_context.workflow_node_id = workflow_node.id + JOIN w_node ON w_node.workflow_id = workflow.id + JOIN w_node_context ON w_node_context.node_id = w_node.id WHERE project.projectkey = $1 AND ( - workflow_node_context.default_pipeline_parameters::TEXT LIKE $2 + w_node_context.default_pipeline_parameters::TEXT LIKE $2 OR - workflow_node_context.default_payload::TEXT LIKE $2 + w_node_context.default_payload::TEXT LIKE $2 ); ` var datas []CountVarInWorkflowData @@ -109,10 +114,10 @@ func UpdateIcon(db gorp.SqlExecutor, workflowID int64, icon string) error { func UpdateMetadata(db gorp.SqlExecutor, workflowID int64, metadata sdk.Metadata) error { b, err := json.Marshal(metadata) if err != nil { - return err + return sdk.WithStack(err) } if _, err := db.Exec("update workflow set metadata = $1 where id = $2", b, workflowID); err != nil { - return err + return sdk.WithStack(err) } return nil @@ -283,7 +288,6 @@ func Load(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk observability.Tag("with_pipeline", opts.DeepPipeline), observability.Tag("only_root", opts.OnlyRootNode), observability.Tag("with_base64_keys", opts.Base64Keys), - observability.Tag("without_node", opts.WithoutNode), ) defer end() @@ -317,10 +321,8 @@ func Load(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk } res.ProjectKey = proj.Key - if !opts.WithoutNode { - if err := IsValid(ctx, store, db, res, proj, u); err != nil { - return nil, sdk.WrapError(err, "Unable to valid workflow") - } + if err := IsValid(ctx, store, db, res, proj, u); err != nil { + return nil, sdk.WrapError(err, "Unable to valid workflow") } return res, nil @@ -362,10 +364,8 @@ func LoadByID(db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, id int6 return nil, sdk.WrapError(err, "Unable to load workflow %d", id) } - if !opts.WithoutNode { - if err := IsValid(context.TODO(), store, db, res, proj, u); err != nil { - return nil, sdk.WrapError(err, "Unable to valid workflow") - } + if err := IsValid(context.TODO(), store, db, res, proj, u); err != nil { + return nil, sdk.WrapError(err, "Unable to valid workflow") } return res, nil } @@ -377,8 +377,9 @@ func LoadByPipelineName(db gorp.SqlExecutor, projectKey string, pipName string) select distinct workflow.* from workflow join project on project.id = workflow.project_id - join workflow_node on workflow_node.workflow_id = workflow.id - join pipeline on pipeline.id = workflow_node.pipeline_id + join w_node on w_node.workflow_id = workflow.id + join w_node_context on w_node_context.node_id = w_node.id + join pipeline on pipeline.id = w_node_context.pipeline_id where project.projectkey = $1 and pipeline.name = $2 and workflow.to_delete = false order by workflow.name asc` @@ -406,9 +407,9 @@ func LoadByApplicationName(db gorp.SqlExecutor, projectKey string, appName strin select distinct workflow.* from workflow join project on project.id = workflow.project_id - join workflow_node on workflow_node.workflow_id = workflow.id - join workflow_node_context on workflow_node_context.workflow_node_id = workflow_node.id - join application on workflow_node_context.application_id = application.id + join w_node on w_node.workflow_id = workflow.id + join w_node_context on w_node_context.node_id = w_node.id + join application on w_node_context.application_id = application.id where project.projectkey = $1 and application.name = $2 and workflow.to_delete = false order by workflow.name asc` @@ -436,9 +437,9 @@ func LoadByEnvName(db gorp.SqlExecutor, projectKey string, envName string) ([]sd select distinct workflow.* from workflow join project on project.id = workflow.project_id - join workflow_node on workflow_node.workflow_id = workflow.id - join workflow_node_context on workflow_node_context.workflow_node_id = workflow_node.id - join environment on workflow_node_context.environment_id = environment.id + join w_node on w_node.workflow_id = workflow.id + join w_node_context on w_node_context.node_id = w_node.id + join environment on w_node_context.environment_id = environment.id where project.projectkey = $1 and environment.name = $2 and workflow.to_delete = false order by workflow.name asc` @@ -549,29 +550,6 @@ func load(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk res.HookModels = map[int64]sdk.WorkflowHookModel{} res.OutGoingHookModels = map[int64]sdk.WorkflowHookModel{} - if !opts.WithoutNode { - _, next = observability.Span(ctx, "workflow.load.loadNodes") - err := loadWorkflowRoot(ctx, db, store, proj, &res, u, opts) - next() - - if err != nil { - return nil, sdk.WrapError(err, "Unable to load workflow root") - } - - // Load joins - if !opts.OnlyRootNode { - _, next = observability.Span(ctx, "workflow.load.loadJoins") - joins, errJ := loadJoins(ctx, db, store, proj, &res, u, opts) - next() - - if errJ != nil { - return nil, sdk.WrapError(errJ, "Load> Unable to load workflow joins") - } - res.Joins = joins - } - - } - if opts.WithFavorites { _, next = observability.Span(ctx, "workflow.load.loadFavorite") fav, errF := loadFavorite(db, &res, u) @@ -618,27 +596,9 @@ func load(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk log.Debug("Load> Load workflow (%s/%s)%d took %.3f seconds", res.ProjectKey, res.Name, res.ID, delta) w := &res - if !opts.WithoutNode { - _, next = observability.Span(ctx, "workflow.load.Sort") - Sort(w) - next() - } return w, nil } -func loadWorkflowRoot(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, u *sdk.User, opts LoadOptions) error { - var err error - w.Root, err = loadNode(ctx, db, store, proj, w, w.RootID, u, opts) - if err != nil { - if sdk.ErrorIs(err, sdk.ErrWorkflowNodeNotFound) { - log.Debug("Load> Unable to load root %d for workflow %d", w.RootID, w.ID) - return nil - } - return sdk.WrapError(err, "Unable to load workflow root %d", w.RootID) - } - return nil -} - func loadFavorite(db gorp.SqlExecutor, w *sdk.Workflow, u *sdk.User) (bool, error) { count, err := db.SelectInt("SELECT COUNT(1) FROM workflow_favorite WHERE user_id = $1 AND workflow_id = $2", u.ID, w.ID) if err != nil { @@ -653,6 +613,13 @@ func Insert(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, p *sdk.Proj return sdk.WrapError(err, "Unable to validate workflow") } + if w.WorkflowData.Node.Context != nil && w.WorkflowData.Node.Context.ApplicationID != 0 { + var err error + if w.WorkflowData.Node.Context.DefaultPayload, err = DefaultPayload(context.TODO(), db, store, p, w); err != nil { + log.Warning("postWorkflowHandler> Cannot set default payload : %v", err) + } + } + if w.HistoryLength == 0 { w.HistoryLength = sdk.DefaultHistoryLength } @@ -689,16 +656,7 @@ func Insert(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, p *sdk.Proj } } - if w.Root == nil { - return sdk.WrapError(sdk.ErrWorkflowInvalidRoot, "Root node is not here") - } - - if errIN := insertNode(db, store, w, w.Root, u, false); errIN != nil { - return sdk.WrapError(errIN, "Unable to insert workflow root node") - } - w.RootID = w.Root.ID - - if w.Root.IsLinkedToRepo() { + if w.WorkflowData.Node.IsLinkedToRepo(w) { if w.Metadata == nil { w.Metadata = sdk.Metadata{} } @@ -714,34 +672,17 @@ func Insert(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, p *sdk.Proj } if err := UpdateMetadata(db, w.ID, w.Metadata); err != nil { - return sdk.WrapError(err, "Unable to insert workflow metadata (%#v, %d)", w.Root, w.ID) + return err } } - if _, err := db.Exec("UPDATE workflow SET root_node_id = $2 WHERE id = $1", w.ID, w.Root.ID); err != nil { - return sdk.WrapError(err, "Unable to insert workflow (%#v, %d)", w.Root, w.ID) - } - - for i := range w.Joins { - j := &w.Joins[i] - if err := insertJoin(db, store, w, j, u); err != nil { - return sdk.WrapError(err, "Unable to insert update workflow(%d) join (%#v)", w.ID, j) + // Manage new hooks + if len(w.WorkflowData.Node.Hooks) > 0 { + if err := hookRegistration(context.TODO(), db, store, p, w, nil); err != nil { + return err } } - // TODO Delete in last migration step - hooks := w.GetHooks() - w.WorkflowData.Node.Hooks = make([]sdk.NodeHook, 0, len(hooks)) - for _, h := range hooks { - w.WorkflowData.Node.Hooks = append(w.WorkflowData.Node.Hooks, sdk.NodeHook{ - Ref: h.Ref, - HookModelID: h.WorkflowHookModelID, - Config: h.Config, - UUID: h.UUID, - HookModelName: h.WorkflowHookModel.Name, - }) - } - if err := InsertWorkflowData(db, w); err != nil { return sdk.WrapError(err, "Insert> Unable to insert Workflow Data") } @@ -759,8 +700,6 @@ func Insert(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, p *sdk.Proj return sdk.WrapError(err, "Insert> Unable to create workflow data") } - event.PublishWorkflowAdd(p.Key, *w, u) - return nil } @@ -930,56 +869,25 @@ func RenameNode(db gorp.SqlExecutor, w *sdk.Workflow) error { } // Update updates a workflow -func Update(ctx context.Context, db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, oldWorkflow *sdk.Workflow, p *sdk.Project, u *sdk.User) error { +func Update(ctx context.Context, db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, p *sdk.Project, u *sdk.User, uptOption UpdateOptions) error { ctx, end := observability.Span(ctx, "workflow.Update") defer end() if err := IsValid(ctx, store, db, w, p, u); err != nil { return err } - // Delete all OLD JOIN - for _, j := range oldWorkflow.Joins { - if err := deleteJoin(db, j); err != nil { - return sdk.WrapError(err, "unable to delete all joins on workflow(%d - %s)", w.ID, w.Name) - } - } - - if err := DeleteNotifications(db, oldWorkflow.ID); err != nil { + if err := DeleteNotifications(db, w.ID); err != nil { return sdk.WrapError(err, "unable to delete all notifications on workflow(%d - %s)", w.ID, w.Name) } - // Delete old Root Node - if oldWorkflow.Root != nil { - if _, err := db.Exec("update workflow set root_node_id = null where id = $1", w.ID); err != nil { - return sdk.WrapError(err, "Unable to detach workflow root") - } - - if err := deleteNode(db, oldWorkflow, oldWorkflow.Root); err != nil { - return sdk.WrapError(err, "unable to delete root node on workflow(%d - %s)", w.ID, w.Name) - } - } - // Delete workflow data - if err := DeleteWorkflowData(db, *oldWorkflow); err != nil { + if err := DeleteWorkflowData(db, *w); err != nil { return sdk.WrapError(err, "Update> unable to delete workflow data(%d - %s)", w.ID, w.Name) } // Delete all node ID w.ResetIDs() - if err := insertNode(db, store, w, w.Root, u, false); err != nil { - return sdk.WrapError(err, "unable to update root node on workflow(%d - %s)", w.ID, w.Name) - } - w.RootID = w.Root.ID - - // Insert new JOIN - for i := range w.Joins { - j := &w.Joins[i] - if err := insertJoin(db, store, w, j, u); err != nil { - return sdk.WrapError(err, "Unable to update workflow(%d) join (%#v)", w.ID, j) - } - } - filteredPurgeTags := []string{} for _, t := range w.PurgeTags { if t != "" { @@ -988,21 +896,24 @@ func Update(ctx context.Context, db gorp.SqlExecutor, store cache.Store, w *sdk. } w.PurgeTags = filteredPurgeTags - if w.Icon == "" { - w.Icon = oldWorkflow.Icon + if w.WorkflowData.Node.Context != nil && w.WorkflowData.Node.Context.ApplicationID != 0 { + var err error + if w.WorkflowData.Node.Context.DefaultPayload, err = DefaultPayload(ctx, db, store, p, w); err != nil { + log.Warning("putWorkflowHandler> Cannot set default payload : %v", err) + } } - // TODO: DELETE in step 3: Synchronize HOOK datas - hooks := w.GetHooks() - w.WorkflowData.Node.Hooks = make([]sdk.NodeHook, 0, len(hooks)) - for _, h := range hooks { - w.WorkflowData.Node.Hooks = append(w.WorkflowData.Node.Hooks, sdk.NodeHook{ - Ref: h.Ref, - HookModelID: h.WorkflowHookModelID, - Config: h.Config, - UUID: h.UUID, - HookModelName: h.WorkflowHookModel.Name, - }) + if !uptOption.DisableHookManagement { + if err := hookRegistration(ctx, db, store, p, w, uptOption.OldWorkflow); err != nil { + return err + } + if uptOption.OldWorkflow != nil { + hookToDelete := computeHookToDelete(w, uptOption.OldWorkflow) + if err := hookUnregistration(ctx, db, store, p, hookToDelete); err != nil { + return err + } + } + } if err := InsertWorkflowData(db, w); err != nil { @@ -1024,7 +935,6 @@ func Update(ctx context.Context, db gorp.SqlExecutor, store cache.Store, w *sdk. } *w = sdk.Workflow(dbw) - event.PublishWorkflowUpdate(p.Key, *w, *oldWorkflow, u) return nil } @@ -1040,29 +950,11 @@ func MarkAsDelete(db gorp.SqlExecutor, w *sdk.Workflow) error { func Delete(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, w *sdk.Workflow) error { log.Debug("Delete> deleting workflow %d", w.ID) - //Detach root from workflow - if _, err := db.Exec("update workflow set root_node_id = null where id = $1", w.ID); err != nil { - return sdk.WrapError(err, "Unable to detach workflow root") - } - - hooks := w.GetHooks() // Delete all hooks - if err := DeleteHookConfiguration(ctx, db, store, p, hooks); err != nil { + if err := hookUnregistration(ctx, db, store, p, w.WorkflowData.GetHooks()); err != nil { return sdk.WrapError(err, "Unable to delete hooks from workflow") } - // Delete all JOINs - for _, j := range w.Joins { - if err := deleteJoin(db, j); err != nil { - return sdk.WrapError(err, "unable to delete all join on workflow(%d - %s)", w.ID, w.Name) - } - } - - //Delete root - if err := deleteNode(db, w, w.Root); err != nil { - return sdk.WrapError(err, "Unable to delete workflow root") - } - if err := DeleteWorkflowData(db, *w); err != nil { return sdk.WrapError(err, "Delete> Unable to delete workflow data") } @@ -1098,19 +990,9 @@ func IsValid(ctx context.Context, store cache.Store, db gorp.SqlExecutor, w *sdk return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Invalid workflow name. It should match %s", sdk.NamePattern)) } - //Check duplicate refs - refs := w.References() - for i, ref1 := range refs { - for j, ref2 := range refs { - if ref1 == ref2 && i != j { - return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Duplicate reference %s", ref1)) - } - } - } - //Check refs - for _, j := range w.Joins { - if len(j.SourceNodeRefs) == 0 { + for _, j := range w.WorkflowData.Joins { + if len(j.JoinContext) == 0 { return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Source node references is mandatory")) } } @@ -1134,77 +1016,6 @@ func IsValid(ctx context.Context, store cache.Store, db gorp.SqlExecutor, w *sdk w.OutGoingHookModels = make(map[int64]sdk.WorkflowHookModel) } - if w.WorkflowData == nil { - //Checks application are in the current project - apps := w.InvolvedApplications() - for _, appID := range apps { - var found bool - for _, a := range proj.Applications { - if appID == a.ID { - found = true - break - } - } - if !found { - return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Unknown application %d", appID)) - } - } - - //Checks pipelines are in the current project - pips := w.InvolvedPipelines() - for _, pipID := range pips { - var found bool - for _, p := range proj.Pipelines { - if pipID == p.ID { - found = true - break - } - } - if !found { - return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Unknown pipeline %d", pipID)) - } - } - - //Checks environments are in the current project - envs := w.InvolvedEnvironments() - for _, envID := range envs { - var found bool - for _, e := range proj.Environments { - if envID == e.ID { - found = true - break - } - } - if !found { - return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Unknown environments %d", envID)) - } - } - - //Checks integrations are in the current project - pfs := w.InvolvedIntegrations() - for _, id := range pfs { - var found bool - for _, p := range proj.Integrations { - if id == p.ID { - found = true - break - } - } - if !found { - return sdk.NewError(sdk.ErrWorkflowInvalid, fmt.Errorf("Unknown integrations %d", id)) - } - } - - //Check contexts - nodes := w.Nodes(true) - for _, n := range nodes { - if err := n.CheckApplicationDeploymentStrategies(proj); err != nil { - return sdk.NewError(sdk.ErrWorkflowInvalid, err) - } - } - return nil - } - if w.WorkflowData.Node.Context != nil && w.WorkflowData.Node.Context.DefaultPayload != nil { defaultPayload, err := w.WorkflowData.Node.Context.DefaultPayloadToMap() if err != nil { @@ -1357,16 +1168,6 @@ func checkEnvironment(db gorp.SqlExecutor, proj *sdk.Project, w *sdk.Workflow, n if n.Context.EnvironmentID != 0 { env, ok := w.Environments[n.Context.EnvironmentID] if !ok { - found := false - for _, e := range proj.Environments { - if e.ID == n.Context.EnvironmentID { - found = true - break - } - } - if !found { - return sdk.WithStack(sdk.ErrNoEnvironment) - } // Load environment from db to get stage/jobs envDB, err := environment.LoadEnvironmentByID(db, n.Context.EnvironmentID) @@ -1374,6 +1175,11 @@ func checkEnvironment(db gorp.SqlExecutor, proj *sdk.Project, w *sdk.Workflow, n return sdk.WrapError(err, "unable to load environment %d", n.Context.EnvironmentID) } env = *envDB + + if env.ProjectID != proj.ID { + return sdk.NewErrorFrom(sdk.ErrResourceNotInProject, "can not found a environment with id %d", n.Context.EnvironmentID) + } + w.Environments[n.Context.EnvironmentID] = env } n.Context.EnvironmentName = env.Name @@ -1395,17 +1201,15 @@ func checkApplication(store cache.Store, db gorp.SqlExecutor, proj *sdk.Project, if n.Context.ApplicationID != 0 { app, ok := w.Applications[n.Context.ApplicationID] if !ok { - found := false - for _, a := range proj.Applications { - if a.ID == n.Context.ApplicationID { - app = a - found = true - break - } + appDB, errA := application.LoadByID(db, store, n.Context.ApplicationID, application.LoadOptions.WithDeploymentStrategies, application.LoadOptions.WithVariables) + if errA != nil { + return errA } - if !found { - return sdk.WithStack(sdk.ErrApplicationNotFound) + app = *appDB + if app.ProjectKey != proj.Key { + return sdk.NewErrorFrom(sdk.ErrResourceNotInProject, "can not found a application with id %d", n.Context.ApplicationID) } + w.Applications[n.Context.ApplicationID] = app } n.Context.ApplicationName = app.Name @@ -1427,23 +1231,17 @@ func checkPipeline(ctx context.Context, db gorp.SqlExecutor, proj *sdk.Project, if n.Context.PipelineID != 0 { pip, ok := w.Pipelines[n.Context.PipelineID] if !ok { - found := false - for _, p := range proj.Pipelines { - if p.ID == n.Context.PipelineID { - found = true - break - } - } - if !found { - return sdk.NewErrorFrom(sdk.ErrPipelineNotFound, "Can not found a pipeline with id %d", n.Context.PipelineID) - } - // Load pipeline from db to get stage/jobs pipDB, err := pipeline.LoadPipelineByID(ctx, db, n.Context.PipelineID, true) if err != nil { return sdk.WrapError(err, "unable to load pipeline %d", n.Context.PipelineID) } pip = *pipDB + + if pip.ProjectKey != proj.Key { + return sdk.NewErrorFrom(sdk.ErrResourceNotInProject, "can not found a pipeline with id %d", n.Context.PipelineID) + } + w.Pipelines[n.Context.PipelineID] = pip } n.Context.PipelineName = pip.Name @@ -1489,23 +1287,6 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj } } - // TODO HOOK Config is compute on old struct - TO DELETE WITH OLD STRUCT - if oldWf != nil { - hookLoop: - for _, oldH := range oldWf.Root.Hooks { - for i := range oldWf.WorkflowData.Node.Hooks { - h := &oldWf.WorkflowData.Node.Hooks[i] - if oldH.UUID == h.UUID { - h.Config = make(map[string]sdk.WorkflowNodeHookConfigValue, len(oldH.Config)) - for k, v := range oldH.Config { - h.Config[k] = v - } - continue hookLoop - } - } - } - } - // if a old workflow as code exists, we want to check if the new workflow is also as code on the same repository if oldWf != nil && oldWf.FromRepository != "" && (opts == nil || opts.FromRepository != oldWf.FromRepository) { return nil, nil, sdk.WithStack(sdk.ErrWorkflowAlreadyAsCode) @@ -1606,10 +1387,11 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj // If the workflow is "as-code", it should always be linked to a git repository if opts != nil && opts.FromRepository != "" { - if wf.Root.Context.Application == nil { + if wf.WorkflowData.Node.Context.ApplicationID == 0 { return nil, nil, sdk.WithStack(sdk.ErrApplicationMandatoryOnWorkflowAsCode) } - if wf.Root.Context.Application.VCSServer == "" || wf.Root.Context.Application.RepositoryFullname == "" { + app := wf.Applications[wf.WorkflowData.Node.Context.ApplicationID] + if app.VCSServer == "" || app.RepositoryFullname == "" { return nil, nil, sdk.WithStack(sdk.ErrApplicationMandatoryOnWorkflowAsCode) } } @@ -1623,6 +1405,13 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj if err := tx.Commit(); err != nil { return nil, nil, sdk.WrapError(err, "Cannot commit transaction") } + + if oldWf != nil { + event.PublishWorkflowUpdate(proj.Key, *wf, *oldWf, u) + } else { + event.PublishWorkflowAdd(proj.Key, *wf, u) + } + log.Debug("workflow %s updated", wf.Name) } diff --git a/engine/api/workflow/dao_data.go b/engine/api/workflow/dao_data.go index 8d8bdc0400..e5e2f11e80 100644 --- a/engine/api/workflow/dao_data.go +++ b/engine/api/workflow/dao_data.go @@ -8,6 +8,14 @@ import ( "github.com/ovh/cds/sdk/log" ) +// CountPipeline Count the number of workflow that use the given pipeline +func CountPipeline(db gorp.SqlExecutor, pipelineID int64) (bool, error) { + query := `SELECT count(1) FROM w_node_context WHERE pipeline_id= $1` + nbWorkfow := -1 + err := db.QueryRow(query, pipelineID).Scan(&nbWorkfow) + return nbWorkfow != 0, err +} + // DeleteWorkflowData delete the relation representation of the workflow func DeleteWorkflowData(db gorp.SqlExecutor, w sdk.Workflow) error { if w.WorkflowData == nil { diff --git a/engine/api/workflow/dao_data_context.go b/engine/api/workflow/dao_data_context.go index 66ff3fa32f..c6ac1f8fc6 100644 --- a/engine/api/workflow/dao_data_context.go +++ b/engine/api/workflow/dao_data_context.go @@ -49,6 +49,12 @@ func insertNodeContextData(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.Node) er return sdk.WrapError(errDP, "insertNodeContextData> Cannot stringify default payload") } + for _, cond := range n.Context.Conditions.PlainConditions { + if _, ok := sdk.WorkflowConditionsOperators[cond.Operator]; !ok { + return sdk.ErrWorkflowConditionBadOperator + } + } + var errC error tempContext.Conditions, errC = gorpmapping.JSONToNullString(n.Context.Conditions) if errC != nil { diff --git a/engine/api/workflow/dao_data_hook.go b/engine/api/workflow/dao_data_hook.go index fd5f381b0f..2d971b2af6 100644 --- a/engine/api/workflow/dao_data_hook.go +++ b/engine/api/workflow/dao_data_hook.go @@ -1,7 +1,8 @@ package workflow import ( - "fmt" + "database/sql" + "encoding/json" "github.com/go-gorp/gorp" @@ -9,61 +10,99 @@ import ( "github.com/ovh/cds/sdk" ) -func insertNodeHookData(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.Node) error { - if n.Hooks == nil || len(n.Hooks) == 0 { - return nil +// CountHooksByApplication count hooks by application id +func CountHooksByApplication(db gorp.SqlExecutor, appID int64) (int64, error) { + query := ` + SELECT count(w_node_hook.*) + FROM w_node_hook + JOIN w_node_context ON w_node_context.node_id = w_node_hook.node_id + WHERE w_node_context.application_id = $1; + ` + count, err := db.SelectInt(query, appID) + if err != nil { + return 0, sdk.WithStack(err) } - hookToKeep := make([]sdk.NodeHook, 0) - for i := range n.Hooks { - h := &n.Hooks[i] - h.NodeID = n.ID + return count, nil +} - model := w.HookModels[h.HookModelID] - if model.Name == sdk.RepositoryWebHookModelName && n.Context.ApplicationID == 0 { - continue +// LoadHookByUUID load a hook by his uuid +func LoadHookByUUID(db gorp.SqlExecutor, uuid string) (sdk.NodeHook, error) { + var hook sdk.NodeHook + var res dbNodeHookData + if err := db.SelectOne(&res, "select * from w_node_hook where uuid = $1", uuid); err != nil { + if err == sql.ErrNoRows { + return hook, sdk.WithStack(sdk.ErrNotFound) } + return hook, sdk.WithStack(err) + } - //Configure the hook - h.Config[sdk.HookConfigProject] = sdk.WorkflowNodeHookConfigValue{ - Value: w.ProjectKey, - Configurable: false, - } + model, err := LoadHookModelByID(db, res.HookModelID) + if err != nil { + return hook, sdk.WithStack(err) + } + res.HookModelName = model.Name + return sdk.NodeHook(res), nil +} - h.Config[sdk.HookConfigWorkflow] = sdk.WorkflowNodeHookConfigValue{ - Value: w.Name, - Configurable: false, - } +// LoadAllHooks returns all hooks +func LoadAllHooks(db gorp.SqlExecutor) ([]sdk.NodeHook, error) { + models, err := LoadHookModels(db) + if err != nil { + return nil, err + } - h.Config[sdk.HookConfigWorkflowID] = sdk.WorkflowNodeHookConfigValue{ - Value: fmt.Sprint(w.ID), - Configurable: false, + var res []dbNodeHookData + if _, err := db.Select(&res, "select * from w_node_hook"); err != nil { + if err == sql.ErrNoRows { + return nil, nil } + return nil, sdk.WithStack(err) + } - if model.Name == sdk.RepositoryWebHookModelName || model.Name == sdk.GitPollerModelName || model.Name == sdk.GerritHookModelName { - if n.Context.ApplicationID == 0 || w.Applications[n.Context.ApplicationID].RepositoryFullname == "" || w.Applications[n.Context.ApplicationID].VCSServer == "" { - return sdk.NewErrorFrom(sdk.ErrForbidden, "cannot create a git poller or repository webhook on an application without a repository") - } - h.Config[sdk.HookConfigVCSServer] = sdk.WorkflowNodeHookConfigValue{ - Value: w.Applications[n.Context.ApplicationID].VCSServer, - Configurable: false, - } - h.Config[sdk.HookConfigRepoFullName] = sdk.WorkflowNodeHookConfigValue{ - Value: w.Applications[n.Context.ApplicationID].RepositoryFullname, - Configurable: false, + nodes := make([]sdk.NodeHook, 0, len(res)) + for i := range res { + if err := res[i].PostGet(db); err != nil { + return nil, err + } + for _, m := range models { + if res[i].HookModelID == m.ID { + res[i].HookModelName = m.Name + break } } + nodes = append(nodes, sdk.NodeHook(res[i])) + } + + return nodes, nil +} + +func (h *dbNodeHookData) PostGet(db gorp.SqlExecutor) error { + var configS string + if err := db.SelectOne(&configS, "select config from w_node_hook where id = $1", h.ID); err != nil { + return sdk.WithStack(err) + } + if err := json.Unmarshal([]byte(configS), &h.Config); err != nil { + return sdk.WithStack(err) + } + return nil +} + +func insertNodeHookData(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.Node) error { + if n.Hooks == nil || len(n.Hooks) == 0 { + return nil + } + + for i := range n.Hooks { + h := &n.Hooks[i] + h.NodeID = n.ID dbHook := dbNodeHookData(*h) if err := db.Insert(&dbHook); err != nil { return sdk.WrapError(err, "insertNodeHookData> Unable to insert workflow node hook") } h.ID = dbHook.ID - - hookToKeep = append(hookToKeep, *h) } - - n.Hooks = hookToKeep return nil } diff --git a/engine/api/workflow/dao_fork.go b/engine/api/workflow/dao_fork.go deleted file mode 100644 index d0d0e364e9..0000000000 --- a/engine/api/workflow/dao_fork.go +++ /dev/null @@ -1,134 +0,0 @@ -package workflow - -import ( - "context" - "database/sql" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/cache" - "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" -) - -func insertFork(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, node *sdk.WorkflowNode, fork *sdk.WorkflowNodeFork, u *sdk.User) error { - fork.WorkflowNodeID = node.ID - - dbFork := dbNodeFork(*fork) - if err := db.Insert(&dbFork); err != nil { - return sdk.WrapError(err, "Unable to insert fork") - } - *fork = sdk.WorkflowNodeFork(dbFork) - - //Setup destination triggers - for i := range fork.Triggers { - t := &fork.Triggers[i] - if errJT := insertForkTrigger(db, store, w, fork, t, u); errJT != nil { - return sdk.WrapError(errJT, "insertFork> Unable to insert or update trigger") - } - } - - return nil -} - -func insertForkTrigger(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, fork *sdk.WorkflowNodeFork, trigger *sdk.WorkflowNodeForkTrigger, u *sdk.User) error { - trigger.WorkflowForkID = fork.ID - trigger.ID = 0 - trigger.WorkflowDestNodeID = 0 - - //Setup destination node - if err := insertNode(db, store, w, &trigger.WorkflowDestNode, u, false); err != nil { - return sdk.WrapError(err, "Unable to setup destination node %d on trigger %d", trigger.WorkflowDestNode.ID, trigger.ID) - } - trigger.WorkflowDestNodeID = trigger.WorkflowDestNode.ID - - //Insert trigger - dbt := dbNodeForkTrigger(*trigger) - if err := db.Insert(&dbt); err != nil { - return sdk.WrapError(err, "Unable to insert trigger") - } - trigger.ID = dbt.ID - trigger.WorkflowDestNode.TriggerSrcForkID = trigger.ID - - // Update node trigger ID - if err := updateWorkflowForkTriggerSrc(db, &trigger.WorkflowDestNode); err != nil { - return sdk.WrapError(err, "Unable to update node %d for trigger %d", trigger.WorkflowDestNode.ID, trigger.ID) - } - - return nil -} - -func updateWorkflowForkTriggerSrc(db gorp.SqlExecutor, n *sdk.WorkflowNode) error { - //Update node - query := "UPDATE workflow_node SET workflow_fork_trigger_src_id = $1 WHERE id = $2" - if _, err := db.Exec(query, n.TriggerSrcForkID, n.ID); err != nil { - return sdk.WrapError(err, "Unable to set workflow_fork_trigger_src_id ON node %d", n.ID) - } - return nil -} - -func loadForks(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, node *sdk.WorkflowNode, u *sdk.User, opts LoadOptions) ([]sdk.WorkflowNodeFork, error) { - res := []dbNodeFork{} - if _, err := db.Select(&res, "select * FROM workflow_node_fork where workflow_node_id = $1", node.ID); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "loadForks") - } - - forks := make([]sdk.WorkflowNodeFork, len(res)) - for i := range res { - res[i].WorkflowNodeID = node.ID - forks[i] = sdk.WorkflowNodeFork(res[i]) - - //Select triggers id - var triggerIDs []int64 - if _, err := db.Select(&triggerIDs, "select id from workflow_node_fork_trigger where workflow_node_fork_id = $1", forks[i].ID); err != nil { - if err == sql.ErrNoRows { - continue - } - return nil, sdk.WrapError(err, "Unable to load for triggers id for hook %d", forks[i].ID) - } - - //Load triggers - for _, t := range triggerIDs { - jt, err := loadForkTrigger(ctx, db, store, proj, w, t, u, opts) - if err != nil { - if sdk.Cause(err) == sql.ErrNoRows { - log.Warning("loadForks> trigger %d not found", t) - continue - } - return nil, sdk.WrapError(err, "Unable to load hook trigger %d", t) - } - forks[i].Triggers = append(forks[i].Triggers, jt) - } - } - - return forks, nil -} - -func loadForkTrigger(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, id int64, u *sdk.User, opts LoadOptions) (sdk.WorkflowNodeForkTrigger, error) { - var t sdk.WorkflowNodeForkTrigger - - dbtrigger := dbNodeForkTrigger{} - //Load the trigger - if err := db.SelectOne(&dbtrigger, "select * from workflow_node_fork_trigger where id = $1", id); err != nil { - if err == sql.ErrNoRows { - return t, nil - } - return t, sdk.WrapError(err, "Unable to load trigger %d", id) - } - - t = sdk.WorkflowNodeForkTrigger(dbtrigger) - //Load node destination - if t.WorkflowDestNodeID != 0 { - dest, err := loadNode(ctx, db, store, proj, w, t.WorkflowDestNodeID, u, opts) - if err != nil { - return t, sdk.WrapError(err, "Unable to load destination node %d", t.WorkflowDestNodeID) - } - t.WorkflowDestNode = *dest - t.WorkflowDestNode.TriggerSrcForkID = t.ID - } - - return t, nil -} diff --git a/engine/api/workflow/dao_hook.go b/engine/api/workflow/dao_hook.go deleted file mode 100644 index dcaf6ac394..0000000000 --- a/engine/api/workflow/dao_hook.go +++ /dev/null @@ -1,415 +0,0 @@ -package workflow - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/cache" - "github.com/ovh/cds/engine/api/database/gorpmapping" - "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" -) - -// UpdateHook Update a workflow node hook -func UpdateHook(db gorp.SqlExecutor, h *sdk.WorkflowNodeHook) error { - dbhook := NodeHook(*h) - if _, err := db.Update(&dbhook); err != nil { - return sdk.WrapError(err, "Cannot update hook") - } - if err := dbhook.PostInsert(db); err != nil { - return sdk.WrapError(err, "Cannot post update hook") - } - return nil -} - -// DeleteHook Delete a workflow node hook -func DeleteHook(db gorp.SqlExecutor, h *sdk.WorkflowNodeHook) error { - dbhook := NodeHook(*h) - if _, err := db.Delete(&dbhook); err != nil { - return sdk.WrapError(err, "Cannot update hook") - } - return nil -} - -// insertHook inserts a hook -func insertHook(db gorp.SqlExecutor, node *sdk.WorkflowNode, hook *sdk.WorkflowNodeHook) error { - hook.WorkflowNodeID = node.ID - if hook.WorkflowHookModelID == 0 { - hook.WorkflowHookModelID = hook.WorkflowHookModel.ID - } - - var icon string - if hook.WorkflowHookModelID != 0 { - model, errm := LoadHookModelByID(db, hook.WorkflowHookModelID) - if errm != nil { - return sdk.WrapError(errm, "insertHook> Unable to load model %d", hook.WorkflowHookModelID) - } - hook.WorkflowHookModel = *model - icon = hook.WorkflowHookModel.Icon - } else { - model, errm := LoadHookModelByName(db, hook.WorkflowHookModel.Name) - if errm != nil { - return sdk.WrapError(errm, "insertHook> Unable to load model %s", hook.WorkflowHookModel.Name) - } - hook.WorkflowHookModel = *model - } - hook.WorkflowHookModelID = hook.WorkflowHookModel.ID - - // Check configuration of the hook vs the model - for k := range hook.WorkflowHookModel.DefaultConfig { - if _, ok := hook.Config[k]; !ok { - // Add missing conf - hook.Config[k] = hook.WorkflowHookModel.DefaultConfig[k] - } - } - - // if it's a new hook - if hook.UUID == "" { - hook.UUID = sdk.UUID() - if hook.Ref == "" { - hook.Ref = fmt.Sprintf("%d", time.Now().Unix()) - } - - hook.Config["hookIcon"] = sdk.WorkflowNodeHookConfigValue{ - Value: icon, - Configurable: false, - } - } - - dbhook := NodeHook(*hook) - if err := db.Insert(&dbhook); err != nil { - return sdk.WrapError(err, "Unable to insert hook") - } - *hook = sdk.WorkflowNodeHook(dbhook) - return nil -} - -//PostInsert is a db hook -func (r *NodeHook) PostInsert(db gorp.SqlExecutor) error { - sConfig, errgo := gorpmapping.JSONToNullString(r.Config) - if errgo != nil { - return errgo - } - - if _, err := db.Exec("update workflow_node_hook set config = $2 where id = $1", r.ID, sConfig); err != nil { - return err - } - return nil -} - -//PostGet is a db hook -func (r *NodeHook) PostGet(db gorp.SqlExecutor) error { - configDB, err := db.SelectNullStr("select config from workflow_node_hook where id = $1", r.ID) - if err != nil { - return err - } - - var conf sdk.WorkflowNodeHookConfig - if err := gorpmapping.JSONNullString(configDB, &conf); err != nil { - return err - } - - r.Config = conf - //Load the model - model, err := LoadHookModelByID(db, r.WorkflowHookModelID) - if err != nil { - return err - } - - r.WorkflowHookModel = *model - - return nil -} - -// LoadAllHooks returns all hooks -func LoadAllHooks(db gorp.SqlExecutor) ([]sdk.WorkflowNodeHook, error) { - res := []NodeHook{} - if _, err := db.Select(&res, "select id, uuid, ref, workflow_hook_model_id, workflow_node_id from workflow_node_hook"); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "LoadAllHooks") - } - - nodes := []sdk.WorkflowNodeHook{} - for i := range res { - if err := res[i].PostGet(db); err != nil { - return nil, sdk.WrapError(err, "LoadAllHooks") - } - nodes = append(nodes, sdk.WorkflowNodeHook(res[i])) - } - - return nodes, nil -} - -func loadHooks(db gorp.SqlExecutor, w *sdk.Workflow, node *sdk.WorkflowNode) ([]sdk.WorkflowNodeHook, error) { - res := []NodeHook{} - if _, err := db.Select(&res, "select id, uuid, ref, workflow_hook_model_id, workflow_node_id from workflow_node_hook where workflow_node_id = $1", node.ID); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "loadHooks") - } - - nodes := []sdk.WorkflowNodeHook{} - for i := range res { - if err := res[i].PostGet(db); err != nil { - return nil, sdk.WrapError(err, "loadHooks") - } - res[i].WorkflowNodeID = node.ID - w.HookModels[res[i].WorkflowHookModelID] = res[i].WorkflowHookModel - nodes = append(nodes, sdk.WorkflowNodeHook(res[i])) - } - return nodes, nil -} - -func loadOutgoingHooks(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, node *sdk.WorkflowNode, u *sdk.User, opts LoadOptions) ([]sdk.WorkflowNodeOutgoingHook, error) { - res := []nodeOutgoingHook{} - if _, err := db.Select(&res, "select id, name, workflow_hook_model_id from workflow_node_outgoing_hook where workflow_node_id = $1", node.ID); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "loadOutgoingHooks") - } - - hooks := make([]sdk.WorkflowNodeOutgoingHook, len(res)) - for i := range res { - if err := res[i].PostGet(db); err != nil { - return nil, sdk.WrapError(err, "loadOutgoingHooks") - } - res[i].WorkflowNodeID = node.ID - hooks[i] = sdk.WorkflowNodeOutgoingHook(res[i]) - w.OutGoingHookModels[hooks[i].WorkflowHookModelID] = hooks[i].WorkflowHookModel - - //Select triggers id - var triggerIDs []int64 - if _, err := db.Select(&triggerIDs, "select id from workflow_node_outgoing_hook_trigger where workflow_node_outgoing_hook_id = $1", hooks[i].ID); err != nil { - if err == sql.ErrNoRows { - return nil, sdk.WrapError(err, "Unable to load hook triggers id for hook %d", hooks[i].ID) - } - return nil, sdk.WrapError(err, "Unable to load hook triggers id for hook %d", hooks[i].ID) - } - - //Load triggers - for _, t := range triggerIDs { - jt, err := loadHookTrigger(ctx, db, store, proj, w, &hooks[i], t, u, opts) - if err != nil { - if sdk.Cause(err) == sql.ErrNoRows { - log.Info("nodeOutgoingHook.PostGet> trigger %d not found", t) - continue - } - return nil, sdk.WrapError(err, "Unable to load hook trigger %d", t) - } - - hooks[i].Triggers = append(hooks[i].Triggers, jt) - } - } - - return hooks, nil -} - -// LoadHookByUUID loads a single hook -func LoadHookByUUID(db gorp.SqlExecutor, uuid string) (*sdk.WorkflowNodeHook, error) { - query := ` - SELECT id, uuid, ref, workflow_hook_model_id, workflow_node_id - FROM workflow_node_hook - WHERE uuid = $1` - - res := NodeHook{} - if err := db.SelectOne(&res, query, uuid); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WithStack(err) - } - - if err := res.PostGet(db); err != nil { - return nil, sdk.WrapError(err, "cannot load postget") - } - wNodeHook := sdk.WorkflowNodeHook(res) - - return &wNodeHook, nil -} - -// LoadHooksByNodeID loads hooks linked to a nodeID -func LoadHooksByNodeID(db gorp.SqlExecutor, nodeID int64) ([]sdk.WorkflowNodeHook, error) { - query := ` - SELECT id, uuid, ref, workflow_hook_model_id, workflow_node_id - FROM workflow_node_hook - WHERE workflow_node_id = $1` - - res := []NodeHook{} - if _, err := db.Select(&res, query, nodeID); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WithStack(err) - } - - nodeHooks := make([]sdk.WorkflowNodeHook, len(res)) - for i, nh := range res { - if err := nh.PostGet(db); err != nil { - return nil, sdk.WrapError(err, "cannot load postget") - } - nodeHooks[i] = sdk.WorkflowNodeHook(nh) - } - - return nodeHooks, nil -} - -// insertOutgoingHook inserts a outgoing hook -func insertOutgoingHook(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, node *sdk.WorkflowNode, hook *sdk.WorkflowNodeOutgoingHook, u *sdk.User) error { - hook.WorkflowNodeID = node.ID - - var icon string - if hook.WorkflowHookModelID != 0 { - model, has := w.OutGoingHookModels[hook.WorkflowHookModelID] - if !has { - modelDB, errm := LoadOutgoingHookModelByID(db, hook.WorkflowHookModelID) - if errm != nil { - return sdk.WrapError(errm, "insertHook> Unable to load model %d", hook.WorkflowHookModelID) - } - model = *modelDB - } - hook.WorkflowHookModel = model - } else { - model, errm := LoadOutgoingHookModelByName(db, hook.WorkflowHookModel.Name) - if errm != nil { - return sdk.WrapError(errm, "insertHook> Unable to load model %s", hook.WorkflowHookModel.Name) - } - hook.WorkflowHookModel = *model - icon = model.Icon - } - hook.WorkflowHookModelID = hook.WorkflowHookModel.ID - - hook.Config["hookIcon"] = sdk.WorkflowNodeHookConfigValue{ - Value: icon, - Configurable: false, - } - hook.Config[sdk.HookConfigProject] = sdk.WorkflowNodeHookConfigValue{Value: w.ProjectKey} - hook.Config[sdk.HookConfigWorkflow] = sdk.WorkflowNodeHookConfigValue{Value: w.Name} - - //Checks minimal configuration upon its model - for k := range hook.WorkflowHookModel.DefaultConfig { - if configuredValue, has := hook.Config[k]; !has { - return sdk.NewErrorFrom(sdk.ErrInvalidHookConfiguration, "hook %s invalid configuration. %s is missing", hook.Name, k) - } else if configuredValue.Value == "" { - if k == "payload" { // Workaround to handle empty payload - configuredValue.Value = "{}" - hook.Config[k] = configuredValue - } else { - return sdk.NewErrorFrom(sdk.ErrInvalidHookConfiguration, "hook %s invalid configuration. %s is missing", hook.Name, k) - } - } - } - - dbhook := nodeOutgoingHook(*hook) - if err := db.Insert(&dbhook); err != nil { - return sdk.WrapError(err, "Unable to insert hook: %+v", dbhook) - } - *hook = sdk.WorkflowNodeOutgoingHook(dbhook) - - //Setup destination triggers - for i := range hook.Triggers { - t := &hook.Triggers[i] - if errJT := insertOutgoingTrigger(db, store, w, *hook, t, u); errJT != nil { - return sdk.WrapError(errJT, "insertOutgoingHook> Unable to insert or update trigger") - } - } - - return nil -} - -// PostInsert is a db hook -func (h *nodeOutgoingHook) PostInsert(db gorp.SqlExecutor) error { - sConfig, errgo := gorpmapping.JSONToNullString(h.Config) - if errgo != nil { - return errgo - } - - if _, err := db.Exec("update workflow_node_outgoing_hook set config = $2 where id = $1", h.ID, sConfig); err != nil { - return err - } - return nil -} - -// PostGet is a db hook -func (h *nodeOutgoingHook) PostGet(db gorp.SqlExecutor) error { - resConfig, err := db.SelectNullStr("select config from workflow_node_outgoing_hook where id = $1", h.ID) - if err != nil { - return err - } - - conf := sdk.WorkflowNodeHookConfig{} - - if err := gorpmapping.JSONNullString(resConfig, &conf); err != nil { - return err - } - - h.Config = conf - //Load the model - model, err := LoadOutgoingHookModelByID(db, h.WorkflowHookModelID) - if err != nil { - return err - } - - h.WorkflowHookModel = *model - h.Ref = fmt.Sprintf("%d", h.ID) - - return nil -} - -func insertOutgoingTrigger(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, h sdk.WorkflowNodeOutgoingHook, trigger *sdk.WorkflowNodeOutgoingHookTrigger, u *sdk.User) error { - trigger.WorkflowNodeOutgoingHookID = h.ID - trigger.ID = 0 - - //Setup destination node - if errN := insertNode(db, store, w, &trigger.WorkflowDestNode, u, false); errN != nil { - return sdk.WrapError(errN, "insertOutgoingTrigger> Unable to setup destination node") - } - trigger.WorkflowDestNodeID = trigger.WorkflowDestNode.ID - - //Insert trigger - dbt := outgoingHookTrigger(*trigger) - if err := db.Insert(&dbt); err != nil { - return sdk.WrapError(err, "Unable to insert trigger") - } - trigger.ID = dbt.ID - trigger.WorkflowDestNode.TriggerHookSrcID = trigger.ID - - // Update node trigger ID - if err := updateWorkflowTriggerHookSrc(db, &trigger.WorkflowDestNode); err != nil { //FIX - return sdk.WrapError(err, "Unable to update node %d for trigger %d", trigger.WorkflowDestNode.ID, trigger.ID) - } - - return nil -} - -func loadHookTrigger(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, hook *sdk.WorkflowNodeOutgoingHook, id int64, u *sdk.User, opts LoadOptions) (sdk.WorkflowNodeOutgoingHookTrigger, error) { - var t sdk.WorkflowNodeOutgoingHookTrigger - - dbtrigger := outgoingHookTrigger{} - //Load the trigger - if err := db.SelectOne(&dbtrigger, "select * from workflow_node_outgoing_hook_trigger where workflow_node_outgoing_hook_id = $1 and id = $2", hook.ID, id); err != nil { - if err == sql.ErrNoRows { - return t, nil - } - return t, sdk.WrapError(err, "Unable to load trigger %d", id) - } - - t = sdk.WorkflowNodeOutgoingHookTrigger(dbtrigger) - //Load node destination - if t.WorkflowDestNodeID != 0 { - dest, err := loadNode(ctx, db, store, proj, w, t.WorkflowDestNodeID, u, opts) - if err != nil { - return t, sdk.WrapError(err, "Unable to load destination node %d", t.WorkflowDestNodeID) - } - t.WorkflowDestNode = *dest - } - - return t, nil -} diff --git a/engine/api/workflow/dao_join.go b/engine/api/workflow/dao_join.go deleted file mode 100644 index d5581bb0d0..0000000000 --- a/engine/api/workflow/dao_join.go +++ /dev/null @@ -1,189 +0,0 @@ -package workflow - -import ( - "context" - "database/sql" - "fmt" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/cache" - "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" -) - -func loadJoins(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, u *sdk.User, opts LoadOptions) ([]sdk.WorkflowNodeJoin, error) { - joinIDs := []int64{} - _, err := db.Select(&joinIDs, "select id from workflow_node_join where workflow_id = $1", w.ID) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "Unable to load join IDs on workflow %d", w.ID) - } - - joins := []sdk.WorkflowNodeJoin{} - for _, id := range joinIDs { - j, errJ := loadJoin(ctx, db, store, proj, w, id, u, opts) - if errJ != nil { - return nil, sdk.WrapError(errJ, "loadJoins> Unable to load join %d on workflow %d", id, w.ID) - } - joins = append(joins, *j) - } - - return joins, nil -} - -func loadJoin(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, id int64, u *sdk.User, opts LoadOptions) (*sdk.WorkflowNodeJoin, error) { - dbjoin := Join{} - //Load the join - if err := db.SelectOne(&dbjoin, "select * from workflow_node_join where id = $1 and workflow_id = $2", id, w.ID); err != nil { - return nil, sdk.WrapError(err, "Unable to load join %d", id) - } - dbjoin.WorkflowID = w.ID - - //Load sources - if _, err := db.Select(&dbjoin.SourceNodeIDs, "select workflow_node_id from workflow_node_join_source where workflow_node_join_id = $1", id); err != nil { - return nil, sdk.WrapError(err, "Unable to load join %d sources", id) - } - j := sdk.WorkflowNodeJoin(dbjoin) - - for _, id := range j.SourceNodeIDs { - j.SourceNodeRefs = append(j.SourceNodeRefs, fmt.Sprintf("%d", id)) - } - - //Select triggers id - var triggerIDs []int64 - if _, err := db.Select(&triggerIDs, "select id from workflow_node_join_trigger where workflow_node_join_id = $1", id); err != nil { - if err == sql.ErrNoRows { - return nil, sdk.WrapError(err, "Unable to load join triggers id for join %d", id) - } - return nil, sdk.WrapError(err, "Unable to load join triggers id for join %d", id) - } - - //Load trigegrs - for _, t := range triggerIDs { - jt, err := loadJoinTrigger(ctx, db, store, proj, w, &j, t, u, opts) - if err != nil { - return nil, sdk.WrapError(err, "Unable to load join trigger %d", t) - } - //If the trigger has not been found, skip it - if jt == nil { - log.Warning("workflow.loadJoin> Trigger id=%d not found bu referenced by join_id %d", t, id) - continue - } - j.Triggers = append(j.Triggers, *jt) - } - j.Ref = fmt.Sprintf("%d", j.ID) - - return &j, nil -} - -func loadJoinTrigger(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, node *sdk.WorkflowNodeJoin, id int64, u *sdk.User, opts LoadOptions) (*sdk.WorkflowNodeJoinTrigger, error) { - dbtrigger := JoinTrigger{} - //Load the trigger - if err := db.SelectOne(&dbtrigger, "select * from workflow_node_join_trigger where workflow_node_join_id = $1 and id = $2", node.ID, id); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "Unable to load trigger %d", id) - } - - t := sdk.WorkflowNodeJoinTrigger(dbtrigger) - //Load node destination - if t.WorkflowDestNodeID != 0 { - dest, err := loadNode(ctx, db, store, proj, w, t.WorkflowDestNodeID, u, opts) - if err != nil { - return nil, sdk.WrapError(err, "Unable to load destination node %d", t.WorkflowDestNodeID) - } - t.WorkflowDestNode = *dest - } - - return &t, nil -} - -func insertJoin(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, n *sdk.WorkflowNodeJoin, u *sdk.User) error { - log.Debug("insertOrUpdateJoin> %#v", n) - n.WorkflowID = w.ID - n.ID = 0 - n.SourceNodeIDs = nil - dbJoin := Join(*n) - - //Check references to sources - if len(n.SourceNodeRefs) == 0 { - return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Invalid joins references") - } - - for _, s := range n.SourceNodeRefs { - foundRef := w.GetNodeByRef(s) - if foundRef == nil { - return sdk.WrapError(sdk.ErrWorkflowNodeRef, "insertOrUpdateJoin> Invalid joins references %s", s) - } - log.Debug("insertOrUpdateJoin> Found reference %s: %d on %d", s, foundRef.ID, foundRef.PipelineID) - if foundRef.ID == 0 { - log.Debug("insertOrUpdateJoin> insert or update reference node (%s) %d on %s/%d", s, foundRef.ID, foundRef.Name, foundRef.PipelineID) - if errN := insertNode(db, store, w, foundRef, u, true); errN != nil { - return sdk.WrapError(errN, "insertOrUpdateJoin> Unable to insert or update source node %s", foundRef.Name) - } - } - n.SourceNodeIDs = append(n.SourceNodeIDs, foundRef.ID) - } - - //Insert the join - if err := db.Insert(&dbJoin); err != nil { - return sdk.WrapError(err, "Unable to insert workflow node join") - } - n.ID = dbJoin.ID - - //Setup destination triggers - for i := range n.Triggers { - t := &n.Triggers[i] - if errJT := insertJoinTrigger(db, store, w, *n, t, u); errJT != nil { - return sdk.WrapError(errJT, "insertOrUpdateJoin> Unable to insert or update join trigger") - } - } - - //Insert associations with sources - query := "insert into workflow_node_join_source(workflow_node_id, workflow_node_join_id) values ($1, $2)" - for _, source := range n.SourceNodeIDs { - if _, err := db.Exec(query, source, n.ID); err != nil { - return sdk.WrapError(err, "Unable to insert associations between node %d and join %d", source, n.ID) - } - } - - return nil -} - -func insertJoinTrigger(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, j sdk.WorkflowNodeJoin, trigger *sdk.WorkflowNodeJoinTrigger, u *sdk.User) error { - trigger.WorkflowNodeJoinID = j.ID - trigger.ID = 0 - - //Setup destination node - if errN := insertNode(db, store, w, &trigger.WorkflowDestNode, u, false); errN != nil { - return sdk.WrapError(errN, "insertOrUpdateJoinTrigger> Unable to setup destination node") - } - trigger.WorkflowDestNodeID = trigger.WorkflowDestNode.ID - - //Insert trigger - dbt := JoinTrigger(*trigger) - if err := db.Insert(&dbt); err != nil { - return sdk.WrapError(err, "Unable to insert trigger") - } - trigger.ID = dbt.ID - trigger.WorkflowDestNode.TriggerJoinSrcID = trigger.ID - - // Update node trigger ID - if err := updateWorkflowTriggerJoinSrc(db, &trigger.WorkflowDestNode); err != nil { - return sdk.WrapError(err, "Unable to update node %d for trigger %d", trigger.WorkflowDestNode.ID, trigger.ID) - } - - return nil -} - -func deleteJoin(db gorp.SqlExecutor, n sdk.WorkflowNodeJoin) error { - j := Join(n) - if _, err := db.Delete(&j); err != nil { - return sdk.WrapError(err, "Unable to delete join %d", j.ID) - } - return nil -} diff --git a/engine/api/workflow/dao_node.go b/engine/api/workflow/dao_node.go index 0a7c7596ad..5776f23a4a 100644 --- a/engine/api/workflow/dao_node.go +++ b/engine/api/workflow/dao_node.go @@ -1,577 +1,7 @@ package workflow import ( - "context" - "database/sql" - "encoding/json" - "fmt" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/application" - "github.com/ovh/cds/engine/api/cache" - "github.com/ovh/cds/engine/api/database/gorpmapping" - "github.com/ovh/cds/engine/api/environment" - "github.com/ovh/cds/engine/api/integration" - "github.com/ovh/cds/engine/api/observability" - "github.com/ovh/cds/engine/api/permission" - "github.com/ovh/cds/engine/api/pipeline" "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" ) var nodeNamePattern = sdk.NamePatternRegex - -func updateWorkflowTriggerSrc(db gorp.SqlExecutor, n *sdk.WorkflowNode) error { - //Update node - query := "UPDATE workflow_node SET workflow_trigger_src_id = $1 WHERE id = $2" - if _, err := db.Exec(query, n.TriggerSrcID, n.ID); err != nil { - return sdk.WrapError(err, "Unable to set workflow_trigger_src_id ON node %d", n.ID) - } - return nil -} - -func updateWorkflowTriggerJoinSrc(db gorp.SqlExecutor, n *sdk.WorkflowNode) error { - //Update node - query := "UPDATE workflow_node SET workflow_trigger_join_src_id = $1 WHERE id = $2" - if _, err := db.Exec(query, n.TriggerJoinSrcID, n.ID); err != nil { - return sdk.WrapError(err, "Unable to set workflow_trigger_join_src_id ON node %d", n.ID) - } - return nil -} - -func updateWorkflowTriggerHookSrc(db gorp.SqlExecutor, n *sdk.WorkflowNode) error { - //Update node - query := "UPDATE workflow_node SET workflow_outgoing_hook_trigger_id = $1 WHERE id = $2" - if _, err := db.Exec(query, n.TriggerHookSrcID, n.ID); err != nil { - return sdk.WrapError(err, "updateWorkflowTriggerHookSrc> Unable to set workflow_outgoing_hook_trigger_id ON node %d", n.ID) - } - return nil -} - -func insertNode(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, n *sdk.WorkflowNode, u *sdk.User, skipDependencies bool) error { - log.Debug("insertNode> insert or update node %s %d (%s) on %d", n.Name, n.ID, n.Ref, n.PipelineID) - - if !nodeNamePattern.MatchString(n.Name) { - return sdk.WrapError(sdk.ErrInvalidNodeNamePattern, "insertNode> node has a wrong name %s", n.Name) - } - - n.WorkflowID = w.ID - - // Init context - if n.Context == nil { - n.Context = &sdk.WorkflowNodeContext{} - } - - if n.Context.ApplicationID == 0 && n.Context.Application != nil { - n.Context.ApplicationID = n.Context.Application.ID - } - if n.Context.EnvironmentID == 0 && n.Context.Environment != nil { - n.Context.EnvironmentID = n.Context.Environment.ID - } - - if n.Context.ProjectIntegrationID == 0 && n.Context.ProjectIntegration != nil { - n.Context.ProjectIntegrationID = n.Context.ProjectIntegration.ID - } - - //Checks pipeline parameters - if len(n.Context.DefaultPipelineParameters) > 0 { - defaultPipParams := make([]sdk.Parameter, 0, len(n.Context.DefaultPipelineParameters)) - for i := range n.Context.DefaultPipelineParameters { - var paramFound bool - param := &n.Context.DefaultPipelineParameters[i] - for _, pipParam := range w.Pipelines[n.PipelineID].Parameter { - if pipParam.Name == param.Name { - param.Type = pipParam.Type - paramFound = true - } - } - - if paramFound { - defaultPipParams = append(defaultPipParams, *param) - } - } - n.Context.DefaultPipelineParameters = defaultPipParams - } - - if n.Name != w.WorkflowData.Node.Name && n.Context.DefaultPayload != nil { - defaultPayloadMap, err := n.Context.DefaultPayloadToMap() - if err == nil && len(defaultPayloadMap) > 0 { - log.Error("%v", sdk.WrapError(sdk.ErrInvalidNodeDefaultPayload, "payload on node %s", n.Name)) - // TODO: return error when all migrations are done - n.Context.DefaultPayload = nil - } - if err != nil { - n.Context.DefaultPayload = nil - } - } - - if n.ID == 0 { - //Insert new node - dbwn := Node(*n) - if err := db.Insert(&dbwn); err != nil { - return sdk.WrapError(err, "Unable to insert workflow node %s-%s", n.Name, n.Ref) - } - n.ID = dbwn.ID - } - - if w.Pipelines == nil { - w.Pipelines = map[int64]sdk.Pipeline{} - } - pip, has := w.Pipelines[n.PipelineID] - //Load the pipeline if not found - if !has { - loadedPip, err := pipeline.LoadPipelineByID(context.TODO(), db, n.PipelineID, true) - if err != nil { - return sdk.WrapError(err, "Unable to load pipeline for workflow node %s-%s", n.Name, n.Ref) - } - w.Pipelines[n.PipelineID] = *loadedPip - pip = *loadedPip - } - n.PipelineName = pip.Name - - if skipDependencies { - return nil - } - - //Insert context - n.Context.WorkflowNodeID = n.ID - if err := insertNodeContext(db, n.Context); err != nil { - return sdk.WrapError(err, "Unable to insert workflow node %d context", n.ID) - } - - if n.Context.Application == nil && n.Context.ApplicationID != 0 { - app, errA := application.LoadByID(db, store, n.Context.ApplicationID) - if errA != nil { - return sdk.WrapError(errA, "InsertOrUpdateNode> Cannot load application %d", n.Context.ApplicationID) - } - n.Context.Application = app - } - - //Insert hooks - hooksUUIDs := []string{} - for i := range n.Hooks { - h := &n.Hooks[i] - - if h.WorkflowHookModel.Name == sdk.RepositoryWebHookModelName && n.Context.ApplicationID == 0 { - // Remove repository webhook - hooksUUIDs = append(hooksUUIDs, h.UUID) - continue - } - - //Configure the hook - h.Config[sdk.HookConfigProject] = sdk.WorkflowNodeHookConfigValue{ - Value: w.ProjectKey, - Configurable: false, - } - - h.Config[sdk.HookConfigWorkflow] = sdk.WorkflowNodeHookConfigValue{ - Value: w.Name, - Configurable: false, - } - - h.Config[sdk.HookConfigWorkflowID] = sdk.WorkflowNodeHookConfigValue{ - Value: fmt.Sprint(w.ID), - Configurable: false, - } - - if h.WorkflowHookModel.Name == sdk.RepositoryWebHookModelName || h.WorkflowHookModel.Name == sdk.GitPollerModelName { - if n.Context.Application == nil || n.Context.Application.RepositoryFullname == "" || n.Context.Application.VCSServer == "" { - return sdk.WrapError(sdk.ErrForbidden, "InsertOrUpdateNode> Cannot create a git poller or repository webhook on an application without a repository") - } - h.Config["vcsServer"] = sdk.WorkflowNodeHookConfigValue{ - Value: n.Context.Application.VCSServer, - Configurable: false, - } - h.Config["repoFullName"] = sdk.WorkflowNodeHookConfigValue{ - Value: n.Context.Application.RepositoryFullname, - Configurable: false, - } - } - - //Insert the hook - if err := insertHook(db, n, h); err != nil { - return sdk.WrapError(err, "Unable to insert workflow node hook") - } - } - - // Delete hook if needed - if len(hooksUUIDs) > 0 { - hooks := []sdk.WorkflowNodeHook{} - for _, h := range n.Hooks { - found := false - for _, uuid := range hooksUUIDs { - if uuid == h.UUID { - found = true - break - } - } - if !found { - hooks = append(hooks, h) - } - } - n.Hooks = hooks - } - - //Insert triggers - for i := range n.Triggers { - log.Debug("inserting trigger") - t := &n.Triggers[i] - if errT := insertTrigger(db, store, w, n, t, u); errT != nil { - return sdk.WrapError(errT, "unable to insert workflow node trigger") - } - } - - //Insert outgoing hooks - for i := range n.OutgoingHooks { - h := &n.OutgoingHooks[i] - log.Debug("inserting outgoing hook %+v", h) - //Insert the hook - if err := insertOutgoingHook(db, store, w, n, h, u); err != nil { - return sdk.WrapError(err, "unable to insert workflow node outgoing hook") - } - } - - // Insert forks - for i := range n.Forks { - f := &n.Forks[i] - //Insert the hook - if err := insertFork(db, store, w, n, f, u); err != nil { - return sdk.WrapError(err, "unable to insert workflow node fork") - } - } - - return nil -} - -type sqlContext struct { - ID int64 `db:"id"` - WorkflowNodeID int64 `db:"workflow_node_id"` - AppID sql.NullInt64 `db:"application_id"` - EnvID sql.NullInt64 `db:"environment_id"` - ProjectIntegrationID sql.NullInt64 `db:"project_integration_id"` - DefaultPayload sql.NullString `db:"default_payload"` - DefaultPipelineParameters sql.NullString `db:"default_pipeline_parameters"` - Conditions sql.NullString `db:"conditions"` - Mutex sql.NullBool `db:"mutex"` -} - -// UpdateNodeContext updates the node context in database -func UpdateNodeContext(db gorp.SqlExecutor, c *sdk.WorkflowNodeContext) error { - var sqlContext = sqlContext{} - sqlContext.ID = c.ID - sqlContext.WorkflowNodeID = c.WorkflowNodeID - sqlContext.Mutex = sql.NullBool{Bool: c.Mutex, Valid: true} - - // Set ApplicationID in context - if c.ApplicationID != 0 { - sqlContext.AppID = sql.NullInt64{Int64: c.ApplicationID, Valid: true} - } - - // Set EnvironmentID in context - if c.EnvironmentID != 0 { - sqlContext.EnvID = sql.NullInt64{Int64: c.EnvironmentID, Valid: true} - } - - if c.ProjectIntegrationID != 0 { - sqlContext.ProjectIntegrationID = sql.NullInt64{Int64: c.ProjectIntegrationID, Valid: true} - } - - // Set DefaultPayload in context - if c.DefaultPayload != nil { - b, errM := json.Marshal(c.DefaultPayload) - if errM != nil { - return sdk.WrapError(errM, "updateNodeContext> Unable to marshall workflow node context(%d) default payload", c.ID) - } - sqlContext.DefaultPayload = sql.NullString{String: string(b), Valid: true} - } - - // Set PipelineParameters in context - if c.DefaultPipelineParameters != nil { - b, errM := json.Marshal(c.DefaultPipelineParameters) - if errM != nil { - return sdk.WrapError(errM, "updateNodeContext> Unable to marshall workflow node context(%d) default pipeline parameters", c.ID) - } - sqlContext.DefaultPipelineParameters = sql.NullString{String: string(b), Valid: true} - } - - for _, cond := range c.Conditions.PlainConditions { - if _, ok := sdk.WorkflowConditionsOperators[cond.Operator]; !ok { - return sdk.ErrWorkflowConditionBadOperator - } - } - - var errC error - sqlContext.Conditions, errC = gorpmapping.JSONToNullString(c.Conditions) - if errC != nil { - return sdk.WrapError(errC, "updateNodeContext> Unable to marshall workflow node context(%d) conditions", c.ID) - } - - if _, err := db.Update(&sqlContext); err != nil { - return sdk.WrapError(err, "Unable to update workflow node context(%d)", c.ID) - } - return nil -} - -func insertNodeContext(db gorp.SqlExecutor, c *sdk.WorkflowNodeContext) error { - if err := db.QueryRow("INSERT INTO workflow_node_context (workflow_node_id) VALUES ($1) RETURNING id", c.WorkflowNodeID).Scan(&c.ID); err != nil { - return sdk.WrapError(err, "Unable to insert workflow node context") - } - - return UpdateNodeContext(db, c) -} - -// CountPipeline Count the number of workflow that use the given pipeline -func CountPipeline(db gorp.SqlExecutor, pipelineID int64) (bool, error) { - query := `SELECT count(1) FROM workflow_node WHERE pipeline_id= $1` - nbWorkfow := -1 - err := db.QueryRow(query, pipelineID).Scan(&nbWorkfow) - return nbWorkfow != 0, err -} - -// loadNode loads a node in a workflow -func loadNode(c context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, id int64, u *sdk.User, opts LoadOptions) (*sdk.WorkflowNode, error) { - c, end := observability.Span(c, "workflow.loadNode", - observability.Tag(observability.TagWorkflow, w.Name), - observability.Tag(observability.TagProjectKey, proj.Key), - observability.Tag("with_pipeline", opts.DeepPipeline), - observability.Tag("only_root", opts.OnlyRootNode), - observability.Tag("with_base64_keys", opts.Base64Keys), - observability.Tag("without_node", opts.WithoutNode), - ) - defer end() - - dbwn := Node{} - if err := db.SelectOne(&dbwn, "select * from workflow_node where workflow_id = $1 and id = $2", w.ID, id); err != nil { - if err == sql.ErrNoRows { - return nil, sdk.ErrWorkflowNodeNotFound - } - return nil, err - } - - wn := sdk.WorkflowNode(dbwn) - wn.WorkflowID = w.ID - wn.Ref = fmt.Sprintf("%d", dbwn.ID) - - if !opts.OnlyRootNode { - //Load triggers - triggers, errTrig := loadTriggers(c, db, store, proj, w, &wn, u, opts) - if errTrig != nil { - return nil, sdk.WrapError(errTrig, "LoadNode> Unable to load triggers of %d", id) - } - wn.Triggers = triggers - - // Load outgoing hooks - ohooks, errHooks := loadOutgoingHooks(c, db, store, proj, w, &wn, u, opts) - if errHooks != nil { - return nil, sdk.WrapError(errHooks, "LoadNode> Unable to load outgoing hooks of %d", id) - } - wn.OutgoingHooks = ohooks - - // load forks - forks, errForks := loadForks(c, db, store, proj, w, &wn, u, opts) - if errForks != nil { - return nil, sdk.WrapError(errForks, "LoadNode> Unable to load forks of %d", id) - } - wn.Forks = forks - } - - //Load context - ctx, errCtx := LoadNodeContext(db, store, proj, wn.ID, u, opts) - if errCtx != nil { - return nil, sdk.WrapError(errCtx, "LoadNode> Unable to load context of %d", id) - } - wn.Context = ctx - - // Add application in maps - if w.Applications == nil { - w.Applications = map[int64]sdk.Application{} - } - if ctx.Application != nil { - w.Applications[ctx.Application.ID] = *ctx.Application - } - - // Add environment in maps - if w.Environments == nil { - w.Environments = map[int64]sdk.Environment{} - } - if ctx.Environment != nil { - w.Environments[ctx.Environment.ID] = *ctx.Environment - } - - //Load hooks - hooks, errHooks := loadHooks(db, w, &wn) - if errHooks != nil { - return nil, sdk.WrapError(errHooks, "LoadNode> Unable to load hooks of %d", id) - } - wn.Hooks = hooks - - //Load pipeline - if w.Pipelines == nil { - w.Pipelines = map[int64]sdk.Pipeline{} - } - pip, has := w.Pipelines[wn.PipelineID] - if !has { - newPip, err := pipeline.LoadPipelineByID(c, db, wn.PipelineID, opts.DeepPipeline) - if err != nil { - return nil, sdk.WrapError(err, "Unable to load pipeline of %d", id) - } - - w.Pipelines[wn.PipelineID] = *newPip - pip = *newPip - } - wn.PipelineName = pip.Name - - if wn.Name == "" { - wn.Name = pip.Name - } - - return &wn, nil -} - -// LoadNodeContextByNodeName load the context for a given node name and user -func LoadNodeContextByNodeName(db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, u *sdk.User, workflowName, nodeName string, opts LoadOptions) (*sdk.WorkflowNodeContext, error) { - dbnc := NodeContext{} - query := ` - SELECT workflow_node_context.id, workflow_node_context.workflow_node_id - FROM workflow_node_context - JOIN workflow_node ON workflow_node.id = workflow_node_context.workflow_node_id - JOIN workflow ON workflow.id = workflow_node.workflow_id - JOIN project ON workflow.project_id = project.id - WHERE workflow_node.name = $1 - AND project.id = $2 - AND workflow.name = $3 - ` - if err := db.SelectOne(&dbnc, query, nodeName, proj.ID, workflowName); err != nil { - return nil, sdk.WrapError(err, "Unable to load node context %s in workflow %s in project %s", nodeName, workflowName, proj.Name) - } - ctx := sdk.WorkflowNodeContext(dbnc) - - if err := postLoadNodeContext(db, store, proj, u, &ctx, opts); err != nil { - return nil, sdk.WrapError(err, "Unable to load node context dependencies") - } - - return &ctx, nil -} - -// LoadNodeContext load the context for a given node id and user -func LoadNodeContext(db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, nodeID int64, u *sdk.User, opts LoadOptions) (*sdk.WorkflowNodeContext, error) { - dbnc := NodeContext{} - if err := db.SelectOne(&dbnc, "select id from workflow_node_context where workflow_node_id = $1", nodeID); err != nil { - return nil, sdk.WrapError(err, "Unable to load node context %d", nodeID) - } - ctx := sdk.WorkflowNodeContext(dbnc) - ctx.WorkflowNodeID = nodeID - - if err := postLoadNodeContext(db, store, proj, u, &ctx, opts); err != nil { - return nil, sdk.WrapError(err, "Unable to load node context dependencies") - } - - return &ctx, nil -} - -func postLoadNodeContext(db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, u *sdk.User, ctx *sdk.WorkflowNodeContext, opts LoadOptions) error { - var sqlContext = sqlContext{} - if err := db.SelectOne(&sqlContext, - "select application_id, environment_id, default_payload, default_pipeline_parameters, conditions, mutex, project_integration_id from workflow_node_context where id = $1", ctx.ID); err != nil { - return err - } - if sqlContext.AppID.Valid { - ctx.ApplicationID = sqlContext.AppID.Int64 - } - if sqlContext.EnvID.Valid { - ctx.EnvironmentID = sqlContext.EnvID.Int64 - } - if sqlContext.ProjectIntegrationID.Valid { - ctx.ProjectIntegrationID = sqlContext.ProjectIntegrationID.Int64 - } - if sqlContext.Mutex.Valid { - ctx.Mutex = sqlContext.Mutex.Bool - } - - //Unmarshal payload - if err := gorpmapping.JSONNullString(sqlContext.DefaultPayload, &ctx.DefaultPayload); err != nil { - return sdk.WrapError(err, "Unable to unmarshall context %d default payload", ctx.ID) - } - - //Unmarshal pipeline parameters - if err := gorpmapping.JSONNullString(sqlContext.DefaultPipelineParameters, &ctx.DefaultPipelineParameters); err != nil { - return sdk.WrapError(err, "Unable to unmarshall context %d default pipeline parameters", ctx.ID) - } - - //Load the application in the context - if ctx.ApplicationID != 0 { - app, err := application.LoadByID(db, store, ctx.ApplicationID, application.LoadOptions.WithVariables, application.LoadOptions.WithDeploymentStrategies) - if err != nil { - return sdk.WrapError(err, "Unable to load application %d", ctx.ApplicationID) - } - if opts.Base64Keys { - if err := application.LoadAllBase64Keys(db, app); err != nil { - return sdk.WrapError(err, "Unable to load application %d base64keys", ctx.ApplicationID) - } - } else { - if err := application.LoadAllKeys(db, app); err != nil { - return sdk.WrapError(err, "Unable to load application %d keys", ctx.ApplicationID) - } - } - - ctx.Application = app - } - - //Load the env in the context - if ctx.EnvironmentID != 0 { - env, err := environment.LoadEnvironmentByID(db, ctx.EnvironmentID) - if err != nil { - return sdk.WrapError(err, "Unable to load env %d", ctx.EnvironmentID) - } - ctx.Environment = env - - if opts.Base64Keys { - if errE := environment.LoadAllBase64Keys(db, env); errE != nil { - return sdk.WrapError(errE, "postLoadNodeContext> Unable to load env %d keys", ctx.EnvironmentID) - } - } - - ctx.Environment.Permission = permission.ProjectPermission(env.ProjectKey, u) - } - - //Load the integration in the context - if ctx.ProjectIntegrationID != 0 { - if len(proj.Integrations) == 0 { - integrations, err := integration.LoadIntegrationsByProjectID(db, proj.ID, false) - if err != nil { - return sdk.WrapError(err, "Unable to load integrations for this project %d", proj.ID) - } - proj.Integrations = integrations - } - for _, pf := range proj.Integrations { - if pf.ID == ctx.ProjectIntegrationID { - ctx.ProjectIntegration = &pf - break - } - } - if ctx.ProjectIntegration == nil { - return sdk.WrapError(fmt.Errorf("unable to find integration id = %d", ctx.ProjectIntegrationID), "postLoadNodeContext") - } - } - - if err := gorpmapping.JSONNullString(sqlContext.Conditions, &ctx.Conditions); err != nil { - return sdk.WrapError(err, "Unable to unmarshall context %d conditions", ctx.ID) - } - - return nil -} - -//deleteNode deletes nodes and all its children -func deleteNode(db gorp.SqlExecutor, w *sdk.Workflow, node *sdk.WorkflowNode) error { - if node == nil { - return nil - } - log.Debug("deleteNode> Delete node %d %s", node.ID, node.Name) - - dbwn := Node(*node) - if _, err := db.Delete(&dbwn); err != nil { - return sdk.WrapError(err, "Unable to delete node %d", dbwn.ID) - } - return nil -} diff --git a/engine/api/workflow/dao_node_run.go b/engine/api/workflow/dao_node_run.go index c1c5e9a8b2..0d796ff118 100644 --- a/engine/api/workflow/dao_node_run.go +++ b/engine/api/workflow/dao_node_run.go @@ -733,17 +733,17 @@ func PreviousNodeRunVCSInfos(db gorp.SqlExecutor, projectKey string, wf *sdk.Wor queryPrevious := ` SELECT workflow_node_run.vcs_branch, workflow_node_run.vcs_tag, workflow_node_run.vcs_hash, workflow_node_run.vcs_repository, workflow_node_run.num FROM workflow_node_run - JOIN workflow_node ON workflow_node.name = workflow_node_run.workflow_node_name AND workflow_node.name = $1 AND workflow_node.workflow_id = $2 - JOIN workflow_node_context ON workflow_node_context.workflow_node_id = workflow_node.id + JOIN w_node ON w_node.name = workflow_node_run.workflow_node_name AND w_node.name = $1 AND w_node.workflow_id = $2 + JOIN w_node_context ON w_node_context.node_id = w_node.id WHERE workflow_node_run.vcs_hash IS NOT NULL AND workflow_node_run.num < $3 - AND workflow_node_context.application_id = $4 + AND w_node_context.application_id = $4 ` argPrevious := []interface{}{nodeName, wf.ID, current.BuildNumber, appID} if envID > 0 { argPrevious = append(argPrevious, envID) - queryPrevious += "AND workflow_node_context.environment_id = $5" + queryPrevious += "AND w_node_context.environment_id = $5" } queryPrevious += fmt.Sprintf(" ORDER BY workflow_node_run.num DESC LIMIT 1") diff --git a/engine/api/workflow/dao_notification.go b/engine/api/workflow/dao_notification.go index c2c77b8863..ebc298d616 100644 --- a/engine/api/workflow/dao_notification.go +++ b/engine/api/workflow/dao_notification.go @@ -3,7 +3,6 @@ package workflow import ( "database/sql" "github.com/go-gorp/gorp" - "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/ovh/cds/sdk" ) @@ -47,36 +46,15 @@ func loadNotification(db gorp.SqlExecutor, w *sdk.Workflow, id int64) (sdk.Workf dbnotif.WorkflowID = w.ID //Load sources - var ids []struct { - OldNodeID int64 `db:"workflow_node_id"` - NewNodeID sql.NullInt64 `db:"node_id"` - } - if _, err := db.Select(&ids, "select workflow_node_id, node_id from workflow_notification_source where workflow_notification_id = $1", id); err != nil { + var ids []int64 + if _, err := db.Select(&ids, "select node_id from workflow_notification_source where workflow_notification_id = $1", id); err != nil { return sdk.WorkflowNotification{}, sdk.WrapError(err, "Unable to load notification %d sources", id) } - - dbnotif.SourceNodeIDs = make([]int64, 0, len(ids)) - dbnotif.NodeIDs = make([]int64, 0, len(ids)) - for _, ID := range ids { - dbnotif.SourceNodeIDs = append(dbnotif.SourceNodeIDs, ID.OldNodeID) - if ID.NewNodeID.Valid { - i := ID.NewNodeID.Int64 - dbnotif.NodeIDs = append(dbnotif.NodeIDs, i) - } else { - oldN := w.GetNode(ID.OldNodeID) - if oldN != nil { - newN := w.WorkflowData.NodeByName(oldN.Name) - if newN != nil { - dbnotif.NodeIDs = append(dbnotif.NodeIDs, newN.ID) - } - } - } - } - + dbnotif.NodeIDs = ids n := sdk.WorkflowNotification(dbnotif) - for _, id := range n.SourceNodeIDs { - notifNode := w.GetNode(id) + for _, id := range n.NodeIDs { + notifNode := w.WorkflowData.NodeByID(id) if notifNode != nil { n.SourceNodeRefs = append(n.SourceNodeRefs, notifNode.Name) } @@ -89,7 +67,6 @@ func loadNotification(db gorp.SqlExecutor, w *sdk.Workflow, id int64) (sdk.Workf func InsertNotification(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.WorkflowNotification) error { n.WorkflowID = w.ID n.ID = 0 - n.SourceNodeIDs = nil n.NodeIDs = nil dbNotif := Notification(*n) @@ -99,13 +76,6 @@ func InsertNotification(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.WorkflowNot } for _, s := range n.SourceNodeRefs { - //Search references - var foundRef = w.GetNodeByName(s) - if foundRef == nil || foundRef.ID == 0 { - return sdk.WrapError(sdk.ErrWorkflowNotificationNodeRef, "insertNotification> Invalid notification references %s", s) - } - n.SourceNodeIDs = append(n.SourceNodeIDs, foundRef.ID) - nodeFoundRef := w.WorkflowData.NodeByName(s) if nodeFoundRef == nil || nodeFoundRef.ID == 0 { return sdk.WrapError(sdk.ErrWorkflowNotificationNodeRef, "insertNotification> Invalid notification references node %s", s) @@ -120,10 +90,10 @@ func InsertNotification(db gorp.SqlExecutor, w *sdk.Workflow, n *sdk.WorkflowNot n.ID = dbNotif.ID //Insert associations with sources - query := "insert into workflow_notification_source(workflow_node_id, workflow_notification_id, node_id) values ($1, $2, $3)" + query := "insert into workflow_notification_source(workflow_notification_id, node_id) values ($1, $2)" for i := range n.NodeIDs { - if _, err := db.Exec(query, n.SourceNodeIDs[i], n.ID, n.NodeIDs[i]); err != nil { - return sdk.WrapError(err, "Unable to insert associations between node %d/%d and notification %d", n.SourceNodeIDs[i], n.NodeIDs[i], n.ID) + if _, err := db.Exec(query, n.ID, n.NodeIDs[i]); err != nil { + return sdk.WrapError(err, "Unable to insert associations between node %d and notification %d", n.NodeIDs[i], n.ID) } } diff --git a/engine/api/workflow/dao_run.go b/engine/api/workflow/dao_run.go index 0b4bec4381..4672d4bc48 100644 --- a/engine/api/workflow/dao_run.go +++ b/engine/api/workflow/dao_run.go @@ -163,10 +163,6 @@ func (r *Run) PostGet(db gorp.SqlExecutor) error { if err := gorpmapping.JSONNullString(res.W, &w); err != nil { return sdk.WrapError(err, "Unable to unmarshal workflow") } - // TODO: to delete when old runs will be purged - for i := range w.Joins { - w.Joins[i].Ref = fmt.Sprintf("%d", w.Joins[i].ID) - } r.Workflow = w i := []sdk.WorkflowRunInfo{} @@ -464,49 +460,6 @@ func loadRunTags(db gorp.SqlExecutor, run *sdk.WorkflowRun) error { return nil } -func MigrateWorkflowRun(ctx context.Context, db gorp.SqlExecutor, run *sdk.WorkflowRun) error { - if run != nil && run.Workflow.WorkflowData == nil && run.Status != sdk.StatusPending.String() { - data := run.Workflow.Migrate(true) - run.Workflow.WorkflowData = &data - - run.Workflow.Applications = make(map[int64]sdk.Application) - run.Workflow.Environments = make(map[int64]sdk.Environment) - run.Workflow.ProjectIntegrations = make(map[int64]sdk.ProjectIntegration) - run.Workflow.HookModels = make(map[int64]sdk.WorkflowHookModel) - run.Workflow.OutGoingHookModels = make(map[int64]sdk.WorkflowHookModel) - - nodes := run.Workflow.Nodes(true) - for _, n := range nodes { - if n.Context == nil { - continue - } - if n.Context.Application != nil && n.Context.Application.ID > 0 { - run.Workflow.Applications[n.Context.Application.ID] = *n.Context.Application - } - if n.Context.Environment != nil && n.Context.Environment.ID > 0 { - run.Workflow.Environments[n.Context.Environment.ID] = *n.Context.Environment - } - if n.Context.ProjectIntegration != nil && n.Context.ProjectIntegration.ID > 0 { - run.Workflow.ProjectIntegrations[n.Context.ProjectIntegration.ID] = *n.Context.ProjectIntegration - } - for _, h := range n.Hooks { - if h.WorkflowHookModel.ID > 0 { - run.Workflow.HookModels[h.WorkflowHookModel.ID] = h.WorkflowHookModel - } - } - for _, h := range n.OutgoingHooks { - if h.WorkflowHookModel.ID > 0 { - run.Workflow.OutGoingHookModels[h.WorkflowHookModel.ID] = h.WorkflowHookModel - } - } - } - if err := UpdateWorkflowRun(ctx, db, run); err != nil { - return sdk.WrapError(err, "unable to migrate old workflow run") - } - } - return nil -} - func loadRun(db gorp.SqlExecutor, loadOpts LoadRunOptions, query string, args ...interface{}) (*sdk.WorkflowRun, error) { runDB := &Run{} if err := db.SelectOne(runDB, query, args...); err != nil { @@ -523,9 +476,6 @@ func loadRun(db gorp.SqlExecutor, loadOpts LoadRunOptions, query string, args .. } wr.Tags = tags - // Here we can ignore error - _ = MigrateWorkflowRun(context.TODO(), db, &wr) - if err := syncNodeRuns(db, &wr, loadOpts); err != nil { return nil, sdk.WrapError(err, "Unable to load workflow node run") } @@ -651,6 +601,7 @@ func CreateRun(db *gorp.DbMap, wf *sdk.Workflow, opts *sdk.WorkflowRunPostHandle Status: sdk.StatusPending.String(), LastExecution: time.Now(), Tags: make([]sdk.WorkflowRunTag, 0), + Workflow: sdk.Workflow{Name: wf.Name}, } if opts != nil && opts.Hook != nil { diff --git a/engine/api/workflow/dao_run_test.go b/engine/api/workflow/dao_run_test.go index 3d41ffb4aa..dac30c3b19 100644 --- a/engine/api/workflow/dao_run_test.go +++ b/engine/api/workflow/dao_run_test.go @@ -49,7 +49,6 @@ func TestCanBeRun(t *testing.T) { Name: "test_1", ProjectID: 1, ProjectKey: "key", - RootID: 10, WorkflowData: &sdk.WorkflowData{ Node: nodeRoot, }, @@ -110,6 +109,16 @@ func TestPurgeWorkflowRun(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + // Default payload on workflow insert + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=master": b := sdk.VCSBranch{ @@ -211,8 +220,6 @@ vcs_ssh_key: proj-blabla PurgeTags: []string{"git.branch"}, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_purge_1", u, workflow.LoadOptions{ @@ -307,8 +314,6 @@ func TestPurgeWorkflowRunWithRunningStatus(t *testing.T) { PurgeTags: []string{"git.branch"}, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_purge_1", u, workflow.LoadOptions{ @@ -485,8 +490,6 @@ vcs_ssh_key: proj-blabla PurgeTags: []string{"git.branch"}, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_purge_1", u, workflow.LoadOptions{ @@ -682,8 +685,6 @@ vcs_ssh_key: proj-blabla PurgeTags: []string{"git.branch"}, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_purge_1", u, workflow.LoadOptions{ @@ -778,7 +779,6 @@ func TestPurgeWorkflowRunWithoutTags(t *testing.T) { }, HistoryLength: 2, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_purge_1", u, workflow.LoadOptions{ @@ -872,7 +872,6 @@ func TestPurgeWorkflowRunWithoutTagsBiggerHistoryLength(t *testing.T) { }, HistoryLength: 20, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_purge_1", u, workflow.LoadOptions{ diff --git a/engine/api/workflow/dao_staticfiles_test.go b/engine/api/workflow/dao_staticfiles_test.go index e94245b3ce..2ed524ea02 100644 --- a/engine/api/workflow/dao_staticfiles_test.go +++ b/engine/api/workflow/dao_staticfiles_test.go @@ -71,8 +71,6 @@ func TestInsertStaticFiles(t *testing.T) { PurgeTags: []string{"git.branch"}, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_staticfiles_1", u, workflow.LoadOptions{ diff --git a/engine/api/workflow/dao_test.go b/engine/api/workflow/dao_test.go index 7276a1ca99..ab9dc5adf6 100644 --- a/engine/api/workflow/dao_test.go +++ b/engine/api/workflow/dao_test.go @@ -1,13 +1,23 @@ package workflow_test import ( + "bytes" "context" + "encoding/json" "fmt" - "sort" - "testing" - "github.com/fsamin/go-dump" + "github.com/ovh/cds/engine/api/bootstrap" + "github.com/ovh/cds/engine/api/repositoriesmanager" + "github.com/ovh/cds/engine/api/services" + "github.com/ovh/cds/sdk/log" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + "io/ioutil" + "net/http" + "reflect" + "sort" + "strings" + "testing" "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/environment" @@ -68,8 +78,6 @@ func TestInsertSimpleWorkflowAndExport(t *testing.T) { }, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) @@ -78,11 +86,10 @@ func TestInsertSimpleWorkflowAndExport(t *testing.T) { assert.Equal(t, w.ID, w1.ID) assert.Equal(t, w.ProjectID, w1.ProjectID) assert.Equal(t, w.Name, w1.Name) - assert.Equal(t, w.Root.PipelineID, w1.Root.PipelineID) - assert.Equal(t, w.Root.PipelineName, w1.Root.PipelineName) - assertEqualNode(t, w.Root, w1.Root) + assert.Equal(t, w.WorkflowData.Node.Context.PipelineID, w1.WorkflowData.Node.Context.PipelineID) + assertEqualNode(t, &w.WorkflowData.Node, &w1.WorkflowData.Node) - assert.False(t, w1.Root.Context.Mutex) + assert.False(t, w1.WorkflowData.Node.Context.Mutex) ws, err := workflow.LoadAll(db, proj.Key) test.NoError(t, err) @@ -186,17 +193,15 @@ func TestInsertSimpleWorkflowWithApplicationAndEnv(t *testing.T) { }, } - (&w).RetroMigrate() - test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) test.NoError(t, err) assert.Equal(t, w.ID, w1.ID) - assert.Equal(t, w.Root.Context.ApplicationID, w1.Root.Context.ApplicationID) - assert.Equal(t, w.Root.Context.EnvironmentID, w1.Root.Context.EnvironmentID) - assert.Equal(t, w.Root.Context.Mutex, w1.Root.Context.Mutex) + assert.Equal(t, w.WorkflowData.Node.Context.ApplicationID, w1.WorkflowData.Node.Context.ApplicationID) + assert.Equal(t, w.WorkflowData.Node.Context.EnvironmentID, w1.WorkflowData.Node.Context.EnvironmentID) + assert.Equal(t, w.WorkflowData.Node.Context.Mutex, w1.WorkflowData.Node.Context.Mutex) } func TestInsertComplexeWorkflowAndExport(t *testing.T) { @@ -318,7 +323,6 @@ func TestInsertComplexeWorkflowAndExport(t *testing.T) { }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) @@ -327,13 +331,12 @@ func TestInsertComplexeWorkflowAndExport(t *testing.T) { assert.Equal(t, w.ID, w1.ID) assert.Equal(t, w.ProjectID, w1.ProjectID) assert.Equal(t, w.Name, w1.Name) - assert.Equal(t, w.Root.PipelineID, w1.Root.PipelineID) - assert.Equal(t, w.Root.PipelineName, w1.Root.PipelineName) - test.Equal(t, len(w.Root.Triggers), len(w1.Root.Triggers)) + assert.Equal(t, w.WorkflowData.Node.Context.PipelineID, w1.WorkflowData.Node.Context.PipelineID) + test.Equal(t, len(w.WorkflowData.Node.Triggers), len(w1.WorkflowData.Node.Triggers)) workflow.Sort(&w) - assertEqualNode(t, w.Root, w1.Root) + assertEqualNode(t, &w.WorkflowData.Node, &w1.WorkflowData.Node) exp, err := exportentities.NewWorkflow(w) test.NoError(t, err) @@ -462,23 +465,20 @@ func TestInsertComplexeWorkflowWithBadOperator(t *testing.T) { assert.Error(t, workflow.Insert(db, cache, &w, proj, u)) } -func assertEqualNode(t *testing.T, n1, n2 *sdk.WorkflowNode) { - t.Logf("assertEqualNode : %d(%s) on %s", n2.ID, n2.Ref, n2.PipelineName) +func assertEqualNode(t *testing.T, n1, n2 *sdk.Node) { + t.Logf("assertEqualNode : %d(%s)", n2.ID, n2.Ref) workflow.SortNode(n1) workflow.SortNode(n2) t.Logf("assertEqualNode : Checking hooks") test.Equal(t, len(n1.Hooks), len(n2.Hooks)) t.Logf("assertEqualNode : Checking triggers") test.Equal(t, len(n1.Triggers), len(n2.Triggers)) - t.Logf("assertEqualNode : Checking out going hooks") - test.Equal(t, len(n1.OutgoingHooks), len(n2.OutgoingHooks)) - assert.Equal(t, n1.PipelineName, n2.PipelineName) for i, t1 := range n1.Triggers { t2 := n2.Triggers[i] - test.Equal(t, len(t1.WorkflowDestNode.Context.Conditions.PlainConditions), len(t2.WorkflowDestNode.Context.Conditions.PlainConditions), "Number of conditions on node does not match") - test.EqualValuesWithoutOrder(t, t1.WorkflowDestNode.Context.Conditions.PlainConditions, t2.WorkflowDestNode.Context.Conditions.PlainConditions, "Conditions on triggers does not match") - assertEqualNode(t, &t1.WorkflowDestNode, &t2.WorkflowDestNode) + test.Equal(t, len(t1.ChildNode.Context.Conditions.PlainConditions), len(t2.ChildNode.Context.Conditions.PlainConditions), "Number of conditions on node does not match") + test.EqualValuesWithoutOrder(t, t1.ChildNode.Context.Conditions.PlainConditions, t2.ChildNode.Context.Conditions.PlainConditions, "Conditions on triggers does not match") + assertEqualNode(t, &t1.ChildNode, &t2.ChildNode) } } func TestUpdateSimpleWorkflowWithApplicationEnvPipelineParametersAndPayload(t *testing.T) { @@ -593,22 +593,18 @@ func TestUpdateSimpleWorkflowWithApplicationEnvPipelineParametersAndPayload(t *t }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) test.NoError(t, err) - w1old, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) - test.NoError(t, err) - t.Logf("Modifying workflow... with %d instead of %d", app2.ID, app.ID) w1.Name = "test_2" w1.WorkflowData.Node.Context.PipelineID = pip2.ID w1.WorkflowData.Node.Context.ApplicationID = app2.ID - test.NoError(t, workflow.Update(context.TODO(), db, cache, w1, w1old, proj, u)) + test.NoError(t, workflow.Update(context.TODO(), db, cache, w1, proj, u, workflow.UpdateOptions{})) t.Logf("Reloading workflow...") w2, err := workflow.LoadByID(db, cache, proj, w1.ID, u, workflow.LoadOptions{}) @@ -775,7 +771,6 @@ func TestInsertComplexeWorkflowWithJoinsAndExport(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - w.RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) @@ -784,9 +779,8 @@ func TestInsertComplexeWorkflowWithJoinsAndExport(t *testing.T) { assert.Equal(t, w.ID, w1.ID) assert.Equal(t, w.ProjectID, w1.ProjectID) assert.Equal(t, w.Name, w1.Name) - assert.Equal(t, w.Root.PipelineID, w1.Root.PipelineID) - assert.Equal(t, w.Root.PipelineName, w1.Root.PipelineName) - test.Equal(t, len(w.Root.Triggers), len(w1.Root.Triggers)) + assert.Equal(t, w.WorkflowData.Node.Context.PipelineID, w1.WorkflowData.Node.Context.PipelineID) + test.Equal(t, len(w.WorkflowData.Node.Triggers), len(w1.WorkflowData.Node.Triggers)) workflow.Sort(&w) @@ -813,20 +807,25 @@ func TestInsertComplexeWorkflowWithJoinsAndExport(t *testing.T) { t.Logf("%s: %s but was undefined", k, v) } } - assertEqualNode(t, w.Root, w1.Root) + assertEqualNode(t, &w.WorkflowData.Node, &w1.WorkflowData.Node) + + assert.EqualValues(t, w.WorkflowData.Joins[0].Triggers[0].ChildNode.Context.Conditions, w1.WorkflowData.Joins[0].Triggers[0].ChildNode.Context.Conditions) + assert.Equal(t, w.WorkflowData.Joins[0].Triggers[0].ChildNode.Context.PipelineID, w1.WorkflowData.Joins[0].Triggers[0].ChildNode.Context.PipelineID) - assert.EqualValues(t, w.Joins[0].Triggers[0].WorkflowDestNode.Context.Conditions, w1.Joins[0].Triggers[0].WorkflowDestNode.Context.Conditions) - assert.Equal(t, w.Joins[0].Triggers[0].WorkflowDestNode.PipelineID, w1.Joins[0].Triggers[0].WorkflowDestNode.PipelineID) + assert.Equal(t, pip1.ID, w.WorkflowData.Node.Context.PipelineID) + assert.Equal(t, pip2.ID, w.WorkflowData.Node.Triggers[0].ChildNode.Context.PipelineID) + assert.Equal(t, pip3.ID, w.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.Context.PipelineID) + assert.Equal(t, pip4.ID, w.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.Triggers[0].ChildNode.Context.PipelineID) - assert.Equal(t, pip1.Name, w.Root.PipelineName) - assert.Equal(t, pip2.Name, w.Root.Triggers[0].WorkflowDestNode.PipelineName) - assert.Equal(t, pip3.Name, w.Root.Triggers[0].WorkflowDestNode.Triggers[0].WorkflowDestNode.PipelineName) - assert.Equal(t, pip4.Name, w.Root.Triggers[0].WorkflowDestNode.Triggers[0].WorkflowDestNode.Triggers[0].WorkflowDestNode.PipelineName) + log.Warning("%d-%d", w1.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.ID, + w1.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.Triggers[0].ChildNode.ID) + + log.Warning("%+v", w1.WorkflowData.Joins[0].JoinContext) test.EqualValuesWithoutOrder(t, []int64{ - w1.Root.Triggers[0].WorkflowDestNode.Triggers[0].WorkflowDestNode.ID, - w1.Root.Triggers[0].WorkflowDestNode.Triggers[0].WorkflowDestNode.Triggers[0].WorkflowDestNode.ID, - }, w1.Joins[0].SourceNodeIDs) - assert.Equal(t, pip5.Name, w.Joins[0].Triggers[0].WorkflowDestNode.PipelineName) + w1.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.ID, + w1.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.Triggers[0].ChildNode.ID, + }, []int64{w1.WorkflowData.Joins[0].JoinContext[0].ParentID, w1.WorkflowData.Joins[0].JoinContext[1].ParentID}) + assert.Equal(t, pip5.ID, w.WorkflowData.Joins[0].Triggers[0].ChildNode.Context.PipelineID) exp, err := exportentities.NewWorkflow(*w1) test.NoError(t, err) @@ -1073,7 +1072,6 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - w.RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) @@ -1104,7 +1102,7 @@ func TestInsertComplexeWorkflowWithComplexeJoins(t *testing.T) { t.Logf("%s: %s but was undefined", k, v) } } - assertEqualNode(t, w.Root, w1.Root) + assertEqualNode(t, &w.WorkflowData.Node, &w1.WorkflowData.Node) } func TestUpdateWorkflowWithJoins(t *testing.T) { @@ -1165,13 +1163,12 @@ func TestUpdateWorkflowWithJoins(t *testing.T) { proj, _ = project.LoadByID(db, cache, proj.ID, u, project.LoadOptions.WithApplications, project.LoadOptions.WithPipelines, project.LoadOptions.WithEnvironments, project.LoadOptions.WithGroups) test.NoError(t, workflow.RenameNode(db, &w)) - w.RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) test.NoError(t, err) - w1old := *w1 + //w1old := *w1 w1.Name = "test_2" w1.WorkflowData.Joins = []sdk.Node{ { @@ -1198,9 +1195,8 @@ func TestUpdateWorkflowWithJoins(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, w1)) - w1.RetroMigrate() - test.NoError(t, workflow.Update(context.TODO(), db, cache, w1, &w1old, proj, u)) + test.NoError(t, workflow.Update(context.TODO(), db, cache, w1, proj, u, workflow.UpdateOptions{})) t.Logf("Reloading workflow...") w2, err := workflow.LoadByID(db, cache, proj, w1.ID, u, workflow.LoadOptions{}) @@ -1249,6 +1245,63 @@ func TestInsertSimpleWorkflowWithHookAndExport(t *testing.T) { } } + mockHookSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeHooks} + test.NoError(t, services.Insert(db, mockHookSservice)) + defer func() { + services.Delete(db, mockHookSservice) // nolint + }() + + services.HTTPClient = mock( + func(r *http.Request) (*http.Response, error) { + body := new(bytes.Buffer) + w := new(http.Response) + enc := json.NewEncoder(body) + w.Body = ioutil.NopCloser(body) + + switch r.URL.String() { + // NEED get REPO + case "/task/bulk": + var hooks map[string]sdk.NodeHook + bts, err := ioutil.ReadAll(r.Body) + if err != nil { + return writeError(w, err) + } + if err := json.Unmarshal(bts, &hooks); err != nil { + return writeError(w, err) + } + k := reflect.ValueOf(hooks).MapKeys()[0].String() + + hooks[k].Config["method"] = sdk.WorkflowNodeHookConfigValue{ + Value: "POST", + Configurable: true, + } + hooks[k].Config["username"] = sdk.WorkflowNodeHookConfigValue{ + Value: "test", + Configurable: false, + } + hooks[k].Config["password"] = sdk.WorkflowNodeHookConfigValue{ + Value: "password", + Configurable: false, + } + hooks[k].Config["payload"] = sdk.WorkflowNodeHookConfigValue{ + Value: "{}", + Configurable: true, + } + hooks[k].Config["URL"] = sdk.WorkflowNodeHookConfigValue{ + Value: "https://www.github.com", + Configurable: true, + } + if err := enc.Encode(hooks); err != nil { + return writeError(w, err) + } + default: + t.Fatalf("UNKNOWN ROUTE: %s", r.URL.String()) + } + + return w, nil + }, + ) + outHookModels, err := workflow.LoadOutgoingHookModels(db) test.NoError(t, err) var outWebHookID int64 @@ -1397,7 +1450,6 @@ func TestInsertSimpleWorkflowWithHookAndExport(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u), "unable to insert workflow") w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{}) @@ -1406,9 +1458,8 @@ func TestInsertSimpleWorkflowWithHookAndExport(t *testing.T) { assert.Equal(t, w.ID, w1.ID) assert.Equal(t, w.ProjectID, w1.ProjectID) assert.Equal(t, w.Name, w1.Name) - assert.Equal(t, w.Root.PipelineID, w1.Root.PipelineID) - assert.Equal(t, w.Root.PipelineName, w1.Root.PipelineName) - assertEqualNode(t, w.Root, w1.Root) + assert.Equal(t, w.WorkflowData.Node.Context.PipelineID, w1.WorkflowData.Node.Context.PipelineID) + assertEqualNode(t, &w.WorkflowData.Node, &w1.WorkflowData.Node) ws, err := workflow.LoadAll(db, proj.Key) test.NoError(t, err) @@ -1418,7 +1469,7 @@ func TestInsertSimpleWorkflowWithHookAndExport(t *testing.T) { return } - assert.Len(t, w.Root.Hooks, 1) + assert.Len(t, w.WorkflowData.Node.Hooks, 1) exp, err := exportentities.NewWorkflow(*w1) test.NoError(t, err) @@ -1429,3 +1480,339 @@ func TestInsertSimpleWorkflowWithHookAndExport(t *testing.T) { test.NoError(t, workflow.Delete(context.TODO(), db, cache, proj, &w)) } + +func TestInsertAndDeleteMultiHook(t *testing.T) { + db, cache, end := test.SetupPG(t, bootstrap.InitiliazeDB) + defer end() + u, _ := assets.InsertAdminUser(db) + test.NoError(t, workflow.CreateBuiltinWorkflowHookModels(db)) + + hookModels, err := workflow.LoadHookModels(db) + test.NoError(t, err) + var webHookID int64 + var schedulerID int64 + var repoWebHookID int64 + for _, h := range hookModels { + if h.Name == sdk.WebHookModel.Name { + webHookID = h.ID + } + if h.Name == sdk.RepositoryWebHookModel.Name { + repoWebHookID = h.ID + } + if h.Name == sdk.SchedulerModel.Name { + schedulerID = h.ID + } + } + + // Create project + key := sdk.RandomString(10) + proj := assets.InsertTestProject(t, db, cache, key, key, u) + assert.NoError(t, repositoriesmanager.InsertForProject(db, proj, &sdk.ProjectVCSServer{ + Name: "github", + Data: map[string]string{ + "token": "foo", + "secret": "bar", + }, + })) + + _, err = db.Exec("DELETE FROM services") + assert.NoError(t, err) + + mockVCSSservice := &sdk.Service{Name: "TestInsertAndDeleteMultiHookVCS", Type: services.TypeVCS} + test.NoError(t, services.Insert(db, mockVCSSservice)) + + mockHookServices := &sdk.Service{Name: "TestInsertAndDeleteMultiHookHook", Type: services.TypeHooks} + test.NoError(t, services.Insert(db, mockHookServices)) + + //This is a mock for the vcs service + services.HTTPClient = mock( + func(r *http.Request) (*http.Response, error) { + body := new(bytes.Buffer) + w := new(http.Response) + enc := json.NewEncoder(body) + w.Body = ioutil.NopCloser(body) + switch r.URL.String() { + // NEED get REPO + + case "/vcs/github/repos/sguiheux/demo": + repo := sdk.VCSRepo{ + URL: "https", + Name: "demo", + ID: "123", + Fullname: "sguiheux/demo", + Slug: "sguiheux", + HTTPCloneURL: "https://github.com/sguiheux/demo.git", + SSHCloneURL: "git://github.com/sguiheux/demo.git", + } + if err := enc.Encode(repo); err != nil { + return writeError(w, err) + } + + // NEED for default payload on insert + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } + case "/task/bulk": + var hooks map[string]sdk.NodeHook + request, err := ioutil.ReadAll(r.Body) + if err != nil { + return writeError(w, err) + } + if err := json.Unmarshal(request, &hooks); err != nil { + return writeError(w, err) + } + if len(hooks) != 1 { + return writeError(w, fmt.Errorf("Must only have 1 hook")) + } + k := reflect.ValueOf(hooks).MapKeys()[0].String() + hooks[k].Config["webHookURL"] = sdk.WorkflowNodeHookConfigValue{ + Value: fmt.Sprintf("http://6.6.6:8080/%s", hooks[k].UUID), + Type: "string", + Configurable: false, + } + + if err := enc.Encode(map[string]sdk.NodeHook{ + hooks[k].UUID: hooks[k], + }); err != nil { + return writeError(w, err) + } + case "/vcs/github/webhooks": + + infos := repositoriesmanager.WebhooksInfos{ + WebhooksDisabled: false, + WebhooksSupported: true, + Icon: "github", + } + if err := enc.Encode(infos); err != nil { + return writeError(w, err) + } + case "/vcs/github/repos/sguiheux/demo/hooks": + pr := sdk.VCSHook{ + ID: "666", + } + if err := enc.Encode(pr); err != nil { + return writeError(w, err) + } + default: + if strings.HasPrefix(r.URL.String(), "/vcs/github/repos/sguiheux/demo/hooks?url=htt") && strings.HasSuffix(r.URL.String(), "&id=666") { + // Do NOTHING + } else { + t.Fatalf("UNKNOWN ROUTE: %s", r.URL.String()) + } + + } + + return w, nil + }, + ) + + pip := &sdk.Pipeline{ + Name: "build", + Stages: []sdk.Stage{ + { + Name: "stage1", + BuildOrder: 1, + Enabled: true, + Jobs: []sdk.Job{ + { + Enabled: true, + Action: sdk.Action{ + Name: "JOb1", + Enabled: true, + Actions: []sdk.Action{ + { + Name: "gitClone", + Type: sdk.BuiltinAction, + Enabled: true, + Parameters: []sdk.Parameter{ + { + Name: "branch", + Value: "{{.git.branch}}", + }, + { + Name: "commit", + Value: "{{.git.hash}}", + }, + { + Name: "directory", + Value: "{{.cds.workspace}}", + }, + { + Name: "password", + Value: "", + }, + { + Name: "privateKey", + Value: "", + }, + { + Name: "url", + Value: "{{.git.url}}", + }, + { + Name: "user", + Value: "", + }, + { + Name: "depth", + Value: "12", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + assert.NoError(t, pipeline.Import(context.TODO(), db, cache, proj, pip, nil, u)) + var errPip error + pip, errPip = pipeline.LoadPipeline(db, proj.Key, pip.Name, true) + assert.NoError(t, errPip) + + // Add application + appS := `version: v1.0 +name: blabla +vcs_server: github +repo: sguiheux/demo +vcs_ssh_key: proj-blabla +` + var eapp = new(exportentities.Application) + assert.NoError(t, yaml.Unmarshal([]byte(appS), eapp)) + app, _, globalError := application.ParseAndImport(db, cache, proj, eapp, application.ImportOptions{Force: true}, nil, u) + assert.NoError(t, globalError) + + proj.Applications = append(proj.Applications, *app) + proj.Pipelines = append(proj.Pipelines, *pip) + + w := sdk.Workflow{ + ProjectID: proj.ID, + ProjectKey: proj.Key, + Name: sdk.RandomString(10), + WorkflowData: &sdk.WorkflowData{ + Node: sdk.Node{ + Name: "root", + Type: sdk.NodeTypePipeline, + Context: &sdk.NodeContext{ + PipelineID: proj.Pipelines[0].ID, + ApplicationID: proj.Applications[0].ID, + }, + Hooks: []sdk.NodeHook{ + { + Config: sdk.RepositoryWebHookModel.DefaultConfig, + HookModelID: repoWebHookID, + }, + }, + }, + }, + Applications: map[int64]sdk.Application{ + proj.Applications[0].ID: proj.Applications[0], + }, + Pipelines: map[int64]sdk.Pipeline{ + proj.Pipelines[0].ID: proj.Pipelines[0], + }, + } + assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) + + // Add check on Hook + assert.Equal(t, "666", w.WorkflowData.Node.Hooks[0].Config["webHookID"].Value) + assert.Equal(t, "github", w.WorkflowData.Node.Hooks[0].Config["hookIcon"].Value) + assert.Equal(t, fmt.Sprintf("http://6.6.6:8080/%s", w.WorkflowData.Node.Hooks[0].UUID), w.WorkflowData.Node.Hooks[0].Config["webHookURL"].Value) + t.Logf("%+v", w.WorkflowData.Node.Hooks[0]) + + // Load workflow + oldW, err := workflow.LoadByID(db, cache, proj, w.ID, u, workflow.LoadOptions{}) + assert.NoError(t, err) + + // Add WEB HOOK + w.WorkflowData.Node.Hooks = append(w.WorkflowData.Node.Hooks, sdk.NodeHook{ + Config: sdk.WebHookModel.DefaultConfig, + HookModelID: webHookID, + }) + + assert.NoError(t, workflow.Update(context.TODO(), db, cache, &w, proj, u, workflow.UpdateOptions{OldWorkflow: oldW})) + + // Add check on HOOKS + assert.Equal(t, 2, len(w.WorkflowData.Node.Hooks)) + for _, h := range w.WorkflowData.Node.Hooks { + if h.HookModelID == repoWebHookID { + assert.True(t, oldW.WorkflowData.Node.Hooks[0].Equals(h)) + } else if h.HookModelID == webHookID { + assert.Equal(t, fmt.Sprintf("http://6.6.6:8080/%s", h.UUID), h.Config["webHookURL"].Value) + } else { + // Must not go here + t.Fail() + } + + } + + oldW, err = workflow.LoadByID(db, cache, proj, w.ID, u, workflow.LoadOptions{}) + assert.NoError(t, err) + + // Add Scheduler + w.WorkflowData.Node.Hooks = append(w.WorkflowData.Node.Hooks, sdk.NodeHook{ + Config: sdk.SchedulerModel.DefaultConfig, + HookModelID: schedulerID, + }) + + assert.NoError(t, workflow.Update(context.TODO(), db, cache, &w, proj, u, workflow.UpdateOptions{OldWorkflow: oldW})) + + // Add check on HOOKS + assert.Equal(t, 3, len(w.WorkflowData.Node.Hooks)) + + oldHooks := oldW.WorkflowData.GetHooks() + for _, h := range w.WorkflowData.Node.Hooks { + if h.HookModelID == repoWebHookID { + assert.True(t, h.Equals(*oldHooks[h.UUID])) + } else if h.HookModelID == webHookID { + assert.True(t, h.Equals(*oldHooks[h.UUID])) + } else if h.HookModelID == schedulerID { + assert.Contains(t, h.Config["payload"].Value, "git.branch") + assert.Contains(t, h.Config["payload"].Value, "git.author") + assert.Contains(t, h.Config["payload"].Value, "git.hash") + assert.Contains(t, h.Config["payload"].Value, "git.hash.before") + assert.Contains(t, h.Config["payload"].Value, "git.message") + assert.Contains(t, h.Config["payload"].Value, "git.repository") + assert.Contains(t, h.Config["payload"].Value, "git.tag") + } else { + // Must not go here + t.Fail() + } + } + + oldW, err = workflow.LoadByID(db, cache, proj, w.ID, u, workflow.LoadOptions{}) + assert.NoError(t, err) + + // Delete repository webhook + var index = 0 + for i, h := range w.WorkflowData.Node.Hooks { + if h.HookModelID == repoWebHookID { + index = i + } + } + w.WorkflowData.Node.Hooks = append(w.WorkflowData.Node.Hooks[:index], w.WorkflowData.Node.Hooks[index+1:]...) + assert.NoError(t, workflow.Update(context.TODO(), db, cache, &w, proj, u, workflow.UpdateOptions{OldWorkflow: oldW})) + + // Add check on HOOKS + assert.Equal(t, 2, len(w.WorkflowData.Node.Hooks)) + + oldHooks = oldW.WorkflowData.GetHooks() + for _, h := range w.WorkflowData.Node.Hooks { + if h.HookModelID == webHookID { + assert.True(t, h.Equals(*oldHooks[h.UUID])) + } else if h.HookModelID == schedulerID { + assert.True(t, h.Equals(*oldHooks[h.UUID])) + } else { + // Must not go here + t.Fail() + } + } + +} diff --git a/engine/api/workflow/dao_trigger.go b/engine/api/workflow/dao_trigger.go deleted file mode 100644 index 466cf2e218..0000000000 --- a/engine/api/workflow/dao_trigger.go +++ /dev/null @@ -1,70 +0,0 @@ -package workflow - -import ( - "context" - "database/sql" - - "github.com/go-gorp/gorp" - - "github.com/ovh/cds/engine/api/cache" - "github.com/ovh/cds/sdk" -) - -// insertTrigger inserts a trigger -func insertTrigger(db gorp.SqlExecutor, store cache.Store, w *sdk.Workflow, node *sdk.WorkflowNode, trigger *sdk.WorkflowNodeTrigger, u *sdk.User) error { - trigger.WorkflowNodeID = node.ID - trigger.ID = 0 - trigger.WorkflowDestNodeID = 0 - - //Setup destination node - if err := insertNode(db, store, w, &trigger.WorkflowDestNode, u, false); err != nil { - return sdk.WrapError(err, "Unable to setup destination node %d on trigger %d", trigger.WorkflowDestNode.ID, trigger.ID) - } - trigger.WorkflowDestNodeID = trigger.WorkflowDestNode.ID - - //Insert trigger - dbt := NodeTrigger(*trigger) - if err := db.Insert(&dbt); err != nil { - return sdk.WrapError(err, "Unable to insert trigger") - } - trigger.ID = dbt.ID - trigger.WorkflowDestNode.TriggerSrcID = trigger.ID - - // Update node trigger ID - if err := updateWorkflowTriggerSrc(db, &trigger.WorkflowDestNode); err != nil { - return sdk.WrapError(err, "Unable to update node %d for trigger %d", trigger.WorkflowDestNode.ID, trigger.ID) - } - - return nil -} - -// LoadTriggers loads trigger from a node -func loadTriggers(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *sdk.Project, w *sdk.Workflow, node *sdk.WorkflowNode, u *sdk.User, opts LoadOptions) ([]sdk.WorkflowNodeTrigger, error) { - dbtriggers := []NodeTrigger{} - if _, err := db.Select(&dbtriggers, "select * from workflow_node_trigger where workflow_node_id = $1 ORDER by workflow_node_trigger.id ASC", node.ID); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, sdk.WrapError(err, "Unable to load triggers") - } - - if len(dbtriggers) == 0 { - return nil, nil - } - - triggers := []sdk.WorkflowNodeTrigger{} - for _, dbt := range dbtriggers { - t := sdk.WorkflowNodeTrigger(dbt) - if t.WorkflowDestNodeID != 0 { - //Load destination node - dest, err := loadNode(ctx, db, store, proj, w, t.WorkflowDestNodeID, u, opts) - if err != nil { - return nil, sdk.WrapError(err, "Unable to load destination node %d", t.WorkflowDestNodeID) - } - t.WorkflowDestNode = *dest - } - - triggers = append(triggers, t) - } - return triggers, nil -} diff --git a/engine/api/workflow/execute_node_run.go b/engine/api/workflow/execute_node_run.go index ce7d3c1535..8e456643ff 100644 --- a/engine/api/workflow/execute_node_run.go +++ b/engine/api/workflow/execute_node_run.go @@ -642,7 +642,7 @@ func NodeBuildParametersFromWorkflow(ctx context.Context, db gorp.SqlExecutor, s } // Add payload from root - if wf.Root.Context.DefaultPayload != nil { + if wf.WorkflowData.Node.Context.DefaultPayload != nil { e := dump.NewDefaultEncoder() e.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()} e.ExtraFields.DetailedMap = false @@ -651,7 +651,7 @@ func NodeBuildParametersFromWorkflow(ctx context.Context, db gorp.SqlExecutor, s e.ExtraFields.Type = false tempParams := sdk.ParametersToMap(res) - m1, errm1 := e.ToStringMap(wf.Root.Context.DefaultPayload) + m1, errm1 := e.ToStringMap(wf.WorkflowData.Node.Context.DefaultPayload) if errm1 == nil { mergedParameters := sdk.ParametersMapMerge(tempParams, m1, sdk.MapMergeOptions.ExcludeGitParams) res = sdk.ParametersFromMap(mergedParameters) diff --git a/engine/api/workflow/gorp_model.go b/engine/api/workflow/gorp_model.go index dce4171ae9..335330785c 100644 --- a/engine/api/workflow/gorp_model.go +++ b/engine/api/workflow/gorp_model.go @@ -12,23 +12,6 @@ import ( // Workflow is a gorp wrapper around sdk.WorkflowData type Workflow sdk.Workflow -// Node is a gorp wrapper around sdk.WorkflowNode -type Node sdk.WorkflowNode - -// NodeContext is a gorp wrapper around sdk.WorkflowNodeContext -type NodeContext sdk.WorkflowNodeContext - -// NodeTrigger is a gorp wrapper around sdk.WorkflowNodeTrigger -type NodeTrigger sdk.WorkflowNodeTrigger - -// Join is a gorp wrapper around sdk.WorkflowNodeJoin -type Join sdk.WorkflowNodeJoin - -// JoinTrigger is a gorp wrapper around sdk.WorkflowNodeJoinTrigger -type JoinTrigger sdk.WorkflowNodeJoinTrigger - -type outgoingHookTrigger sdk.WorkflowNodeOutgoingHookTrigger - // Notification is a gorp wrapper around sdk.WorkflowNotification type Notification sdk.WorkflowNotification @@ -198,15 +181,6 @@ type dbStaticFiles sdk.StaticFiles // RunTag is a gorp wrapper around sdk.WorkflowRunTag type RunTag sdk.WorkflowRunTag -// NodeHook is a gorp wrapper around sdk.WorkflowNodeHook -type NodeHook sdk.WorkflowNodeHook - -type nodeOutgoingHook sdk.WorkflowNodeOutgoingHook - -type dbNodeFork sdk.WorkflowNodeFork - -type dbNodeForkTrigger sdk.WorkflowNodeForkTrigger - // hookModel is a gorp wrapper around sdk.WorkflowHookModel type hookModel sdk.WorkflowHookModel @@ -226,15 +200,6 @@ type dbAsCodeEvents sdk.AsCodeEvent func init() { gorpmapping.Register(gorpmapping.New(Workflow{}, "workflow", true, "id")) - gorpmapping.Register(gorpmapping.New(Node{}, "workflow_node", true, "id")) - gorpmapping.Register(gorpmapping.New(NodeTrigger{}, "workflow_node_trigger", true, "id")) - gorpmapping.Register(gorpmapping.New(NodeContext{}, "workflow_node_context", true, "id")) - gorpmapping.Register(gorpmapping.New(sqlContext{}, "workflow_node_context", true, "id")) - gorpmapping.Register(gorpmapping.New(NodeHook{}, "workflow_node_hook", true, "id")) - gorpmapping.Register(gorpmapping.New(nodeOutgoingHook{}, "workflow_node_outgoing_hook", true, "id")) - gorpmapping.Register(gorpmapping.New(outgoingHookTrigger{}, "workflow_node_outgoing_hook_trigger", true, "id")) - gorpmapping.Register(gorpmapping.New(Join{}, "workflow_node_join", true, "id")) - gorpmapping.Register(gorpmapping.New(JoinTrigger{}, "workflow_node_join_trigger", true, "id")) gorpmapping.Register(gorpmapping.New(Run{}, "workflow_run", true, "id")) gorpmapping.Register(gorpmapping.New(NodeRun{}, "workflow_node_run", true, "id")) gorpmapping.Register(gorpmapping.New(JobRun{}, "workflow_node_run_job", true, "id")) @@ -247,8 +212,6 @@ func init() { gorpmapping.Register(gorpmapping.New(Coverage{}, "workflow_node_run_coverage", false, "workflow_id", "workflow_run_id", "workflow_node_run_id", "repository", "branch")) gorpmapping.Register(gorpmapping.New(dbStaticFiles{}, "workflow_node_run_static_files", true, "id")) gorpmapping.Register(gorpmapping.New(dbNodeRunVulenrabilitiesReport{}, "workflow_node_run_vulnerability", true, "id")) - gorpmapping.Register(gorpmapping.New(dbNodeFork{}, "workflow_node_fork", true, "id")) - gorpmapping.Register(gorpmapping.New(dbNodeForkTrigger{}, "workflow_node_fork_trigger", true, "id")) gorpmapping.Register(gorpmapping.New(dbNodeData{}, "w_node", true, "id")) gorpmapping.Register(gorpmapping.New(dbNodeHookData{}, "w_node_hook", true, "id")) gorpmapping.Register(gorpmapping.New(dbNodeContextData{}, "w_node_context", true, "id")) diff --git a/engine/api/workflow/hook.go b/engine/api/workflow/hook.go index 80933c3745..30a6542838 100644 --- a/engine/api/workflow/hook.go +++ b/engine/api/workflow/hook.go @@ -5,11 +5,11 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/fsamin/go-dump" "github.com/go-gorp/gorp" - "github.com/ovh/cds/engine/api/application" "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/observability" "github.com/ovh/cds/engine/api/repositoriesmanager" @@ -18,159 +18,31 @@ import ( "github.com/ovh/cds/sdk/log" ) -// HookRegistration ensures hooks registration on Hook µService -func HookRegistration(ctx context.Context, db gorp.SqlExecutor, store cache.Store, oldW *sdk.Workflow, wf sdk.Workflow, p *sdk.Project) error { - ctx, end := observability.Span(ctx, "workflow.HookRegistration") - defer end() - - var hookToUpdate map[string]sdk.WorkflowNodeHook - var hookToDelete map[string]sdk.WorkflowNodeHook - if oldW != nil { - hookToUpdate, hookToDelete = mergeAndDiffHook(oldW.GetHooks(), wf.GetHooks()) - } else { - hookToUpdate = wf.GetHooks() - } - - observability.Current(ctx, observability.Tag("hook_update_count", len(hookToUpdate))) - observability.Current(ctx, observability.Tag("hook_delete_count", len(hookToDelete))) - - if len(hookToUpdate) > 0 { - //Push the hook to hooks µService - //Load service "hooks" - srvs, err := services.FindByType(db, services.TypeHooks) - if err != nil { - return sdk.WrapError(err, "Unable to get services dao") - } - - // Update in VCS - for i := range hookToUpdate { - h := hookToUpdate[i] - if oldW != nil && wf.Name != oldW.Name { - configValue := h.Config[sdk.HookConfigWorkflow] - configValue.Value = wf.Name - h.Config[sdk.HookConfigWorkflow] = configValue - hookToUpdate[i] = h - } - } - - //Perform the request on one off the hooks service - if len(srvs) < 1 { - return sdk.WrapError(fmt.Errorf("No hooks service available, please try again"), "Unable to get services dao") - } - - // Update scheduler payload - for i := range hookToUpdate { - h := hookToUpdate[i] - - if h.WorkflowHookModel.Name == sdk.SchedulerModelName { - // Add git.branch in scheduler payload - if wf.Root.IsLinkedToRepo() { - var payloadValues map[string]string - if h.Config["payload"].Value != "" { - var bodyJSON interface{} - //Try to parse the body as an array - bodyJSONArray := []interface{}{} - if err := json.Unmarshal([]byte(h.Config["payload"].Value), &bodyJSONArray); err != nil { - //Try to parse the body as a map - bodyJSONMap := map[string]interface{}{} - if err2 := json.Unmarshal([]byte(h.Config["payload"].Value), &bodyJSONMap); err2 == nil { - bodyJSON = bodyJSONMap - } - } else { - bodyJSON = bodyJSONArray - } - - //Go Dump - var errDump error - payloadValues, errDump = dump.ToStringMap(bodyJSON) - if errDump != nil { - return sdk.WrapError(errDump, "HookRegistration> Cannot dump payload %+v", h.Config["payload"].Value) - } - } - - // try get git.branch on defaultPayload - if payloadValues["git.branch"] == "" { - defaultPayloadMap, errP := wf.Root.Context.DefaultPayloadToMap() - if errP != nil { - return sdk.WrapError(errP, "HookRegistration> Cannot read node default payload") - } - if defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitBranch"] != "" { - payloadValues["git.branch"] = defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitBranch"] - } - if defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitRepository"] != "" { - payloadValues["git.repository"] = defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitRepository"] - } - } - - // try get git.branch on repo linked - if payloadValues["git.branch"] == "" { - defaultPayload, errDefault := DefaultPayload(ctx, db, store, p, &wf) - if errDefault != nil { - return sdk.WrapError(errDefault, "HookRegistration> Unable to get default payload") - } - var errDump error - payloadValues, errDump = dump.ToStringMap(defaultPayload) - if errDump != nil { - return sdk.WrapError(errDump, "HookRegistration> Cannot dump payload %+v", h.Config["payload"].Value) - } - } - - payloadStr, errM := json.MarshalIndent(&payloadValues, "", " ") - if errM != nil { - return sdk.WrapError(errM, "HookRegistration> Cannot marshal hook config payload : %s", errM) - } - pl := h.Config["payload"] - pl.Value = string(payloadStr) - h.Config["payload"] = pl - hookToUpdate[i] = h - } - } - } - - // Create hook on µservice - _, code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/task/bulk", hookToUpdate, &hookToUpdate) - if errHooks != nil || code >= 400 { - return sdk.WrapError(errHooks, "HookRegistration> Unable to create hooks [%d]", code) - } - - // Create vcs configuration ( always after hook creation to have webhook URL) + update hook in DB - for i := range hookToUpdate { - h := hookToUpdate[i] - v, ok := h.Config["webHookID"] - if h.WorkflowHookModel.Name == sdk.RepositoryWebHookModelName && h.Config["vcsServer"].Value != "" && (!ok || v.Value == "") { - if err := createVCSConfiguration(ctx, db, store, p, &h); err != nil { - return sdk.WrapError(err, "Cannot update vcs configuration") - } - } - if err := UpdateHook(db, &h); err != nil { - return sdk.WrapError(err, "Cannot update hook") - } +func computeHookToDelete(newWorkflow *sdk.Workflow, oldWorkflow *sdk.Workflow) map[string]*sdk.NodeHook { + hookToDelete := make(map[string]*sdk.NodeHook) + currentHooks := newWorkflow.WorkflowData.GetHooks() + for k, h := range oldWorkflow.WorkflowData.GetHooks() { + if _, has := currentHooks[k]; !has { + hookToDelete[k] = h } } + return hookToDelete +} - if len(hookToDelete) > 0 { - if err := DeleteHookConfiguration(ctx, db, store, p, hookToDelete); err != nil { - return sdk.WrapError(err, "Cannot remove hook configuration") - } +func hookUnregistration(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, hookToDelete map[string]*sdk.NodeHook) error { + if len(hookToDelete) == 0 { + return nil } - return nil -} -// DeleteHookConfiguration delete hooks configuration (and their vcs configuration) -func DeleteHookConfiguration(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, hookToDelete map[string]sdk.WorkflowNodeHook) error { - ctx, end := observability.Span(ctx, "workflow.DeleteHookConfiguration") - defer end() // Delete from vcs configuration if needed - count := 0 for _, h := range hookToDelete { - count++ - if h.WorkflowHookModel.Name == sdk.RepositoryWebHookModelName { + if h.HookModelName == sdk.RepositoryWebHookModelName { // Call VCS to know if repository allows webhook and get the configuration fields projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value) if projectVCSServer != nil { client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, projectVCSServer) if errclient != nil { - return sdk.WrapError(errclient, "deleteHookConfiguration> Cannot get vcs client") + return errclient } vcsHook := sdk.VCSHook{ Method: "POST", @@ -181,12 +53,6 @@ func DeleteHookConfiguration(ctx context.Context, db gorp.SqlExecutor, store cac if err := client.DeleteHook(ctx, h.Config["repoFullName"].Value, vcsHook); err != nil { log.Error("deleteHookConfiguration> Cannot delete hook on repository %s", err) } - observability.Current(ctx, observability.Tag(fmt.Sprintf("UUID_%d", count), h.UUID)) - observability.Current(ctx, observability.Tag(fmt.Sprintf("VCS_ID_%d", count), vcsHook.ID)) - h.Config["webHookID"] = sdk.WorkflowNodeHookConfigValue{ - Value: vcsHook.ID, - Configurable: false, - } } } } @@ -195,21 +61,184 @@ func DeleteHookConfiguration(ctx context.Context, db gorp.SqlExecutor, store cac //Load service "hooks" srvs, err := services.FindByType(db, services.TypeHooks) if err != nil { - return sdk.WrapError(err, "Unable to get services dao") + return err } _, code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodDelete, "/task/bulk", hookToDelete, nil) if errHooks != nil || code >= 400 { // if we return an error, transaction will be rollbacked => hook will in database be not anymore on gitlab/bitbucket/github. // so, it's just a warn log - log.Warning("HookRegistration> Unable to delete old hooks [%d]: %s", code, errHooks) + log.Error("HookRegistration> unable to delete old hooks [%d]: %s", code, errHooks) + } + return nil +} + +func hookRegistration(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, wf *sdk.Workflow, oldWorkflow *sdk.Workflow) error { + var oldHooks map[string]*sdk.NodeHook + if oldWorkflow != nil { + oldHooks = oldWorkflow.WorkflowData.GetHooks() + } + if len(wf.WorkflowData.Node.Hooks) <= 0 { + return nil + } + + srvs, err := services.FindByType(db, services.TypeHooks) + if err != nil { + return sdk.WrapError(err, "unable to get services dao") + } + + //Perform the request on one off the hooks service + if len(srvs) < 1 { + return sdk.WrapError(fmt.Errorf("no hooks service available, please try again"), "Unable to get services dao") + } + + hookToUpdate := make(map[string]sdk.NodeHook) + for i := range wf.WorkflowData.Node.Hooks { + h := &wf.WorkflowData.Node.Hooks[i] + + if h.UUID == "" && h.Ref == "" { + // generate uuid + h.UUID = sdk.UUID() + if h.Ref == "" { + h.Ref = fmt.Sprintf("%d", time.Now().Unix()) + } + } else if oldHooks != nil { + // search previous hook configuration + previousHook, has := oldHooks[h.UUID] + // If previous hook is the same, we do nothing + if has && h.Equals(*previousHook) { + continue + } + } + + h.Config[sdk.HookConfigProject] = sdk.WorkflowNodeHookConfigValue{ + Value: wf.ProjectKey, + Configurable: false, + } + h.Config[sdk.HookConfigWorkflow] = sdk.WorkflowNodeHookConfigValue{ + Value: wf.Name, + Configurable: false, + } + h.Config[sdk.HookConfigWorkflowID] = sdk.WorkflowNodeHookConfigValue{ + Value: fmt.Sprint(wf.ID), + Configurable: false, + } + if h.HookModelName == sdk.RepositoryWebHookModelName || h.HookModelName == sdk.GitPollerModelName || h.HookModelName == sdk.GerritHookModelName { + if wf.WorkflowData.Node.Context.ApplicationID == 0 || wf.Applications[wf.WorkflowData.Node.Context.ApplicationID].RepositoryFullname == "" || wf.Applications[wf.WorkflowData.Node.Context.ApplicationID].VCSServer == "" { + return sdk.NewErrorFrom(sdk.ErrForbidden, "cannot create a git poller or repository webhook on an application without a repository") + } + h.Config[sdk.HookConfigVCSServer] = sdk.WorkflowNodeHookConfigValue{ + Value: wf.Applications[wf.WorkflowData.Node.Context.ApplicationID].VCSServer, + Configurable: false, + } + h.Config[sdk.HookConfigRepoFullName] = sdk.WorkflowNodeHookConfigValue{ + Value: wf.Applications[wf.WorkflowData.Node.Context.ApplicationID].RepositoryFullname, + Configurable: false, + } + } + + if err := updateSchedulerPayload(ctx, db, store, p, wf, h); err != nil { + return err + } + hookToUpdate[h.UUID] = *h } + + if len(hookToUpdate) > 0 { + // Create hook on µservice + _, code, errHooks := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/task/bulk", hookToUpdate, &hookToUpdate) + if errHooks != nil || code >= 400 { + return sdk.WrapError(errHooks, "unable to create hooks [%d]", code) + } + + hooks := wf.WorkflowData.GetHooks() + for i := range hookToUpdate { + hooks[i].Config = hookToUpdate[i].Config + } + + // Create vcs configuration ( always after hook creation to have webhook URL) + update hook in DB + for i := range wf.WorkflowData.Node.Hooks { + h := &wf.WorkflowData.Node.Hooks[i] + v, ok := h.Config["webHookID"] + if h.HookModelName == sdk.RepositoryWebHookModelName && h.Config["vcsServer"].Value != "" && (!ok || v.Value == "") { + if err := createVCSConfiguration(ctx, db, store, p, h); err != nil { + return sdk.WrapError(err, "Cannot update vcs configuration") + } + } + } + } + return nil } -func createVCSConfiguration(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, h *sdk.WorkflowNodeHook) error { +func updateSchedulerPayload(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, wf *sdk.Workflow, h *sdk.NodeHook) error { + if h.HookModelName != sdk.SchedulerModelName { + return nil + } + // Add git.branch in scheduler payload + if wf.WorkflowData.Node.IsLinkedToRepo(wf) { + var payloadValues map[string]string + if h.Config["payload"].Value != "" { + var bodyJSON interface{} + //Try to parse the body as an array + bodyJSONArray := []interface{}{} + if err := json.Unmarshal([]byte(h.Config["payload"].Value), &bodyJSONArray); err != nil { + //Try to parse the body as a map + bodyJSONMap := map[string]interface{}{} + if err2 := json.Unmarshal([]byte(h.Config["payload"].Value), &bodyJSONMap); err2 == nil { + bodyJSON = bodyJSONMap + } + } else { + bodyJSON = bodyJSONArray + } + + //Go Dump + var errDump error + payloadValues, errDump = dump.ToStringMap(bodyJSON) + if errDump != nil { + return sdk.WrapError(errDump, "cannot dump payload %+v", h.Config["payload"].Value) + } + } + + // try get git.branch on defaultPayload + if payloadValues["git.branch"] == "" { + defaultPayloadMap, errP := wf.WorkflowData.Node.Context.DefaultPayloadToMap() + if errP != nil { + return sdk.WrapError(errP, "cannot read node default payload") + } + if defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitBranch"] != "" { + payloadValues["git.branch"] = defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitBranch"] + } + if defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitRepository"] != "" { + payloadValues["git.repository"] = defaultPayloadMap["WorkflowNodeContextDefaultPayloadVCS.GitRepository"] + } + } + + // try get git.branch on repo linked + if payloadValues["git.branch"] == "" { + defaultPayload, errDefault := DefaultPayload(ctx, db, store, p, wf) + if errDefault != nil { + return sdk.WrapError(errDefault, "unable to get default payload") + } + var errDump error + payloadValues, errDump = dump.ToStringMap(defaultPayload) + if errDump != nil { + return sdk.WrapError(errDump, "cannot dump payload %+v", h.Config["payload"].Value) + } + } + + payloadStr, errM := json.MarshalIndent(&payloadValues, "", " ") + if errM != nil { + return sdk.WrapError(errM, "cannot marshal hook config payload : %s", errM) + } + pl := h.Config["payload"] + pl.Value = string(payloadStr) + h.Config["payload"] = pl + } + return nil +} + +func createVCSConfiguration(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, h *sdk.NodeHook) error { ctx, end := observability.Span(ctx, "workflow.createVCSConfiguration", observability.Tag("UUID", h.UUID)) defer end() - // Call VCS to know if repository allows webhook and get the configuration fields projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value) if projectVCSServer == nil { @@ -240,11 +269,6 @@ func createVCSConfiguration(ctx context.Context, db gorp.SqlExecutor, store cach Value: vcsHook.ID, Configurable: false, } - h.Config["webHookURL"] = sdk.WorkflowNodeHookConfigValue{ - Value: vcsHook.URL, - Configurable: false, - Type: sdk.HookConfigTypeString, - } h.Config[sdk.HookConfigIcon] = sdk.WorkflowNodeHookConfigValue{ Value: webHookInfo.Icon, Configurable: false, @@ -254,86 +278,28 @@ func createVCSConfiguration(ctx context.Context, db gorp.SqlExecutor, store cach return nil } -func mergeAndDiffHook(oldHooks map[string]sdk.WorkflowNodeHook, newHooks map[string]sdk.WorkflowNodeHook) (hookToUpdate map[string]sdk.WorkflowNodeHook, hookToDelete map[string]sdk.WorkflowNodeHook) { - hookToUpdate = make(map[string]sdk.WorkflowNodeHook) - hookToDelete = make(map[string]sdk.WorkflowNodeHook) - - for o := range oldHooks { - for n := range newHooks { - if oldHooks[o].Ref == newHooks[n].Ref { - nh := newHooks[n] - nh.UUID = oldHooks[o].UUID - if nh.Config == nil { - nh.Config = sdk.WorkflowNodeHookConfig{} - } - //Useful for RepositoryWebHook - if webhookID, ok := oldHooks[o].Config["webHookID"]; ok { - nh.Config["webHookID"] = webhookID - } - if oldIcon, ok := oldHooks[o].Config["hookIcon"]; oldHooks[o].WorkflowHookModelID == newHooks[n].WorkflowHookModelID && ok { - nh.Config["hookIcon"] = oldIcon - } - newHooks[n] = nh - } - } - } - - for key, hNew := range newHooks { - hold, ok := oldHooks[key] - // if new hook - if !ok || !hNew.Equals(hold) { - hookToUpdate[key] = newHooks[key] - continue - } - } - - for _, oldH := range oldHooks { - var exist bool - for _, newH := range newHooks { - if oldH.UUID == newH.UUID { - exist = true - break - } - } - if !exist { - hookToDelete[oldH.UUID] = oldH - } - } - return -} - // DefaultPayload returns the default payload for the workflow root func DefaultPayload(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, wf *sdk.Workflow) (interface{}, error) { - if wf.Root.Context == nil { + if wf.WorkflowData.Node.Context == nil || wf.WorkflowData.Node.Context.ApplicationID == 0 { return nil, nil } var defaultPayload interface{} - // Load application if not available - if wf.Root.Context != nil && wf.Root.Context.Application == nil && wf.Root.Context.ApplicationID != 0 { - app, errLa := application.LoadByID(db, store, wf.Root.Context.ApplicationID) - if errLa != nil { - return wf.Root.Context.DefaultPayload, sdk.WrapError(errLa, "DefaultPayload> unable to load application by id %d", wf.Root.Context.ApplicationID) - } - wf.Root.Context.Application = app - } - if wf.Root.Context.Application == nil { - return wf.Root.Context.DefaultPayload, nil - } + app := wf.Applications[wf.WorkflowData.Node.Context.ApplicationID] - if wf.Root.Context.Application.RepositoryFullname != "" { + if app.RepositoryFullname != "" { defaultBranch := "master" - projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, wf.Root.Context.Application.VCSServer) + projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, app.VCSServer) if projectVCSServer != nil { client, errclient := repositoriesmanager.AuthorizedClient(ctx, db, store, p.Key, projectVCSServer) if errclient != nil { - return wf.Root.Context.DefaultPayload, sdk.WrapError(errclient, "DefaultPayload> Cannot get authorized client") + return wf.WorkflowData.Node.Context.DefaultPayload, sdk.WrapError(errclient, "DefaultPayload> Cannot get authorized client") } - branches, errBr := client.Branches(ctx, wf.Root.Context.Application.RepositoryFullname) + branches, errBr := client.Branches(ctx, app.RepositoryFullname) if errBr != nil { - return wf.Root.Context.DefaultPayload, sdk.WrapError(errBr, "DefaultPayload> Cannot get branches for %s", wf.Root.Context.Application.RepositoryFullname) + return wf.WorkflowData.Node.Context.DefaultPayload, sdk.WrapError(errBr, "DefaultPayload> Cannot get branches for %s", app.RepositoryFullname) } for _, branch := range branches { @@ -344,23 +310,23 @@ func DefaultPayload(ctx context.Context, db gorp.SqlExecutor, store cache.Store, } } - defaultPayload = wf.Root.Context.DefaultPayload - if !wf.Root.Context.HasDefaultPayload() { + defaultPayload = wf.WorkflowData.Node.Context.DefaultPayload + if !wf.WorkflowData.Node.Context.HasDefaultPayload() { structuredDefaultPayload := sdk.WorkflowNodeContextDefaultPayloadVCS{ GitBranch: defaultBranch, - GitRepository: wf.Root.Context.Application.RepositoryFullname, + GitRepository: app.RepositoryFullname, } defaultPayloadBtes, _ := json.Marshal(structuredDefaultPayload) if err := json.Unmarshal(defaultPayloadBtes, &defaultPayload); err != nil { return nil, err } - } else if defaultPayloadMap, err := wf.Root.Context.DefaultPayloadToMap(); err == nil && defaultPayloadMap["git.branch"] == "" { + } else if defaultPayloadMap, err := wf.WorkflowData.Node.Context.DefaultPayloadToMap(); err == nil && defaultPayloadMap["git.branch"] == "" { defaultPayloadMap["git.branch"] = defaultBranch - defaultPayloadMap["git.repository"] = wf.Root.Context.Application.RepositoryFullname + defaultPayloadMap["git.repository"] = app.RepositoryFullname defaultPayload = defaultPayloadMap } } else { - defaultPayload = wf.Root.Context.DefaultPayload + defaultPayload = wf.WorkflowData.Node.Context.DefaultPayload } return defaultPayload, nil diff --git a/engine/api/workflow/hook_test.go b/engine/api/workflow/hook_test.go deleted file mode 100644 index 4b53120ef6..0000000000 --- a/engine/api/workflow/hook_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package workflow - -import ( - "reflect" - "testing" - - "github.com/ovh/cds/sdk" -) - -func Test_mergeAndDiffHook(t *testing.T) { - type args struct { - oldHooks map[string]sdk.WorkflowNodeHook - newHooks map[string]sdk.WorkflowNodeHook - } - tests := []struct { - name string - args args - wantHookToUpdate map[string]sdk.WorkflowNodeHook - wantHookToDelete map[string]sdk.WorkflowNodeHook - }{ - { - name: "one to update", - args: args{ - oldHooks: map[string]sdk.WorkflowNodeHook{ - "my-uuid-a": {Ref: "AAA", UUID: "my-uuid-a"}, - }, - newHooks: map[string]sdk.WorkflowNodeHook{ - "my-uuid-b": {Ref: "BBB"}, - "my-uuid-a": {Ref: "AAA", UUID: "my-uuid-a"}, - }, - }, - wantHookToUpdate: map[string]sdk.WorkflowNodeHook{ - "my-uuid-b": {Ref: "BBB"}, - }, - wantHookToDelete: map[string]sdk.WorkflowNodeHook{}, - }, - { - name: "one delete", - args: args{ - oldHooks: map[string]sdk.WorkflowNodeHook{ - "my-uuid-a": {Ref: "AAA", UUID: "my-uuid-a"}, - }, - newHooks: map[string]sdk.WorkflowNodeHook{}, - }, - wantHookToUpdate: map[string]sdk.WorkflowNodeHook{}, - wantHookToDelete: map[string]sdk.WorkflowNodeHook{ - "my-uuid-a": {Ref: "AAA", UUID: "my-uuid-a"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotHookToUpdate, gotHookToDelete := mergeAndDiffHook(tt.args.oldHooks, tt.args.newHooks) - if !reflect.DeepEqual(gotHookToUpdate, tt.wantHookToUpdate) { - t.Errorf("mergeAndDiffHook() gotHookToUpdate = %v, want %v", gotHookToUpdate, tt.wantHookToUpdate) - } - if !reflect.DeepEqual(gotHookToDelete, tt.wantHookToDelete) { - t.Errorf("mergeAndDiffHook() gotHookToDelete = %v, want %v", gotHookToDelete, tt.wantHookToDelete) - } - }) - } -} diff --git a/engine/api/workflow/process_node_test.go b/engine/api/workflow/process_node_test.go index e69a5e731c..51f550c3ae 100644 --- a/engine/api/workflow/process_node_test.go +++ b/engine/api/workflow/process_node_test.go @@ -51,11 +51,14 @@ func TestHookRunWithoutPayloadProcessNodeBuildParameter(t *testing.T) { }, })) + _, err = db.Exec("DELETE FROM services") + assert.NoError(t, err) + mockVCSSservice := &sdk.Service{Name: "TestHookRunWithoutPayloadProcessNodeBuildParameter", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) - defer func() { - _ = services.Delete(db, mockVCSSservice) // nolint - }() + + mockHookServices := &sdk.Service{Name: "TestHookRunWithoutPayloadProcessNodeBuildParameter", Type: services.TypeHooks} + test.NoError(t, services.Insert(db, mockHookServices)) //This is a mock for the vcs service services.HTTPClient = mock( @@ -64,7 +67,6 @@ func TestHookRunWithoutPayloadProcessNodeBuildParameter(t *testing.T) { w := new(http.Response) enc := json.NewEncoder(body) w.Body = ioutil.NopCloser(body) - switch r.URL.String() { // NEED get REPO case "/vcs/github/repos/sguiheux/demo": @@ -80,6 +82,16 @@ func TestHookRunWithoutPayloadProcessNodeBuildParameter(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + // NEED for default payload on insert + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=master": b := sdk.VCSBranch{ @@ -104,6 +116,18 @@ func TestHookRunWithoutPayloadProcessNodeBuildParameter(t *testing.T) { if err := enc.Encode(c); err != nil { return writeError(w, err) } + case "/task/bulk": + var hooks map[string]sdk.NodeHook + bts, err := ioutil.ReadAll(r.Body) + if err != nil { + return writeError(w, err) + } + if err := json.Unmarshal(bts, &hooks); err != nil { + return writeError(w, err) + } + if err := enc.Encode(hooks); err != nil { + return writeError(w, err) + } default: t.Fatalf("UNKNOWN ROUTE: %s", r.URL.String()) } @@ -149,13 +173,10 @@ func TestHookRunWithoutPayloadProcessNodeBuildParameter(t *testing.T) { }, } - (&w).RetroMigrate() - w.WorkflowData.Node.Context.DefaultPayload = map[string]string{ "git.branch": "master", "git.repository": "sguiheux/demo", } - w.Root.Context.DefaultPayload = w.WorkflowData.Node.Context.DefaultPayload assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) @@ -204,11 +225,13 @@ func TestHookRunWithHashOnlyProcessNodeBuildParameter(t *testing.T) { }, })) + _, err = db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestHookRunWithHashOnlyProcessNodeBuildParameter", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) - defer func() { - _ = services.Delete(db, mockVCSSservice) // nolint - }() + + mockHookServices := &sdk.Service{Name: "TestHookRunWithHashOnlyProcessNodeBuildParameter", Type: services.TypeHooks} + test.NoError(t, services.Insert(db, mockHookServices)) //This is a mock for the vcs service services.HTTPClient = mock( @@ -217,7 +240,6 @@ func TestHookRunWithHashOnlyProcessNodeBuildParameter(t *testing.T) { w := new(http.Response) enc := json.NewEncoder(body) w.Body = ioutil.NopCloser(body) - switch r.URL.String() { // NEED get REPO case "/vcs/github/repos/sguiheux/demo": @@ -233,6 +255,16 @@ func TestHookRunWithHashOnlyProcessNodeBuildParameter(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + // NEED for default payload on insert + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET COMMIT TO GET AUTHOR AND MESSAGE case "/vcs/github/repos/sguiheux/demo/commits/currentcommit": c := sdk.VCSCommit{ @@ -247,6 +279,18 @@ func TestHookRunWithHashOnlyProcessNodeBuildParameter(t *testing.T) { if err := enc.Encode(c); err != nil { return writeError(w, err) } + case "/task/bulk": + var hooks map[string]sdk.NodeHook + bts, err := ioutil.ReadAll(r.Body) + if err != nil { + return writeError(w, err) + } + if err := json.Unmarshal(bts, &hooks); err != nil { + return writeError(w, err) + } + if err := enc.Encode(hooks); err != nil { + return writeError(w, err) + } default: t.Fatalf("UNKNOWN ROUTE: %s", r.URL.String()) } @@ -292,13 +336,10 @@ func TestHookRunWithHashOnlyProcessNodeBuildParameter(t *testing.T) { }, } - (&w).RetroMigrate() - w.WorkflowData.Node.Context.DefaultPayload = map[string]string{ "git.branch": "master", "git.repository": "sguiheux/demo", } - w.Root.Context.DefaultPayload = w.WorkflowData.Node.Context.DefaultPayload assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) @@ -346,6 +387,8 @@ func TestManualRunWithPayloadProcessNodeBuildParameter(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunWithPayloadProcessNodeBuildParameter", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -375,6 +418,16 @@ func TestManualRunWithPayloadProcessNodeBuildParameter(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + // NEED for default payload on insert + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": b := sdk.VCSBranch{ @@ -437,7 +490,6 @@ func TestManualRunWithPayloadProcessNodeBuildParameter(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -483,6 +535,8 @@ func TestManualRunBranchAndCommitInPayloadProcessNodeBuildParameter(t *testing.T }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBranchAndCommitInPayloadProcessNodeBuildParameter", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -512,6 +566,16 @@ func TestManualRunBranchAndCommitInPayloadProcessNodeBuildParameter(t *testing.T if err := enc.Encode(repo); err != nil { return writeError(w, err) } + // NEED for default payload on insert + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": t.Fatalf("No need to get branch: %s", r.URL.String()) @@ -567,7 +631,6 @@ func TestManualRunBranchAndCommitInPayloadProcessNodeBuildParameter(t *testing.T }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -621,6 +684,8 @@ func TestManualRunBuildParameterMultiApplication(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -790,7 +855,6 @@ func TestManualRunBuildParameterMultiApplication(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -860,6 +924,8 @@ func TestGitParamOnPipelineWithoutApplication(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -889,6 +955,17 @@ func TestGitParamOnPipelineWithoutApplication(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "defaultCommit", + } + if err := enc.Encode([]sdk.VCSBranch{ + b, + }); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": b := sdk.VCSBranch{ @@ -964,7 +1041,6 @@ func TestGitParamOnPipelineWithoutApplication(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -1029,6 +1105,8 @@ func TestGitParamOnApplicationWithoutRepo(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -1058,6 +1136,15 @@ func TestGitParamOnApplicationWithoutRepo(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "defaultcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": b := sdk.VCSBranch{ @@ -1134,7 +1221,6 @@ func TestGitParamOnApplicationWithoutRepo(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -1195,6 +1281,8 @@ func TestGitParamOn2ApplicationSameRepo(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -1231,6 +1319,15 @@ func TestGitParamOn2ApplicationSameRepo(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: false, + DisplayID: "feat/branch", + LatestCommit: "mylastcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": repoBranch++ @@ -1315,7 +1412,6 @@ func TestGitParamOn2ApplicationSameRepo(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -1381,6 +1477,8 @@ func TestGitParamWithJoin(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -1417,6 +1515,15 @@ func TestGitParamWithJoin(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "defaultcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": repoBranch++ @@ -1511,7 +1618,6 @@ func TestGitParamWithJoin(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -1584,6 +1690,8 @@ func TestGitParamOn2ApplicationSameRepoWithFork(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunBuildParameterMultiApplication", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -1620,6 +1728,15 @@ func TestGitParamOn2ApplicationSameRepoWithFork(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "defaultcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": repoBranch++ @@ -1713,7 +1830,6 @@ func TestGitParamOn2ApplicationSameRepoWithFork(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN @@ -1779,6 +1895,8 @@ func TestManualRunWithPayloadAndRunCondition(t *testing.T) { }, })) + _, err := db.Exec("DELETE FROM services") + assert.NoError(t, err) mockVCSSservice := &sdk.Service{Name: "TestManualRunWithPayloadProcessNodeBuildParameter", Type: services.TypeVCS} test.NoError(t, services.Insert(db, mockVCSSservice)) defer func() { @@ -1808,6 +1926,15 @@ func TestManualRunWithPayloadAndRunCondition(t *testing.T) { if err := enc.Encode(repo); err != nil { return writeError(w, err) } + case "/vcs/github/repos/sguiheux/demo/branches": + b := sdk.VCSBranch{ + Default: true, + DisplayID: "master", + LatestCommit: "defaultcommit", + } + if err := enc.Encode([]sdk.VCSBranch{b}); err != nil { + return writeError(w, err) + } // NEED GET BRANCH TO GET LASTEST COMMIT case "/vcs/github/repos/sguiheux/demo/branches/?branch=feat%2Fbranch": b := sdk.VCSBranch{ @@ -1892,7 +2019,6 @@ func TestManualRunWithPayloadAndRunCondition(t *testing.T) { }, } - (&w).RetroMigrate() assert.NoError(t, workflow.Insert(db, cache, &w, proj, u)) // CREATE RUN diff --git a/engine/api/workflow/repository.go b/engine/api/workflow/repository.go index 43c426c9b8..810cd9daad 100644 --- a/engine/api/workflow/repository.go +++ b/engine/api/workflow/repository.go @@ -263,14 +263,15 @@ func pollRepositoryOperation(c context.Context, db gorp.SqlExecutor, store cache func createOperationRequest(w sdk.Workflow, opts sdk.WorkflowRunPostHandlerOption) (sdk.Operation, error) { ope := sdk.Operation{} - if w.Root.Context.Application == nil { + if w.WorkflowData.Node.Context.ApplicationID == 0 { return ope, sdk.WrapError(sdk.ErrApplicationNotFound, "CreateFromRepository> Workflow node root does not have a application context") } + app := w.Applications[w.WorkflowData.Node.Context.ApplicationID] ope = sdk.Operation{ - VCSServer: w.Root.Context.Application.VCSServer, - RepoFullName: w.Root.Context.Application.RepositoryFullname, + VCSServer: app.VCSServer, + RepoFullName: app.RepositoryFullname, URL: w.FromRepository, - RepositoryStrategy: w.Root.Context.Application.RepositoryStrategy, + RepositoryStrategy: app.RepositoryStrategy, Setup: sdk.OperationSetup{ Checkout: sdk.OperationCheckout{ Branch: "", diff --git a/engine/api/workflow/run_workflow.go b/engine/api/workflow/run_workflow.go index 78336f646c..2553a486b0 100644 --- a/engine/api/workflow/run_workflow.go +++ b/engine/api/workflow/run_workflow.go @@ -129,7 +129,7 @@ func StartWorkflowRun(ctx context.Context, db *gorp.DbMap, store cache.Store, p } if !permission.AccessToWorkflowNode(&wr.Workflow, fromNode, u, permission.PermissionReadExecute) { - return nil, sdk.WrapError(sdk.ErrNoPermExecution, "not enough right on root node %d", wr.Workflow.Root.ID) + return nil, sdk.WrapError(sdk.ErrNoPermExecution, "not enough right on root node %d", wr.Workflow.WorkflowData.Node.ID) } // Continue the current workflow run diff --git a/engine/api/workflow/run_workflow_test.go b/engine/api/workflow/run_workflow_test.go index d47813848e..7e3017a632 100644 --- a/engine/api/workflow/run_workflow_test.go +++ b/engine/api/workflow/run_workflow_test.go @@ -101,7 +101,6 @@ func TestManualRun1(t *testing.T) { }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{ @@ -258,7 +257,7 @@ func TestManualRun2(t *testing.T) { }, }, } - (&w).RetroMigrate() + test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{ @@ -408,7 +407,6 @@ func TestManualRun3(t *testing.T) { }, } - (&w).RetroMigrate() proj, _ = project.LoadByID(db, cache, proj.ID, u, project.LoadOptions.WithApplications, project.LoadOptions.WithPipelines, project.LoadOptions.WithEnvironments, project.LoadOptions.WithGroups, project.LoadOptions.WithVariablesWithClearPassword, project.LoadOptions.WithKeys) test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) @@ -690,7 +688,6 @@ func TestNoStage(t *testing.T) { }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{ DeepPipeline: true, @@ -765,7 +762,6 @@ func TestNoJob(t *testing.T) { }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) w1, err := workflow.Load(context.TODO(), db, cache, proj, "test_1", u, workflow.LoadOptions{ DeepPipeline: true, diff --git a/engine/api/workflow/sort.go b/engine/api/workflow/sort.go index 058513a1cb..9a1e07e70e 100644 --- a/engine/api/workflow/sort.go +++ b/engine/api/workflow/sort.go @@ -12,20 +12,19 @@ func Sort(w *sdk.Workflow) { if w == nil { return } - SortNode(w.Root) + SortNode(&w.WorkflowData.Node) } // SortNode sort the content of a node -func SortNode(n *sdk.WorkflowNode) { +func SortNode(n *sdk.Node) { if n == nil { return } sortNodeHooks(&n.Hooks) sortNodeTriggers(&n.Triggers) - sortNodeOutGoingHooks(&n.OutgoingHooks) } -func sortNodeHooks(hooks *[]sdk.WorkflowNodeHook) { +func sortNodeHooks(hooks *[]sdk.NodeHook) { if hooks == nil { return } @@ -34,43 +33,18 @@ func sortNodeHooks(hooks *[]sdk.WorkflowNodeHook) { }) } -func sortNodeOutGoingHooks(hooks *[]sdk.WorkflowNodeOutgoingHook) { - if hooks == nil { - return - } - sort.Slice(*hooks, func(i, j int) bool { - return (*hooks)[i].ID < (*hooks)[j].ID - }) - for i := range *hooks { - sortNodeOutGoingHookTriggers(&(*hooks)[i].Triggers) - } -} - -func sortNodeTriggers(triggers *[]sdk.WorkflowNodeTrigger) { +func sortNodeTriggers(triggers *[]sdk.NodeTrigger) { if triggers == nil { return } for i := range *triggers { - sortNodeHooks(&(*triggers)[i].WorkflowDestNode.Hooks) - } - - sort.Slice(*triggers, func(i, j int) bool { - t1 := &(*triggers)[i] - t2 := &(*triggers)[j] - - return strings.Compare(t1.WorkflowDestNode.Name, t2.WorkflowDestNode.Name) < 0 - }) -} - -func sortNodeOutGoingHookTriggers(triggers *[]sdk.WorkflowNodeOutgoingHookTrigger) { - if triggers == nil { - return + sortNodeHooks(&(*triggers)[i].ChildNode.Hooks) } sort.Slice(*triggers, func(i, j int) bool { t1 := &(*triggers)[i] t2 := &(*triggers)[j] - return strings.Compare(t1.WorkflowDestNode.Name, t2.WorkflowDestNode.Name) < 0 + return strings.Compare(t1.ChildNode.Name, t2.ChildNode.Name) < 0 }) } diff --git a/engine/api/workflow/workflow_exporter.go b/engine/api/workflow/workflow_exporter.go index 75be50d163..e4e4ac37e8 100644 --- a/engine/api/workflow/workflow_exporter.go +++ b/engine/api/workflow/workflow_exporter.go @@ -80,36 +80,34 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk wf.Template = i.Template } - apps := wf.GetApplications() - envs := wf.GetEnvironments() - pips := wf.GetPipelines() - //Reload app to retrieve secrets - for i := range apps { - app := &apps[i] + for i := range wf.Applications { + app := wf.Applications[i] vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword()) if errv != nil { return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name) } app.Variable = vars - if err := application.LoadAllDecryptedKeys(db, app); err != nil { + if err := application.LoadAllDecryptedKeys(db, &app); err != nil { return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name) } + wf.Applications[i] = app } //Reload env to retrieve secrets - for i := range envs { - env := &envs[i] + for i := range wf.Environments { + env := wf.Environments[i] vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword()) if errv != nil { return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name) } env.Variable = vars - if err := environment.LoadAllDecryptedKeys(db, env); err != nil { + if err := environment.LoadAllDecryptedKeys(db, &env); err != nil { return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name) } + wf.Environments[i] = env } buffw := new(bytes.Buffer) @@ -123,7 +121,7 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk wp.Workflow.Name = wf.Name wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes()) - for _, a := range apps { + for _, a := range wf.Applications { if a.FromRepository != wf.FromRepository { // don't export if coming from an other repository continue } @@ -137,7 +135,7 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk }) } - for _, e := range envs { + for _, e := range wf.Environments { if e.FromRepository != wf.FromRepository { // don't export if coming from an other repository continue } @@ -151,7 +149,7 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk }) } - for _, p := range pips { + for _, p := range wf.Pipelines { if p.FromRepository != wf.FromRepository { // don't export if coming from an other repository continue } diff --git a/engine/api/workflow/workflow_exporter_test.go b/engine/api/workflow/workflow_exporter_test.go index 236def07d6..a482860f06 100644 --- a/engine/api/workflow/workflow_exporter_test.go +++ b/engine/api/workflow/workflow_exporter_test.go @@ -120,7 +120,6 @@ func TestPull(t *testing.T) { }, } - (&w).RetroMigrate() proj, _ = project.Load(db, cache, proj.Key, u, project.LoadOptions.WithApplications, project.LoadOptions.WithEnvironments, project.LoadOptions.WithPipelines) test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) diff --git a/engine/api/workflow/workflow_importer.go b/engine/api/workflow/workflow_importer.go index c32c347c6c..150aa03eb3 100644 --- a/engine/api/workflow/workflow_importer.go +++ b/engine/api/workflow/workflow_importer.go @@ -3,16 +3,13 @@ package workflow import ( "context" "fmt" - "github.com/go-gorp/gorp" - "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/api/event" "github.com/ovh/cds/engine/api/group" "github.com/ovh/cds/engine/api/observability" "github.com/ovh/cds/engine/api/workflowtemplate" "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" ) //Import is able to create a new workflow and all its components @@ -30,21 +27,10 @@ func Import(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *s w.HistoryLength = 20 } - //Manage default payload - var err error - if w.Root.Context == nil { - w.Root.Context = &sdk.WorkflowNodeContext{} - } if w.WorkflowData.Node.Context == nil { w.WorkflowData.Node.Context = &sdk.NodeContext{} } - // TODO compute on WD.Node - if w.Root.Context.DefaultPayload, err = DefaultPayload(ctx, db, store, proj, w); err != nil { - log.Warning("workflow.Import> Cannot set default payload : %v", err) - } - w.WorkflowData.Node.Context.DefaultPayload = w.Root.Context.DefaultPayload - // create the workflow if not exists if oldW == nil { if err := Insert(db, store, w, proj, u); err != nil { @@ -54,11 +40,6 @@ func Import(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *s msgChan <- sdk.NewMessage(sdk.MsgWorkflowImportedInserted, w.Name) } - // HookRegistration after workflow.Update. It needs hooks to be created on DB - if errHr := HookRegistration(ctx, db, store, nil, *w, proj); errHr != nil { - return sdk.WrapError(errHr, "Cannot register hook") - } - // set the workflow id on template instance if exist if err := setTemplateData(ctx, db, proj, w, u, wTemplate); err != nil { return err @@ -72,31 +53,31 @@ func Import(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *s } // Retrieve existing hook - oldHooks := oldW.WorkflowData.GetHooksMapRef() - for i := range w.Root.Hooks { - h := &w.Root.Hooks[i] + oldHooksByRef := oldW.WorkflowData.GetHooksMapRef() + for i := range w.WorkflowData.Node.Hooks { + h := &w.WorkflowData.Node.Hooks[i] if h.Ref != "" { - if oldH, has := oldHooks[h.Ref]; has { + if oldH, has := oldHooksByRef[h.Ref]; has { if len(h.Config) == 0 { h.Config = oldH.Config } h.UUID = oldH.UUID + continue } } } - w.ID = oldW.ID - if err := Update(ctx, db, store, w, oldW, proj, u); err != nil { - return sdk.WrapError(err, "Unable to update workflow") - } // HookRegistration after workflow.Update. It needs hooks to be created on DB // Hook registration must only be done on default branch in case of workflow as-code // The derivation branch is set in workflow parser it is not comming from the default branch - if w.DerivationBranch == "" { - if errHr := HookRegistration(ctx, db, store, oldW, *w, proj); errHr != nil { - return sdk.WrapError(errHr, "Cannot register hook") - } + uptOptions := UpdateOptions{ + DisableHookManagement: w.DerivationBranch != "", + OldWorkflow: oldW, + } + + if err := Update(ctx, db, store, w, proj, u, uptOptions); err != nil { + return sdk.WrapError(err, "Unable to update workflow") } if err := importWorkflowGroups(db, w); err != nil { diff --git a/engine/api/workflow/workflow_importer_test.go b/engine/api/workflow/workflow_importer_test.go index d53e6e0bde..63610469a4 100644 --- a/engine/api/workflow/workflow_importer_test.go +++ b/engine/api/workflow/workflow_importer_test.go @@ -464,7 +464,6 @@ func TestImport(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - (tt.args.w).RetroMigrate() workflowExists, err := workflow.Exists(db, proj.Key, tt.args.w.Name) if err != nil { diff --git a/engine/api/workflow/workflow_parser.go b/engine/api/workflow/workflow_parser.go index 7e472a2dcb..4774259018 100644 --- a/engine/api/workflow/workflow_parser.go +++ b/engine/api/workflow/workflow_parser.go @@ -96,7 +96,6 @@ func ParseAndImport(ctx context.Context, db gorp.SqlExecutor, store cache.Store, if w.FromRepository != "" { if len(w.WorkflowData.Node.Hooks) == 0 { - // When you came from run workflow you have uuid if opts.HookUUID != "" && oldW != nil { oldHooks := oldW.WorkflowData.GetHooks() @@ -125,17 +124,13 @@ func ParseAndImport(ctx context.Context, db gorp.SqlExecutor, store cache.Store, }) } - w.RetroMigrate() var err error if w.WorkflowData.Node.Context.DefaultPayload, err = DefaultPayload(ctx, db, store, proj, w); err != nil { return nil, nil, sdk.WrapError(err, "Unable to get default payload") } - w.WorkflowData.Node.Context.DefaultPayload = w.Root.Context.DefaultPayload } } - w.RetroMigrate() - if opts.WorkflowName != "" && w.Name != opts.WorkflowName { return nil, nil, sdk.WrapError(sdk.ErrWorkflowNameImport, "Wrong workflow name") } diff --git a/engine/api/workflow/workflow_parser_test.go b/engine/api/workflow/workflow_parser_test.go index 443c76b6d1..0cbb3655cc 100644 --- a/engine/api/workflow/workflow_parser_test.go +++ b/engine/api/workflow/workflow_parser_test.go @@ -97,10 +97,17 @@ func TestParseAndImport(t *testing.T) { t.Logf("Workflow = \n%s", string(b)) if tt.name == "test-1" { - assert.Len(t, w.Root.Forks, 1) - assert.Equal(t, w.Root.Forks[0].Name, "fork") - assert.Len(t, w.Root.Forks[0].Triggers, 1) - assert.Equal(t, w.Root.Forks[0].Triggers[0].WorkflowDestNode.Name, "third") + assert.Len(t, w.WorkflowData.Node.Triggers, 2) + if w.WorkflowData.Node.Triggers[0].ChildNode.Type == "fork" { + assert.Equal(t, w.WorkflowData.Node.Triggers[0].ChildNode.Name, "fork") + assert.Len(t, w.WorkflowData.Node.Triggers[0].ChildNode.Triggers, 1) + assert.Equal(t, w.WorkflowData.Node.Triggers[0].ChildNode.Triggers[0].ChildNode.Name, "third") + } else { + assert.Equal(t, w.WorkflowData.Node.Triggers[1].ChildNode.Name, "fork") + assert.Len(t, w.WorkflowData.Node.Triggers[1].ChildNode.Triggers, 1) + assert.Equal(t, w.WorkflowData.Node.Triggers[1].ChildNode.Triggers[0].ChildNode.Name, "third") + } + } } diff --git a/engine/api/workflow_ascode.go b/engine/api/workflow_ascode.go index a66971bfb2..bc51fc3fcb 100644 --- a/engine/api/workflow_ascode.go +++ b/engine/api/workflow_ascode.go @@ -104,6 +104,32 @@ func (api *API) postWorkflowAsCodeHandler() service.Handler { return sdk.WithStack(sdk.ErrWorkflowAlreadyAsCode) } + // Check if there is a repository web hook + found := false + for _, h := range wf.WorkflowData.GetHooks() { + if h.HookModelName == sdk.RepositoryWebHookModelName { + found = true + break + } + } + if !found { + h := sdk.NodeHook{ + Config: sdk.RepositoryWebHookModel.DefaultConfig, + HookModelName: sdk.RepositoryWebHookModel.Name, + } + wf.WorkflowData.Node.Hooks = append(wf.WorkflowData.Node.Hooks, h) + + oldW, errOld := workflow.LoadByID(api.mustDB(), api.Cache, proj, wf.ID, u, workflow.LoadOptions{}) + if errOld != nil { + return errOld + } + + if err := workflow.Update(ctx, api.mustDB(), api.Cache, wf, proj, u, workflow.UpdateOptions{OldWorkflow: oldW}); err != nil { + return err + } + } + + // Export workflow + push + create pull request ope, err := workflow.UpdateAsCode(ctx, api.mustDB(), api.Cache, proj, wf, project.EncryptWithBuiltinKey, u) if err != nil { return sdk.WrapError(errW, "unable to migrate workflow as code") diff --git a/engine/api/workflow_ascode_test.go b/engine/api/workflow_ascode_test.go index d59385ddaf..ff4dd42517 100644 --- a/engine/api/workflow_ascode_test.go +++ b/engine/api/workflow_ascode_test.go @@ -97,11 +97,14 @@ func TestPostWorkflowAsCodeHandler(t *testing.T) { return writeError(w, err) } case "/task/bulk": - hooks := map[string]sdk.WorkflowNodeHook{} - h := sdk.WorkflowNodeHook{ - UUID: "ze", + var hooks map[string]sdk.NodeHook + bts, err := ioutil.ReadAll(r.Body) + if err != nil { + return writeError(w, err) + } + if err := json.Unmarshal(bts, &hooks); err != nil { + return writeError(w, err) } - hooks[h.UUID] = h if err := enc.Encode(hooks); err != nil { return writeError(w, err) } @@ -169,7 +172,6 @@ func TestPostWorkflowAsCodeHandler(t *testing.T) { }, } assert.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() var errP error proj, errP = project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithApplicationWithDeploymentStrategies, project.LoadOptions.WithPipelines, project.LoadOptions.WithEnvironments, project.LoadOptions.WithIntegrations) diff --git a/engine/api/workflow_export_test.go b/engine/api/workflow_export_test.go index 79f109018c..e3531c5b44 100644 --- a/engine/api/workflow_export_test.go +++ b/engine/api/workflow_export_test.go @@ -98,7 +98,7 @@ func Test_getWorkflowExportHandler(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() + proj, _ = project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) test.NoError(t, workflow.Insert(api.mustDB(), api.Cache, &w, proj, u)) @@ -219,7 +219,6 @@ func Test_getWorkflowExportHandlerWithPermissions(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() proj, _ = project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) @@ -323,7 +322,6 @@ func Test_getWorkflowPullHandler(t *testing.T) { } test.NoError(t, workflow.RenameNode(db, &w)) - (&w).RetroMigrate() proj, _ = project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) diff --git a/engine/api/workflow_group.go b/engine/api/workflow_group.go index b5e9fc937c..8e0a5ef99b 100644 --- a/engine/api/workflow_group.go +++ b/engine/api/workflow_group.go @@ -29,9 +29,7 @@ func (api *API) deleteWorkflowGroupHandler() service.Handler { return sdk.WrapError(err, "unable to load projet") } - options := workflow.LoadOptions{ - WithoutNode: true, - } + options := workflow.LoadOptions{} wf, err := workflow.Load(ctx, api.mustDB(), api.Cache, proj, name, u, options) if err != nil { return sdk.WrapError(err, "deleteWorkflowGroupHandler") @@ -95,9 +93,7 @@ func (api *API) putWorkflowGroupHandler() service.Handler { return sdk.WrapError(err, "unable to load projet") } - options := workflow.LoadOptions{ - WithoutNode: true, - } + options := workflow.LoadOptions{} wf, err := workflow.Load(ctx, api.mustDB(), api.Cache, proj, name, deprecatedGetUser(ctx), options) if err != nil { return sdk.WrapError(err, "putWorkflowGroupHandler") @@ -152,9 +148,7 @@ func (api *API) postWorkflowGroupHandler() service.Handler { return sdk.WrapError(err, "unable to load projet") } - options := workflow.LoadOptions{ - WithoutNode: true, - } + options := workflow.LoadOptions{} wf, err := workflow.Load(ctx, api.mustDB(), api.Cache, proj, name, deprecatedGetUser(ctx), options) if err != nil { return sdk.WrapError(err, "postWorkflowGroupHandler") diff --git a/engine/api/workflow_group_test.go b/engine/api/workflow_group_test.go index 85fe7c3b16..0525b4f8b6 100644 --- a/engine/api/workflow_group_test.go +++ b/engine/api/workflow_group_test.go @@ -49,8 +49,6 @@ func Test_postWorkflowGroupHandler(t *testing.T) { ProjectKey: proj.Key, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) test.NoError(t, errP) @@ -116,8 +114,6 @@ func Test_postWorkflowGroupWithLessThanRWXProjectHandler(t *testing.T) { ProjectKey: proj.Key, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) test.NoError(t, errP) @@ -177,7 +173,6 @@ func Test_putWorkflowGroupHandler(t *testing.T) { ProjectKey: proj.Key, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) test.NoError(t, errP) @@ -273,8 +268,6 @@ func Test_deleteWorkflowGroupHandler(t *testing.T) { ProjectKey: proj.Key, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) test.NoError(t, errP) @@ -351,8 +344,6 @@ func Test_UpdateProjectPermsWithWorkflow(t *testing.T) { ProjectKey: proj.Key, } - (&newWf).RetroMigrate() - //Prepare request vars := map[string]string{ "permProjectKey": proj.Key, @@ -437,8 +428,6 @@ func Test_PermissionOnWorkflowInferiorOfProject(t *testing.T) { ProjectKey: proj.Key, } - (&newWf).RetroMigrate() - //Prepare request to create workflow vars := map[string]string{ "permProjectKey": proj.Key, @@ -588,8 +577,6 @@ func Test_PermissionOnWorkflowWithRestrictionOnNode(t *testing.T) { ProjectKey: proj.Key, } - (&newWf).RetroMigrate() - //Prepare request to create workflow vars := map[string]string{ "permProjectKey": proj.Key, diff --git a/engine/api/workflow_hook_test.go b/engine/api/workflow_hook_test.go index 9814923d24..5d6ab52a94 100644 --- a/engine/api/workflow_hook_test.go +++ b/engine/api/workflow_hook_test.go @@ -56,7 +56,6 @@ func Test_getWorkflowHookModelsHandlerAsLambdaUser(t *testing.T) { }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, u)) //Prepare request @@ -125,7 +124,6 @@ func Test_getWorkflowHookModelsHandlerAsAdminUser(t *testing.T) { }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(db, cache, &w, proj, admin)) //Prepare request diff --git a/engine/api/workflow_import.go b/engine/api/workflow_import.go index 8725b40f57..e1f7ebb37e 100644 --- a/engine/api/workflow_import.go +++ b/engine/api/workflow_import.go @@ -81,8 +81,6 @@ func (api *API) postWorkflowPreviewHandler() service.Handler { return sdk.WrapError(err, "Unable to rename node") } - wf.RetroMigrate() - return service.WriteJSON(w, wf, http.StatusOK) } } @@ -164,6 +162,12 @@ func (api *API) postWorkflowImportHandler() service.Handler { return sdk.WrapError(err, "Cannot commit transaction") } + if wf != nil { + event.PublishWorkflowUpdate(proj.Key, *wrkflw, *wf, u) + } else { + event.PublishWorkflowAdd(proj.Key, *wrkflw, u) + } + if wrkflw != nil { w.Header().Add(sdk.ResponseWorkflowIDHeader, fmt.Sprintf("%d", wrkflw.ID)) w.Header().Add(sdk.ResponseWorkflowNameHeader, wrkflw.Name) diff --git a/engine/api/workflow_import_test.go b/engine/api/workflow_import_test.go index 9b1b42c17e..f350d09af6 100644 --- a/engine/api/workflow_import_test.go +++ b/engine/api/workflow_import_test.go @@ -75,13 +75,14 @@ metadata: assert.NotNil(t, w) m, _ := dump.ToStringMap(w) + t.Logf("%+v", m) assert.Equal(t, "test_1", m["Workflow.Name"]) - assert.Equal(t, "pip1", m["Workflow.Root.Name"]) - assert.Equal(t, "pip1", m["Workflow.Root.PipelineName"]) - assert.Equal(t, "name", m["Workflow.Root.Context.DefaultPipelineParameters.DefaultPipelineParameters0.Name"]) - assert.Equal(t, "value", m["Workflow.Root.Context.DefaultPipelineParameters.DefaultPipelineParameters0.Value"]) - assert.Equal(t, "pip1_2", m["Workflow.Root.Triggers.Triggers0.WorkflowDestNode.Name"]) - assert.Equal(t, "pip1", m["Workflow.Root.Triggers.Triggers0.WorkflowDestNode.PipelineName"]) + assert.Equal(t, "pip1", m["Workflow.WorkflowData.Node.Name"]) + assert.Equal(t, "pip1", m["Workflow.WorkflowData.Node.Context.PipelineName"]) + assert.Equal(t, "name", m["Workflow.WorkflowData.Node.Context.DefaultPipelineParameters.DefaultPipelineParameters0.Name"]) + assert.Equal(t, "value", m["Workflow.WorkflowData.Node.Context.DefaultPipelineParameters.DefaultPipelineParameters0.Value"]) + assert.Equal(t, "pip1_2", m["Workflow.WorkflowData.Node.Triggers.Triggers0.ChildNode.Name"]) + assert.Equal(t, "pip1", m["Workflow.WorkflowData.Node.Triggers.Triggers0.ChildNode.Context.PipelineName"]) assert.Equal(t, "git.branch,git.author,git.hash", m["Workflow.Metadata.default_tags"]) } @@ -668,7 +669,6 @@ func Test_getWorkflowPushHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj, _ = project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithApplications) test.NoError(t, workflow.Insert(api.mustDB(), api.Cache, &w, proj, u)) diff --git a/engine/api/workflow_queue_test.go b/engine/api/workflow_queue_test.go index 9781b26882..40ad04138a 100644 --- a/engine/api/workflow_queue_test.go +++ b/engine/api/workflow_queue_test.go @@ -100,8 +100,6 @@ func testRunWorkflow(t *testing.T, api *API, router *Router) testRunWorkflowCtx }, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups) test.NoError(t, errP) @@ -937,7 +935,6 @@ func TestPostVulnerabilityReportHandler(t *testing.T) { }, } - (&w).RetroMigrate() p, err := project.Load(db, api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithApplications) assert.NoError(t, err) assert.NoError(t, workflow.Insert(db, api.Cache, &w, p, u)) @@ -1080,8 +1077,6 @@ func TestInsertNewCodeCoverageReport(t *testing.T) { }, } - (&w).RetroMigrate() - p, err := project.Load(db, api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithApplications) assert.NoError(t, err) assert.NoError(t, workflow.Insert(db, api.Cache, &w, p, u)) diff --git a/engine/api/workflow_run.go b/engine/api/workflow_run.go index bb4721d516..eb78b907c3 100644 --- a/engine/api/workflow_run.go +++ b/engine/api/workflow_run.go @@ -226,9 +226,7 @@ func (api *API) postWorkflowRunNumHandler() service.Handler { return sdk.WrapError(err, "unable to load projet") } - options := workflow.LoadOptions{ - WithoutNode: true, - } + options := workflow.LoadOptions{} wf, errW := workflow.Load(ctx, api.mustDB(), api.Cache, proj, name, deprecatedGetUser(ctx), options) if errW != nil { return sdk.WrapError(errW, "postWorkflowRunNumHandler > Cannot load workflow") @@ -596,7 +594,6 @@ func (api *API) getWorkflowCommitsHandler() service.Handler { var app sdk.Application var env sdk.Environment var node *sdk.Node - var wNode *sdk.WorkflowNode if wf != nil { node = wf.WorkflowData.NodeByName(nodeName) if node == nil { @@ -628,9 +625,6 @@ func (api *API) getWorkflowCommitsHandler() service.Handler { } else { // Find hash and branch of ancestor node run var nodeIDsAncestors []int64 - if wNode != nil { - nodeIDsAncestors = wNode.Ancestors(&wfRun.Workflow, false) - } if node != nil { nodeIDsAncestors = node.Ancestors(wfRun.Workflow.WorkflowData) } @@ -821,10 +815,6 @@ func (api *API) postWorkflowRunHandler() service.Handler { if errlr != nil { return sdk.WrapError(errlr, "postWorkflowRunHandler> Unable to load workflow run") } - // MIGRATION TO NEW MODEL - if err := workflow.MigrateWorkflowRun(ctx, api.mustDB(), lastRun); err != nil { - return sdk.WrapError(err, "unable to migrate workflow run") - } } var wf *sdk.Workflow @@ -950,9 +940,6 @@ func (api *API) initWorkflowRun(ctx context.Context, db *gorp.DbMap, cache cache } } wfRun.Workflow = *wf - // TODO will be deleted with old struct - wfRun.Workflow.Root = nil - wfRun.Workflow.Joins = nil } r1, errS := workflow.StartWorkflowRun(ctx, db, cache, p, wfRun, opts, u, asCodeInfosMsg) @@ -989,7 +976,7 @@ func failInitWorkflowRun(ctx context.Context, db *gorp.DbMap, wfRun *sdk.Workflo wfRun.Status = sdk.StatusFail.String() info = sdk.SpawnMsg{ ID: sdk.MsgWorkflowError.ID, - Args: []interface{}{err.Error()}, + Args: []interface{}{sdk.Cause(err).Error()}, } } @@ -1047,9 +1034,7 @@ func (api *API) getDownloadArtifactHandler() service.Handler { return sdk.WrapError(err, "unable to load projet") } - options := workflow.LoadOptions{ - WithoutNode: true, - } + options := workflow.LoadOptions{} work, errW := workflow.Load(ctx, api.mustDB(), api.Cache, proj, name, deprecatedGetUser(ctx), options) if errW != nil { return sdk.WrapError(errW, "getDownloadArtifactHandler> Cannot load workflow") diff --git a/engine/api/workflow_run_test.go b/engine/api/workflow_run_test.go index 3d867ae41e..2903ccdbdb 100644 --- a/engine/api/workflow_run_test.go +++ b/engine/api/workflow_run_test.go @@ -106,7 +106,6 @@ func Test_getWorkflowNodeRunHistoryHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -233,7 +232,6 @@ func Test_getWorkflowRunsHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -387,8 +385,6 @@ func Test_getWorkflowRunsHandlerWithFilter(t *testing.T) { }, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -503,7 +499,6 @@ func Test_getLatestWorkflowRunHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -636,7 +631,6 @@ func Test_getWorkflowRunHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -760,7 +754,6 @@ func Test_getWorkflowNodeRunHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations, project.LoadOptions.WithApplications) test.NoError(t, errP) @@ -891,7 +884,6 @@ func Test_resyncWorkflowRunHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -1045,7 +1037,6 @@ func Test_postWorkflowRunHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -1123,7 +1114,6 @@ func Test_postWorkflowRunAsyncFailedHandler(t *testing.T) { }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithApplications, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -1187,8 +1177,8 @@ func Test_postWorkflowRunAsyncFailedHandler(t *testing.T) { return writeError(w, fmt.Errorf("error for test")) case "/task/bulk": - hooks := map[string]sdk.WorkflowNodeHook{} - hooks["123"] = sdk.WorkflowNodeHook{ + hooks := map[string]sdk.NodeHook{} + hooks["123"] = sdk.NodeHook{ UUID: "123", } if err := enc.Encode(hooks); err != nil { @@ -1345,8 +1335,6 @@ func Test_postWorkflowRunHandlerWithoutRightOnEnvironment(t *testing.T) { }, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithEnvironments) test.NoError(t, errP) @@ -1417,7 +1405,6 @@ func Test_postWorkflowRunHandler_Forbidden(t *testing.T) { }, }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(api.mustDB(), api.Cache, &w, proj2, u)) @@ -1486,7 +1473,6 @@ func Test_postWorkflowRunHandler_BadPayload(t *testing.T) { }, }, } - (&w).RetroMigrate() test.NoError(t, workflow.Insert(api.mustDB(), api.Cache, &w, proj2, u)) @@ -1588,7 +1574,6 @@ func initGetWorkflowNodeRunJobTest(t *testing.T, api *API, db *gorp.DbMap) (*sdk }, } - (&w).RetroMigrate() proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) @@ -1783,8 +1768,6 @@ func Test_deleteWorkflowRunsBranchHandler(t *testing.T) { }, } - (&w).RetroMigrate() - proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations) test.NoError(t, errP) diff --git a/engine/api/workflow_test.go b/engine/api/workflow_test.go index e5d8e32917..89830d8439 100644 --- a/engine/api/workflow_test.go +++ b/engine/api/workflow_test.go @@ -110,7 +110,6 @@ func Test_getWorkflowHandler_AsProvider(t *testing.T) { }, }, } - (&wf).RetroMigrate() test.NoError(t, workflow.Insert(api.mustDB(), api.Cache, &wf, proj, u)) @@ -169,7 +168,6 @@ func Test_getWorkflowHandler_withUsage(t *testing.T) { }, } - (&wf).RetroMigrate() test.NoError(t, workflow.Insert(db, api.Cache, &wf, proj, u)) req := assets.NewAuthentifiedRequest(t, u, pass, "GET", uri+"?withUsage=true", nil) @@ -831,13 +829,13 @@ func TestBenchmarkGetWorkflowsWithoutAPIAsAdmin(t *testing.T) { ProjectID: proj.ID, ProjectKey: proj.Key, Name: sdk.RandomString(10), - Root: &sdk.WorkflowNode{ - Name: "root", - PipelineID: pip.ID, - PipelineName: pip.Name, - Context: &sdk.WorkflowNodeContext{ - Application: &app, - ApplicationID: app.ID, + WorkflowData: &sdk.WorkflowData{ + Node: sdk.Node{ + Name: "root", + Context: &sdk.NodeContext{ + PipelineID: pip.ID, + ApplicationID: app.ID, + }, }, }, } @@ -896,13 +894,13 @@ func TestBenchmarkGetWorkflowsWithAPI(t *testing.T) { ProjectKey: proj.Key, Name: sdk.RandomString(10), Groups: proj.ProjectGroups, - Root: &sdk.WorkflowNode{ - Name: "root", - PipelineID: pip.ID, - PipelineName: pip.Name, - Context: &sdk.WorkflowNodeContext{ - Application: &app, - ApplicationID: app.ID, + WorkflowData: &sdk.WorkflowData{ + Node: sdk.Node{ + Name: "root", + Context: &sdk.NodeContext{ + PipelineID: pip.ID, + ApplicationID: app.ID, + }, }, }, } diff --git a/engine/hooks/hooks_handlers.go b/engine/hooks/hooks_handlers.go index 3b07192cdd..cf46c8e647 100644 --- a/engine/hooks/hooks_handlers.go +++ b/engine/hooks/hooks_handlers.go @@ -137,7 +137,7 @@ func (s *Service) stopTaskHandler() service.Handler { func (s *Service) postTaskHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { //This handler read a sdk.WorkflowNodeHook from the body - hook := &sdk.WorkflowNodeHook{} + hook := &sdk.NodeHook{} if err := service.UnmarshalBody(r, hook); err != nil { return sdk.WithStack(err) } @@ -296,8 +296,7 @@ func (s *Service) deleteTaskHandler() service.Handler { return sdk.WrapError(sdk.ErrNotFound, "Hooks> putTaskHandler> stop task") } - //Delete the task - s.Dao.DeleteTask(t) + s.deleteTask(ctx, t) return nil } @@ -367,7 +366,7 @@ func (s *Service) deleteAllTaskExecutionsHandler() service.Handler { func (s *Service) deleteTaskBulkHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { - hooks := map[string]sdk.WorkflowNodeHook{} + hooks := map[string]sdk.NodeHook{} if err := service.UnmarshalBody(r, &hooks); err != nil { return sdk.WithStack(err) } @@ -383,8 +382,7 @@ func (s *Service) deleteTaskBulkHandler() service.Handler { if err := s.stopTask(t); err != nil { return sdk.WrapError(sdk.ErrNotFound, "Stop task %s", err) } - //Delete the task - s.Dao.DeleteTask(t) + s.deleteTask(ctx, t) } return nil @@ -394,7 +392,7 @@ func (s *Service) deleteTaskBulkHandler() service.Handler { func (s *Service) postTaskBulkHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { //This handler read a sdk.WorkflowNodeHook from the body - hooks := map[string]sdk.WorkflowNodeHook{} + hooks := map[string]sdk.NodeHook{} if err := service.UnmarshalBody(r, &hooks); err != nil { return sdk.WithStack(err) } @@ -413,7 +411,7 @@ func (s *Service) postTaskBulkHandler() service.Handler { } } -func (s *Service) addTask(ctx context.Context, h *sdk.WorkflowNodeHook) error { +func (s *Service) addTask(ctx context.Context, h *sdk.NodeHook) error { //Parse the hook as a task t, err := s.hookToTask(h) if err != nil { @@ -450,7 +448,7 @@ func (s *Service) addAndExecuteTask(ctx context.Context, nr sdk.WorkflowNodeRun) var errNoTask = errors.New("task not found") -func (s *Service) updateTask(ctx context.Context, h *sdk.WorkflowNodeHook) error { +func (s *Service) updateTask(ctx context.Context, h *sdk.NodeHook) error { //Parse the hook as a task t, err := s.hookToTask(h) if err != nil { @@ -478,13 +476,7 @@ func (s *Service) updateTask(ctx context.Context, h *sdk.WorkflowNodeHook) error return nil } -func (s *Service) deleteTask(ctx context.Context, h *sdk.WorkflowNodeHook) error { - //Parse the hook as a task - t, err := s.hookToTask(h) - if err != nil { - return sdk.WrapError(err, "Unable to parse hook") - } - +func (s *Service) deleteTask(ctx context.Context, t *sdk.Task) { switch t.Type { case TypeGerrit: s.stopGerritHookTask(t) @@ -492,8 +484,6 @@ func (s *Service) deleteTask(ctx context.Context, h *sdk.WorkflowNodeHook) error //Delete the task s.Dao.DeleteTask(t) - - return nil } // Status returns sdk.MonitoringStatus, implements interface service.Service diff --git a/engine/hooks/scheduler.go b/engine/hooks/scheduler.go index 9e0af240c6..efe7062751 100644 --- a/engine/hooks/scheduler.go +++ b/engine/hooks/scheduler.go @@ -189,7 +189,7 @@ func (s *Service) deleteTaskExecutionsRoutine(c context.Context) error { } if taskToDelete { - s.Dao.DeleteTask(&t) + s.deleteTask(c, &t) } } diff --git a/engine/hooks/tasks.go b/engine/hooks/tasks.go index 977a057595..426230e840 100644 --- a/engine/hooks/tasks.go +++ b/engine/hooks/tasks.go @@ -90,7 +90,7 @@ func (s *Service) synchronizeTasks(ctx context.Context) error { } } if !found && t.Type != TypeOutgoingWebHook && t.Type != TypeOutgoingWorkflow { - s.Dao.DeleteTask(t) + s.deleteTask(ctx, t) log.Info("Hook> Task %s deleted on synchronization", t.UUID) } } @@ -104,7 +104,7 @@ func (s *Service) synchronizeTasks(ctx context.Context) error { } t, err := s.hookToTask(&h) if err != nil { - log.Error("Hook> Unable to synchronize task %+v: %v", h, err) + log.Error("Hook> Unable to transform hoot to tast task %+v: %v", h, err) continue } s.Dao.SaveTask(t) @@ -141,12 +141,8 @@ func (s *Service) initGerritStreamEvent(ctx context.Context, vcsName string, vcs gerritRepoHooks[vcsName] = true } -func (s *Service) hookToTask(h *sdk.WorkflowNodeHook) (*sdk.Task, error) { - if h.WorkflowHookModel.Type != sdk.WorkflowHookModelBuiltin { - return nil, fmt.Errorf("Unsupported hook type: %s", h.WorkflowHookModel.Type) - } - - switch h.WorkflowHookModel.Name { +func (s *Service) hookToTask(h *sdk.NodeHook) (*sdk.Task, error) { + switch h.HookModelName { case sdk.GerritHookModelName: return &sdk.Task{ UUID: h.UUID, @@ -203,8 +199,7 @@ func (s *Service) hookToTask(h *sdk.WorkflowNodeHook) (*sdk.Task, error) { Type: TypeWorkflowHook, }, nil } - - return nil, fmt.Errorf("Unsupported hook: %s", h.WorkflowHookModel.Name) + return nil, fmt.Errorf("Unsupported hook: %s", h.HookModelName) } func (s *Service) startTasks(ctx context.Context) error { diff --git a/engine/sql/168_disabledConstraint.sql b/engine/sql/168_disabledConstraint.sql new file mode 100644 index 0000000000..5526d7912e --- /dev/null +++ b/engine/sql/168_disabledConstraint.sql @@ -0,0 +1,13 @@ +-- +migrate Up +ALTER TABLE workflow DROP CONSTRAINT fk_workflow_root_node; +ALTER TABLE workflow_node DROP CONSTRAINT fk_workflow_node_pipeline; +ALTER TABLE workflow_node DROP CONSTRAINT fk_workflow_node_workflow; + +ALTER TABLE workflow_notification_source DROP CONSTRAINT fk_workflow_notification_source_node; +ALTER TABLE workflow_notification_source DROP CONSTRAINT workflow_notification_source_pkey; +ALTER TABLE workflow_notification_source ADD PRIMARY KEY (workflow_notification_id, node_id); +ALTER TABLE workflow_notification_source ALTER COLUMN workflow_node_id DROP NOT NULL; + +-- +migrate Down +SELECT 1; + diff --git a/sdk/cdsclient/client_workflow_hooks.go b/sdk/cdsclient/client_workflow_hooks.go index 3bf9c33c11..704469df5f 100644 --- a/sdk/cdsclient/client_workflow_hooks.go +++ b/sdk/cdsclient/client_workflow_hooks.go @@ -7,9 +7,9 @@ import ( "github.com/ovh/cds/sdk" ) -func (c *client) WorkflowAllHooksList() ([]sdk.WorkflowNodeHook, error) { +func (c *client) WorkflowAllHooksList() ([]sdk.NodeHook, error) { url := fmt.Sprintf("/workflow/hook") - w := []sdk.WorkflowNodeHook{} + var w []sdk.NodeHook if _, err := c.GetJSON(context.Background(), url, &w); err != nil { return nil, err } diff --git a/sdk/cdsclient/interface.go b/sdk/cdsclient/interface.go index 0776c391a6..06e082f371 100644 --- a/sdk/cdsclient/interface.go +++ b/sdk/cdsclient/interface.go @@ -301,7 +301,7 @@ type WorkflowClient interface { WorkflowNodeRunArtifactDownload(projectKey string, name string, a sdk.WorkflowNodeRunArtifact, w io.Writer) error WorkflowNodeRunJobStep(projectKey string, workflowName string, number int64, nodeRunID, job int64, step int) (*sdk.BuildState, error) WorkflowNodeRunRelease(projectKey string, workflowName string, runNumber int64, nodeRunID int64, release sdk.WorkflowNodeRunRelease) error - WorkflowAllHooksList() ([]sdk.WorkflowNodeHook, error) + WorkflowAllHooksList() ([]sdk.NodeHook, error) WorkflowCachePush(projectKey, integrationName, ref string, tarContent io.Reader, size int) error WorkflowCachePull(projectKey, integrationName, ref string) (io.Reader, error) WorkflowTemplateInstanceGet(projectKey, workflowName string) (*sdk.WorkflowTemplateInstance, error) diff --git a/sdk/error.go b/sdk/error.go index 141a8ff961..f69cec2260 100644 --- a/sdk/error.go +++ b/sdk/error.go @@ -191,6 +191,8 @@ var ( ErrApplicationMandatoryOnWorkflowAsCode = Error{ID: 174, Status: http.StatusBadRequest} ErrInvalidPassword = Error{ID: 175, Status: http.StatusBadRequest} ErrInvalidPayloadVariable = Error{ID: 176, Status: http.StatusBadRequest} + ErrRepositoryUsedByHook = Error{ID: 177, Status: http.StatusForbidden} + ErrResourceNotInProject = Error{ID: 178, Status: http.StatusForbidden} ) var errorsAmericanEnglish = map[int]string{ @@ -364,6 +366,8 @@ var errorsAmericanEnglish = map[int]string{ ErrApplicationMandatoryOnWorkflowAsCode.ID: "An application linked to a git repository is mandatory on the workflow root", ErrInvalidPayloadVariable.ID: "Your payload cannot contain keys like cds.*", ErrInvalidPassword.ID: "Your value of type password isn't correct", + ErrRepositoryUsedByHook.ID: "There is still a hook on this repository", + ErrResourceNotInProject.ID: "The resource is not attached to the project", } var errorsFrench = map[int]string{ @@ -537,6 +541,8 @@ var errorsFrench = map[int]string{ ErrApplicationMandatoryOnWorkflowAsCode.ID: "Une application liée à un dépôt git est obligatoire à la racine du workflow", ErrInvalidPayloadVariable.ID: "Le payload du workflow ne peut pas contenir de clés nommées cds.*", ErrInvalidPassword.ID: "Votre valeur de type mot de passe n'est pas correct", + ErrRepositoryUsedByHook.ID: "Il y a encore un hook sur ce dépôt", + ErrResourceNotInProject.ID: "La ressource n'est pas lié au projet", } var errorsLanguages = []map[int]string{ diff --git a/sdk/exportentities/workflow.go b/sdk/exportentities/workflow.go index 34aca100f7..7b793bc4b8 100644 --- a/sdk/exportentities/workflow.go +++ b/sdk/exportentities/workflow.go @@ -255,12 +255,12 @@ func WorkflowSkipIfOnlyOneRepoWebhook(w sdk.Workflow, exportedWorkflow *Workflow } for nodeName, hs := range exportedWorkflow.Hooks { - if nodeName == w.Root.Name && len(hs) == 1 { + if nodeName == w.WorkflowData.Node.Name && len(hs) == 1 { if hs[0].Model == sdk.RepositoryWebHookModelName { delete(exportedWorkflow.Hooks, nodeName) if exportedWorkflow.Workflow != nil { for nodeName := range exportedWorkflow.Workflow { - if nodeName == w.Root.Name { + if nodeName == w.WorkflowData.Node.Name { entry := exportedWorkflow.Workflow[nodeName] entry.Payload = nil exportedWorkflow.Workflow[nodeName] = entry diff --git a/sdk/workflow.go b/sdk/workflow.go index ed630ff796..20ed585961 100644 --- a/sdk/workflow.go +++ b/sdk/workflow.go @@ -7,8 +7,6 @@ import ( "regexp" "sort" "time" - - "github.com/fsamin/go-dump" ) // DefaultHistoryLength is the default history length @@ -28,9 +26,6 @@ type Workflow struct { LastModified time.Time `json:"last_modified" db:"last_modified" mapstructure:"-"` ProjectID int64 `json:"project_id,omitempty" db:"project_id" cli:"-"` ProjectKey string `json:"project_key" db:"-" cli:"-"` - RootID int64 `json:"root_id,omitempty" db:"root_node_id" cli:"-"` - Root *WorkflowNode `json:"root,omitempty" db:"-" cli:"-"` - Joins []WorkflowNodeJoin `json:"joins,omitempty" db:"-" cli:"-"` Groups []GroupPermission `json:"groups,omitempty" db:"-" cli:"-"` Permission int `json:"permission,omitempty" db:"-" cli:"-"` Metadata Metadata `json:"metadata" yaml:"metadata" db:"-"` @@ -77,150 +72,23 @@ func (w *Workflow) GetApplication(ID int64) Application { return w.Applications[ID] } -// RetroMigrate temporary method that convert new workflow structure into old workflow structure for backward compatibility -func (w *Workflow) RetroMigrate() { - root := w.WorkflowData.Node.retroMigrate() - w.Root = &root - - w.Joins = nil - if len(w.WorkflowData.Joins) > 0 { - w.Joins = make([]WorkflowNodeJoin, 0, len(w.WorkflowData.Joins)) - for _, j := range w.WorkflowData.Joins { - w.Joins = append(w.Joins, j.retroMigrateJoin()) - } - } - - // Set context on old node - for _, n := range w.Nodes(true) { - node := w.GetNodeByName(n.Name) - if node.Context == nil { - continue - } - - if node.Context.ApplicationID != 0 { - app, ok := w.Applications[node.Context.ApplicationID] - if ok { - node.Context.Application = &app - } - } - if node.Context.EnvironmentID != 0 { - env, ok := w.Environments[node.Context.EnvironmentID] - if ok { - node.Context.Environment = &env - } - } - if node.Context.ProjectIntegrationID != 0 { - pp, ok := w.ProjectIntegrations[node.Context.ProjectIntegrationID] - if ok { - node.Context.ProjectIntegration = &pp - } - } - } -} - -// Migrate old workflow struct into new workflow struct -func (w *Workflow) Migrate(withID bool) WorkflowData { - work := WorkflowData{} - - if w != nil && w.Root != nil { - // Add root node - work.Node = (*w.Root).migrate(withID) - - // Add Join - work.Joins = make([]Node, 0, len(w.Joins)) - for _, j := range w.Joins { - work.Joins = append(work.Joins, j.migrate(withID)) - } - } - return work -} - // WorkflowNotification represents notifications on a workflow type WorkflowNotification struct { ID int64 `json:"id,omitempty" db:"id"` WorkflowID int64 `json:"workflow_id,omitempty" db:"workflow_id"` SourceNodeRefs []string `json:"source_node_ref,omitempty" db:"-"` - SourceNodeIDs []int64 `json:"source_node_id,omitempty" db:"-"` NodeIDs []int64 `json:"node_id,omitempty" db:"-"` Type string `json:"type" db:"type"` Settings UserNotificationSettings `json:"settings" db:"-"` } -func (w *Workflow) Forks() (map[int64]WorkflowNodeFork, map[int64]string) { - forkMap := make(map[int64]WorkflowNodeFork, 0) - forkTriggerMap := make(map[int64]string, 0) - w.Root.ForksMap(&forkMap, &forkTriggerMap) - for _, j := range w.Joins { - for _, t := range j.Triggers { - (&t.WorkflowDestNode).ForksMap(&forkMap, &forkTriggerMap) - } - } - return forkMap, forkTriggerMap -} - -//JoinsID returns joins ID -func (w *Workflow) JoinsID() []int64 { - res := make([]int64, len(w.Joins)) - for i, j := range w.Joins { - res[i] = j.ID - } - return res -} - // ResetIDs resets all nodes and joins ids func (w *Workflow) ResetIDs() { - if w.Root == nil { - return - } - (w.Root).ResetIDs() - for i := range w.Joins { - j := &w.Joins[i] - j.ID = 0 - j.SourceNodeIDs = nil - for tid := range j.Triggers { - t := &j.Triggers[tid] - (&t.WorkflowDestNode).ResetIDs() - } - } - for _, n := range w.WorkflowData.Array() { n.ID = 0 } } -//Nodes returns nodes IDs excluding the root ID -func (w *Workflow) Nodes(withRoot bool) []WorkflowNode { - if w.Root == nil { - return nil - } - - res := []WorkflowNode{} - if withRoot { - res = append(res, w.Root.Nodes()...) - } else { - for _, t := range w.Root.Triggers { - res = append(res, t.WorkflowDestNode.Nodes()...) - } - for _, f := range w.Root.Forks { - for _, t := range f.Triggers { - res = append(res, t.WorkflowDestNode.Nodes()...) - } - } - for i := range w.Root.OutgoingHooks { - for j := range w.Root.OutgoingHooks[i].Triggers { - res = append(res, w.Root.OutgoingHooks[i].Triggers[j].WorkflowDestNode.Nodes()...) - } - } - } - - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.Nodes()...) - } - } - return res -} - //AddTrigger adds a trigger to the destination node from the node found by its name func (w *Workflow) AddTrigger(name string, dest Node) { if w.WorkflowData == nil || w.WorkflowData.Node.Name == "" { @@ -233,222 +101,10 @@ func (w *Workflow) AddTrigger(name string, dest Node) { } } -//GetNodeByRef returns the node given its ref -func (w *Workflow) GetNodeByRef(ref string) *WorkflowNode { - n := w.Root.GetNodeByRef(ref) - if n != nil { - return n - } - for ji := range w.Joins { - j := &w.Joins[ji] - for ti := range j.Triggers { - t := &j.Triggers[ti] - n2 := (&t.WorkflowDestNode).GetNodeByRef(ref) - if n2 != nil { - return n2 - } - } - } - return nil -} - -func (w *Workflow) GetForkByName(name string) *WorkflowNodeFork { - n := w.Root.GetForkByName(name) - if n != nil { - return n - } - for _, j := range w.Joins { - for _, t := range j.Triggers { - n = t.WorkflowDestNode.GetForkByName(name) - if n != nil { - return n - } - } - } - return nil -} - -//GetNodeByName returns the node given its name -func (w *Workflow) GetNodeByName(name string) *WorkflowNode { - n := w.Root.GetNodeByName(name) - if n != nil { - return n - } - for _, j := range w.Joins { - for _, t := range j.Triggers { - n = t.WorkflowDestNode.GetNodeByName(name) - if n != nil { - return n - } - } - } - return nil -} - -//GetNode returns the node given its id -func (w *Workflow) GetNode(id int64) *WorkflowNode { - n := w.Root.GetNode(id) - if n != nil { - return n - } - for _, j := range w.Joins { - for _, t := range j.Triggers { - n = t.WorkflowDestNode.GetNode(id) - if n != nil { - return n - } - } - } - return nil -} - -//GetJoin returns the join given its id -func (w *Workflow) GetJoin(id int64) *WorkflowNodeJoin { - for _, j := range w.Joins { - if j.ID == id { - return &j - } - } - return nil -} - -//TriggersID returns triggers IDs -func (w *Workflow) TriggersID() []int64 { - res := w.Root.TriggersID() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.ID) - res = append(res, t.WorkflowDestNode.TriggersID()...) - } - } - return res -} - -//References returns a slice with all node references -func (w *Workflow) References() []string { - if w.Root == nil { - return nil - } - - res := w.Root.References() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.References()...) - } - } - return res -} - -//InvolvedApplications returns all applications used in the workflow -func (w *Workflow) InvolvedApplications() []int64 { - if w.Root == nil { - return nil - } - - res := w.Root.InvolvedApplications() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedApplications()...) - } - } - return res -} - -//InvolvedPipelines returns all pipelines used in the workflow -func (w *Workflow) InvolvedPipelines() []int64 { - if w.Root == nil { - return nil - } - - res := w.Root.InvolvedPipelines() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedPipelines()...) - } - } - return res -} - -//GetApplications returns all applications used in the workflow -func (w *Workflow) GetApplications() []Application { - if w.Root == nil { - return nil - } - - res := w.Root.GetApplications() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.GetApplications()...) - } - } - - withoutDuplicates := []Application{} - for _, a := range res { - var found bool - for _, d := range withoutDuplicates { - if a.Name == d.Name { - found = true - break - } - } - if !found { - withoutDuplicates = append(withoutDuplicates, a) - } - } - - return withoutDuplicates -} - -//GetEnvironments returns all environments used in the workflow -func (w *Workflow) GetEnvironments() []Environment { - if w.Root == nil { - return nil - } - - res := w.Root.GetEnvironments() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.GetEnvironments()...) - } - } - - withoutDuplicates := []Environment{} - for _, a := range res { - var found bool - for _, d := range withoutDuplicates { - if a.Name == d.Name { - found = true - break - } - } - if !found { - withoutDuplicates = append(withoutDuplicates, a) - } - } - - return withoutDuplicates -} - -//GetPipelines returns all pipelines used in the workflow -func (w *Workflow) GetPipelines() []Pipeline { - if w.Root == nil { - return nil - } - - res := make([]Pipeline, len(w.Pipelines)) - var i int - for _, p := range w.Pipelines { - res[i] = p - i++ - } - return res -} - // GetRepositories returns the list of repositories from applications func (w *Workflow) GetRepositories() []string { - apps := w.GetApplications() repos := map[string]struct{}{} - for _, a := range apps { + for _, a := range w.Applications { if a.RepositoryFullname != "" { repos[a.RepositoryFullname] = struct{}{} } @@ -462,47 +118,6 @@ func (w *Workflow) GetRepositories() []string { return res } -//InvolvedEnvironments returns all environments used in the workflow -func (w *Workflow) InvolvedEnvironments() []int64 { - if w.Root == nil { - return nil - } - - res := w.Root.InvolvedEnvironments() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedEnvironments()...) - } - } - return res -} - -//InvolvedIntegrations returns all integrations used in the workflow -func (w *Workflow) InvolvedIntegrations() []int64 { - if w.Root == nil { - return nil - } - - res := w.Root.InvolvedIntegrations() - for _, j := range w.Joins { - for _, t := range j.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedIntegrations()...) - } - } - return res -} - -//Visit all the workflow and apply the visitor func on all nodes -func (w *Workflow) Visit(visitor func(*WorkflowNode)) { - w.Root.Visit(visitor) - for i := range w.Joins { - for j := range w.Joins[i].Triggers { - n := &w.Joins[i].Triggers[j].WorkflowDestNode - n.Visit(visitor) - } - } -} - //Visit all the workflow and apply the visitor func on all nodes func (w *Workflow) VisitNode(visitor func(*Node, *Workflow)) { w.WorkflowData.Node.VisitNode(w, visitor) @@ -526,18 +141,6 @@ func (w *Workflow) SortNode() { } } -//Sort sorts the workflow -func (w *Workflow) Sort() { - w.Visit(func(n *WorkflowNode) { - n.Sort() - }) - for _, join := range w.Joins { - sort.Slice(join.Triggers, func(i, j int) bool { - return join.Triggers[i].WorkflowDestNode.Name < join.Triggers[j].WorkflowDestNode.Name - }) - } -} - // AssignEmptyType fill node type field func (w *Workflow) AssignEmptyType() { // set node type for join @@ -595,892 +198,6 @@ func (w *Workflow) ValidateType() error { return nil } -//Visit all the workflow and apply the visitor func on the current node and the children -func (n *WorkflowNode) Visit(visitor func(*WorkflowNode)) { - visitor(n) - for i := range n.Triggers { - d := &n.Triggers[i].WorkflowDestNode - d.Visit(visitor) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - d := &n.OutgoingHooks[i].Triggers[j].WorkflowDestNode - d.Visit(visitor) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - d := &n.Forks[i].Triggers[j].WorkflowDestNode - d.Visit(visitor) - } - } -} - -//Sort sorts the workflow node -func (n *WorkflowNode) Sort() { - sort.Slice(n.Triggers, func(i, j int) bool { - return n.Triggers[i].WorkflowDestNode.Name < n.Triggers[j].WorkflowDestNode.Name - }) -} - -//WorkflowNodeJoin aims to joins multiple node into multiple triggers -type WorkflowNodeJoin struct { - ID int64 `json:"id" db:"id"` - Ref string `json:"ref" db:"-"` - WorkflowID int64 `json:"workflow_id" db:"workflow_id"` - SourceNodeIDs []int64 `json:"source_node_id,omitempty" db:"-"` - SourceNodeRefs []string `json:"source_node_ref,omitempty" db:"-"` - Triggers []WorkflowNodeJoinTrigger `json:"triggers,omitempty" db:"-"` -} - -func (j WorkflowNodeJoin) migrate(withID bool) Node { - newNode := Node{ - Name: j.Ref, - Ref: j.Ref, - Type: NodeTypeJoin, - JoinContext: make([]NodeJoin, 0, len(j.SourceNodeIDs)), - Triggers: make([]NodeTrigger, 0, len(j.Triggers)), - } - if newNode.Ref == "" { - newNode.Ref = RandomString(5) - } - if withID { - newNode.ID = j.ID - } - for i := range j.SourceNodeIDs { - newNode.JoinContext = append(newNode.JoinContext, NodeJoin{ - ParentID: j.SourceNodeIDs[i], - ParentName: j.SourceNodeRefs[i], - }) - } - - for _, t := range j.Triggers { - child := t.WorkflowDestNode.migrate(withID) - newNode.Triggers = append(newNode.Triggers, NodeTrigger{ - ParentNodeName: newNode.Name, - ChildNode: child, - }) - } - return newNode -} - -//WorkflowNodeJoinTrigger is a trigger for joins -type WorkflowNodeJoinTrigger struct { - ID int64 `json:"id" db:"id"` - WorkflowNodeJoinID int64 `json:"join_id" db:"workflow_node_join_id"` - WorkflowDestNodeID int64 `json:"workflow_dest_node_id" db:"workflow_dest_node_id"` - WorkflowDestNode WorkflowNode `json:"workflow_dest_node" db:"-"` -} - -//WorkflowNode represents a node in w workflow tree -type WorkflowNode struct { - ID int64 `json:"id" db:"id"` - Name string `json:"name" db:"name"` - Ref string `json:"ref,omitempty" db:"-"` - WorkflowID int64 `json:"workflow_id" db:"workflow_id"` - PipelineID int64 `json:"pipeline_id" db:"pipeline_id"` - PipelineName string `json:"pipeline_name" db:"-"` - DeprecatedPipeline Pipeline `json:"pipeline" db:"-"` - Context *WorkflowNodeContext `json:"context" db:"-"` - TriggerSrcID int64 `json:"-" db:"-"` - TriggerJoinSrcID int64 `json:"-" db:"-"` - TriggerHookSrcID int64 `json:"-" db:"-"` - TriggerSrcForkID int64 `json:"-" db:"-"` - Hooks []WorkflowNodeHook `json:"hooks,omitempty" db:"-"` - Forks []WorkflowNodeFork `json:"forks,omitempty" db:"-"` - Triggers []WorkflowNodeTrigger `json:"triggers,omitempty" db:"-"` - OutgoingHooks []WorkflowNodeOutgoingHook `json:"outgoing_hooks,omitempty" db:"-"` -} - -func (n Node) retroMigrate() WorkflowNode { - newNode := WorkflowNode{ - Ref: n.Ref, - Name: n.Name, - WorkflowID: n.WorkflowID, - Context: &WorkflowNodeContext{ - ProjectIntegrationID: n.Context.ProjectIntegrationID, - EnvironmentID: n.Context.EnvironmentID, - ApplicationID: n.Context.ApplicationID, - DefaultPipelineParameters: n.Context.DefaultPipelineParameters, - DefaultPayload: n.Context.DefaultPayload, - Mutex: n.Context.Mutex, - Conditions: n.Context.Conditions, - }, - PipelineID: n.Context.PipelineID, - OutgoingHooks: nil, - Hooks: make([]WorkflowNodeHook, 0, len(n.Hooks)), - Triggers: nil, - Forks: nil, - } - - for _, h := range n.Hooks { - hook := WorkflowNodeHook{ - UUID: h.UUID, - Ref: h.Ref, - WorkflowHookModelID: h.HookModelID, - Config: h.Config, - } - newNode.Hooks = append(newNode.Hooks, hook) - } - - for _, t := range n.Triggers { - switch t.ChildNode.Type { - case NodeTypePipeline: - trig := WorkflowNodeTrigger{ - WorkflowDestNode: t.ChildNode.retroMigrate(), - } - newNode.Triggers = append(newNode.Triggers, trig) - case NodeTypeFork: - newNode.Forks = append(newNode.Forks, t.ChildNode.retroMigrateFork()) - case NodeTypeOutGoingHook: - newNode.OutgoingHooks = append(newNode.OutgoingHooks, t.ChildNode.retroMigrateOutGoingHook()) - } - } - return newNode -} - -func (n Node) retroMigrateFork() WorkflowNodeFork { - fork := WorkflowNodeFork{ - Name: n.Name, - } - if len(n.Triggers) > 0 { - fork.Triggers = make([]WorkflowNodeForkTrigger, 0, len(n.Triggers)) - } - for _, t := range n.Triggers { - trig := WorkflowNodeForkTrigger{} - switch t.ChildNode.Type { - case NodeTypePipeline: - trig.WorkflowDestNode = t.ChildNode.retroMigrate() - default: - continue - } - fork.Triggers = append(fork.Triggers, trig) - } - return fork -} - -func (n Node) retroMigrateOutGoingHook() WorkflowNodeOutgoingHook { - h := WorkflowNodeOutgoingHook{ - Config: n.OutGoingHookContext.Config, - WorkflowHookModelID: n.OutGoingHookContext.HookModelID, - Ref: n.Ref, - Name: n.Name, - } - if len(n.Triggers) > 0 { - h.Triggers = make([]WorkflowNodeOutgoingHookTrigger, 0, len(n.Triggers)) - for _, t := range n.Triggers { - trig := WorkflowNodeOutgoingHookTrigger{} - switch t.ChildNode.Type { - case NodeTypePipeline: - trig.WorkflowDestNode = t.ChildNode.retroMigrate() - default: - continue - } - h.Triggers = append(h.Triggers, trig) - } - } - return h -} - -func (n Node) retroMigrateJoin() WorkflowNodeJoin { - j := WorkflowNodeJoin{ - Ref: n.Ref, - } - - j.SourceNodeRefs = make([]string, 0, len(n.JoinContext)) - for _, jc := range n.JoinContext { - j.SourceNodeRefs = append(j.SourceNodeRefs, jc.ParentName) - } - - if len(n.Triggers) > 0 { - j.Triggers = make([]WorkflowNodeJoinTrigger, 0, len(n.Triggers)) - for _, t := range n.Triggers { - trig := WorkflowNodeJoinTrigger{} - switch t.ChildNode.Type { - case NodeTypePipeline: - trig.WorkflowDestNode = t.ChildNode.retroMigrate() - default: - continue - } - j.Triggers = append(j.Triggers, trig) - } - } - - return j -} - -func (n WorkflowNode) migrate(withID bool) Node { - newNode := Node{ - WorkflowID: n.WorkflowID, - Type: NodeTypePipeline, - Name: n.Name, - Ref: n.Ref, - Context: &NodeContext{ - PipelineID: n.PipelineID, - ApplicationID: n.Context.ApplicationID, - EnvironmentID: n.Context.EnvironmentID, - ProjectIntegrationID: n.Context.ProjectIntegrationID, - Conditions: n.Context.Conditions, - DefaultPayload: n.Context.DefaultPayload, - DefaultPipelineParameters: n.Context.DefaultPipelineParameters, - Mutex: n.Context.Mutex, - }, - Hooks: make([]NodeHook, 0, len(n.Hooks)), - Triggers: make([]NodeTrigger, 0, len(n.Triggers)+len(n.Forks)+len(n.OutgoingHooks)), - } - if n.Context.ApplicationID == 0 && n.Context.Application != nil { - newNode.Context.ApplicationID = n.Context.Application.ID - } - if n.Context.EnvironmentID == 0 && n.Context.Environment != nil { - newNode.Context.EnvironmentID = n.Context.Environment.ID - } - if n.Context.ProjectIntegrationID == 0 && n.Context.ProjectIntegration != nil { - newNode.Context.ProjectIntegrationID = n.Context.ProjectIntegration.ID - } - if withID { - newNode.ID = n.ID - } - if n.Ref == "" { - n.Ref = n.Name - } - - for _, h := range n.Hooks { - nh := NodeHook{ - Ref: h.Ref, - HookModelID: h.WorkflowHookModelID, - Config: h.Config, - UUID: h.UUID, - } - if withID { - nh.ID = h.ID - } - newNode.Hooks = append(newNode.Hooks, nh) - } - - for _, t := range n.Triggers { - triggeredNode := t.WorkflowDestNode.migrate(withID) - newNode.Triggers = append(newNode.Triggers, NodeTrigger{ - ParentNodeName: n.Name, - ChildNode: triggeredNode, - }) - } - - for _, f := range n.Forks { - forkNode := f.migrate(withID) - newNode.Triggers = append(newNode.Triggers, NodeTrigger{ - ParentNodeName: n.Name, - ChildNode: forkNode, - }) - } - - for _, h := range n.OutgoingHooks { - ogh := h.migrate(withID) - newNode.Triggers = append(newNode.Triggers, NodeTrigger{ - ParentNodeName: n.Name, - ChildNode: ogh, - }) - } - - return newNode -} - -func (n *WorkflowNode) ForksMap(forkMap *map[int64]WorkflowNodeFork, triggerMap *map[int64]string) { - for _, f := range n.Forks { - (*forkMap)[f.ID] = f - for _, t := range f.Triggers { - (*triggerMap)[t.ID] = f.Name - (&t.WorkflowDestNode).ForksMap(forkMap, triggerMap) - } - } - for _, t := range n.Triggers { - (&t.WorkflowDestNode).ForksMap(forkMap, triggerMap) - } - for _, o := range n.OutgoingHooks { - for _, t := range o.Triggers { - (&t.WorkflowDestNode).ForksMap(forkMap, triggerMap) - } - } -} - -// IsLinkedToRepo returns boolean to know if the node is linked to an application which is also linked to a repository -func (n *WorkflowNode) IsLinkedToRepo() bool { - if n == nil { - return false - } - return n.Context != nil && n.Context.Application != nil && n.Context.Application.RepositoryFullname != "" -} - -// Application return an application and a boolean (false if no application) -func (n *WorkflowNode) Application() (a Application, b bool) { - if n == nil { - return a, false - } - if n.Context == nil { - return a, false - } - if n.Context.Application == nil { - return a, false - } - return *n.Context.Application, true -} - -// Environment return an environment and a boolean (false if no environment) -func (n *WorkflowNode) Environment() (e Environment, b bool) { - if n == nil { - return e, false - } - if n.Context == nil { - return e, false - } - if n.Context.Environment == nil { - return e, false - } - return *n.Context.Environment, true -} - -// ProjectIntegration return an ProjectIntegration and a boolean (false if no ProjectIntegration) -func (n *WorkflowNode) ProjectIntegration() (p ProjectIntegration, b bool) { - if n == nil { - return p, false - } - if n.Context == nil { - return p, false - } - if n.Context.ProjectIntegration == nil { - return p, false - } - return *n.Context.ProjectIntegration, true -} - -// EqualsTo returns true if a node has the same pipeline and context than another -func (n *WorkflowNode) EqualsTo(n1 *WorkflowNode) bool { - if n.PipelineID != n1.PipelineID { - return false - } - if n.Context == nil && n1.Context != nil { - return false - } - if n.Context != nil && n1.Context == nil { - return false - } - if n.Context.ApplicationID != n1.Context.ApplicationID { - return false - } - if n.Context.EnvironmentID != n1.Context.EnvironmentID { - return false - } - return true -} - -//GetNodeByRef returns the node given its ref -func (n *WorkflowNode) GetNodeByRef(ref string) *WorkflowNode { - if n == nil { - return nil - } - if n.Ref == ref { - return n - } - for i := range n.Triggers { - t := &n.Triggers[i] - n2 := (&t.WorkflowDestNode).GetNodeByRef(ref) - if n2 != nil { - return n2 - } - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - n2 := (&n.OutgoingHooks[i].Triggers[j].WorkflowDestNode).GetNodeByRef(ref) - if n2 != nil { - return n2 - } - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - n2 := (&n.Forks[i].Triggers[j].WorkflowDestNode).GetNodeByRef(ref) - if n2 != nil { - return n2 - } - } - } - - return nil -} - -func (n *WorkflowNode) GetForkByName(name string) *WorkflowNodeFork { - if n == nil { - return nil - } - for i := range n.Forks { - f := &n.Forks[i] - if f.Name == name { - return f - } - - for j := range f.Triggers { - f2 := (&f.Triggers[j].WorkflowDestNode).GetForkByName(name) - if f2 != nil { - return f2 - } - } - } - - for j := range n.Triggers { - n2 := (&n.Triggers[j].WorkflowDestNode).GetForkByName(name) - if n2 != nil { - return n2 - } - } - - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - n2 := (&n.OutgoingHooks[i].Triggers[j].WorkflowDestNode).GetForkByName(name) - if n2 != nil { - return n2 - } - } - } - return nil -} - -//GetNodeByName returns the node given its name -func (n *WorkflowNode) GetNodeByName(name string) *WorkflowNode { - if n == nil { - return nil - } - if n.Name == name { - return n - } - for _, t := range n.Triggers { - n2 := t.WorkflowDestNode.GetNodeByName(name) - if n2 != nil { - return n2 - } - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - n2 := (&n.OutgoingHooks[i].Triggers[j].WorkflowDestNode).GetNodeByName(name) - if n2 != nil { - return n2 - } - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - n2 := (&n.Forks[i].Triggers[j].WorkflowDestNode).GetNodeByName(name) - if n2 != nil { - return n2 - } - } - } - return nil -} - -//GetNode returns the node given its id -func (n *WorkflowNode) GetNode(id int64) *WorkflowNode { - if n == nil { - return nil - } - if n.ID == id { - return n - } - for _, t := range n.Triggers { - n1 := t.WorkflowDestNode.GetNode(id) - if n1 != nil { - return n1 - } - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - n2 := (&n.OutgoingHooks[i].Triggers[j].WorkflowDestNode).GetNode(id) - if n2 != nil { - return n2 - } - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - n2 := (&n.Forks[i].Triggers[j].WorkflowDestNode).GetNode(id) - if n2 != nil { - return n2 - } - } - } - return nil -} - -// ResetIDs resets node id for the following node and its triggers -func (n *WorkflowNode) ResetIDs() { - n.ID = 0 - for i := range n.Triggers { - t := &n.Triggers[i] - (&t.WorkflowDestNode).ResetIDs() - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - (&n.OutgoingHooks[i].Triggers[j].WorkflowDestNode).ResetIDs() - } - } - - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - (&n.Forks[i].Triggers[j].WorkflowDestNode).ResetIDs() - } - } -} - -//Nodes returns a slice with all node IDs -func (n *WorkflowNode) Nodes() []WorkflowNode { - res := []WorkflowNode{*n} - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.Nodes()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.Nodes()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.Nodes()...) - } - } - return res -} - -func ancestor(id int64, node *WorkflowNode, deep bool) (map[int64]bool, bool) { - res := map[int64]bool{} - if id == node.ID { - return res, true - } - for _, t := range node.Triggers { - if t.WorkflowDestNode.ID == id { - res[node.ID] = true - return res, true - } - ids, ok := ancestor(id, &t.WorkflowDestNode, deep) - if ok { - if len(ids) == 1 || deep { - for k := range ids { - res[k] = true - } - } - if deep { - res[node.ID] = true - } - return res, true - } - } - for i := range node.Forks { - for j := range node.Forks[i].Triggers { - destNode := &node.Forks[i].Triggers[j].WorkflowDestNode - if destNode.ID == id { - res[node.ID] = true - return res, true - } - ids, ok := ancestor(id, destNode, deep) - if ok { - if len(ids) == 1 || deep { - for k := range ids { - res[k] = true - } - } - if deep { - res[node.ID] = true - } - return res, true - } - } - } - return res, false -} - -// Ancestors returns all node ancestors if deep equal true, and only his direct ancestors if deep equal false -func (n *WorkflowNode) Ancestors(w *Workflow, deep bool) []int64 { - if n == nil { - return nil - } - - res, ok := ancestor(n.ID, w.Root, deep) - - if !ok { - joinLoop: - for _, j := range w.Joins { - for _, t := range j.Triggers { - resAncestor, ok := ancestor(n.ID, &t.WorkflowDestNode, deep) - if ok { - if len(resAncestor) == 1 || deep { - for id := range resAncestor { - res[id] = true - } - } - - if len(resAncestor) == 0 || deep { - for _, id := range j.SourceNodeIDs { - res[id] = true - if deep { - node := w.GetNode(id) - if node != nil { - ancerstorRes := node.Ancestors(w, deep) - for _, id := range ancerstorRes { - res[id] = true - } - } - } - } - } - break joinLoop - } - } - } - } - - keys := make([]int64, len(res)) - i := 0 - for k := range res { - keys[i] = k - i++ - } - return keys -} - -//TriggersID returns a slides of triggers IDs -func (n *WorkflowNode) TriggersID() []int64 { - res := []int64{} - for _, t := range n.Triggers { - res = append(res, t.ID) - res = append(res, t.WorkflowDestNode.TriggersID()...) - } - return res -} - -//References returns a slice with all node references -func (n *WorkflowNode) References() []string { - res := []string{} - if n.Ref != "" { - res = []string{n.Ref} - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.References()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.References()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.References()...) - } - } - return res -} - -//InvolvedApplications returns all applications used in the workflow -func (n *WorkflowNode) InvolvedApplications() []int64 { - res := []int64{} - if n.Context != nil { - if n.Context.ApplicationID == 0 && n.Context.Application != nil { - n.Context.ApplicationID = n.Context.Application.ID - } - if n.Context.ApplicationID != 0 { - res = []int64{n.Context.ApplicationID} - } - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedApplications()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.InvolvedApplications()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.InvolvedApplications()...) - } - } - return res -} - -//InvolvedPipelines returns all pipelines used in the workflow -func (n *WorkflowNode) InvolvedPipelines() []int64 { - res := []int64{} - if n.PipelineID != 0 { - res = append(res, n.PipelineID) - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedPipelines()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.InvolvedPipelines()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.InvolvedPipelines()...) - } - } - return res -} - -//GetApplications returns all applications used in the workflow -func (n *WorkflowNode) GetApplications() []Application { - res := []Application{} - if n.Context != nil && n.Context.Application != nil { - res = append(res, *n.Context.Application) - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.GetApplications()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.GetApplications()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.GetApplications()...) - } - } - return res -} - -//GetEnvironments returns all Environments used in the workflow -func (n *WorkflowNode) GetEnvironments() []Environment { - res := []Environment{} - if n.Context != nil && n.Context.Environment != nil { - res = append(res, *n.Context.Environment) - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.GetEnvironments()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.GetEnvironments()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.GetEnvironments()...) - } - } - return res -} - -//InvolvedEnvironments returns all environments used in the workflow -func (n *WorkflowNode) InvolvedEnvironments() []int64 { - res := []int64{} - if n.Context != nil { - if n.Context.EnvironmentID == 0 && n.Context.Environment != nil { - n.Context.EnvironmentID = n.Context.Environment.ID - } - if n.Context.EnvironmentID != 0 { - res = []int64{n.Context.EnvironmentID} - } - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedEnvironments()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.InvolvedEnvironments()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.InvolvedEnvironments()...) - } - } - return res -} - -//InvolvedIntegrations returns all integrations used in the workflow -func (n *WorkflowNode) InvolvedIntegrations() []int64 { - res := []int64{} - if n.Context != nil { - if n.Context.ProjectIntegrationID == 0 && n.Context.ProjectIntegration != nil { - n.Context.ProjectIntegrationID = n.Context.ProjectIntegration.ID - } - if n.Context.ProjectIntegrationID != 0 { - res = []int64{n.Context.ProjectIntegrationID} - } - } - for _, t := range n.Triggers { - res = append(res, t.WorkflowDestNode.InvolvedIntegrations()...) - } - for i := range n.OutgoingHooks { - for j := range n.OutgoingHooks[i].Triggers { - res = append(res, n.OutgoingHooks[i].Triggers[j].WorkflowDestNode.InvolvedIntegrations()...) - } - } - for i := range n.Forks { - for j := range n.Forks[i].Triggers { - res = append(res, n.Forks[i].Triggers[j].WorkflowDestNode.InvolvedIntegrations()...) - } - } - return res -} - -// CheckApplicationDeploymentStrategies checks application deployment strategies -func (n *WorkflowNode) CheckApplicationDeploymentStrategies(proj *Project) error { - if n.Context == nil { - return nil - } - if n.Context.Application == nil { - return nil - } - - var id = n.Context.ProjectIntegrationID - if id == 0 && n.Context.ProjectIntegration != nil { - id = n.Context.ProjectIntegration.ID - } - - if id == 0 { - return nil - } - - pf := proj.GetIntegrationByID(id) - if pf == nil { - return fmt.Errorf("integration unavailable") - } - - for _, a := range proj.Applications { - if a.ID == n.Context.ApplicationID || (n.Context.Application != nil && n.Context.Application.ID == a.ID) { - if _, has := a.DeploymentStrategies[pf.Name]; !has { - return fmt.Errorf("integration %s unavailable", pf.Name) - } - } - } - - return nil -} - -//WorkflowNodeTrigger is a link between two pipelines in a workflow -type WorkflowNodeTrigger struct { - ID int64 `json:"id" db:"id"` - WorkflowNodeID int64 `json:"workflow_node_id" db:"workflow_node_id"` - WorkflowDestNodeID int64 `json:"workflow_dest_node_id" db:"workflow_dest_node_id"` - WorkflowDestNode WorkflowNode `json:"workflow_dest_node" db:"-"` -} - -// WorkflowNodeForkTrigger is a link between a fork and a node -type WorkflowNodeForkTrigger struct { - ID int64 `json:"id" db:"id"` - WorkflowForkID int64 `json:"workflow_node_fork_id" db:"workflow_node_fork_id"` - WorkflowDestNodeID int64 `json:"workflow_dest_node_id" db:"workflow_dest_node_id"` - WorkflowDestNode WorkflowNode `json:"workflow_dest_node" db:"-"` -} - -//WorkflowNodeOutgoingHookTrigger is a link between an outgoing hook and pipeline in a workflow -type WorkflowNodeOutgoingHookTrigger struct { - ID int64 `json:"id" db:"id"` - WorkflowNodeOutgoingHookID int64 `json:"workflow_node_outgoing_hook_id" db:"workflow_node_outgoing_hook_id"` - WorkflowDestNodeID int64 `json:"workflow_dest_node_id" db:"workflow_dest_node_id"` - WorkflowDestNode WorkflowNode `json:"workflow_dest_node" db:"-"` -} - //WorkflowNodeConditions is either an array of WorkflowNodeCondition or a lua script type WorkflowNodeConditions struct { PlainConditions []WorkflowNodeCondition `json:"plain,omitempty" yaml:"check,omitempty"` @@ -1494,55 +211,6 @@ type WorkflowNodeCondition struct { Value string `json:"value" yaml:"value"` } -//WorkflowNodeContext represents a context attached on a node -type WorkflowNodeContext struct { - ID int64 `json:"id" db:"id"` - WorkflowNodeID int64 `json:"workflow_node_id" db:"workflow_node_id"` - ApplicationID int64 `json:"application_id" db:"application_id"` - Application *Application `json:"application,omitempty" db:"-"` - Environment *Environment `json:"environment,omitempty" db:"-"` - EnvironmentID int64 `json:"environment_id" db:"environment_id"` - ProjectIntegration *ProjectIntegration `json:"project_integration" db:"-"` - ProjectIntegrationID int64 `json:"project_integration_id" db:"project_integration_id"` - DefaultPayload interface{} `json:"default_payload,omitempty" db:"-"` - DefaultPipelineParameters []Parameter `json:"default_pipeline_parameters,omitempty" db:"-"` - Conditions WorkflowNodeConditions `json:"conditions,omitempty" db:"-"` - Mutex bool `json:"mutex"` -} - -// HasDefaultPayload returns true if the node has a default payload -func (c *WorkflowNodeContext) HasDefaultPayload() bool { - if c == nil { - return false - } - if c.DefaultPayload == nil { - return false - } - dumper := dump.NewDefaultEncoder() - dumper.ExtraFields.DetailedMap = false - dumper.ExtraFields.DetailedStruct = false - dumper.ExtraFields.Len = false - dumper.ExtraFields.Type = false - m, _ := dumper.ToStringMap(c.DefaultPayload) - return len(m) > 0 -} - -// DefaultPayloadToMap returns default payload to map -func (c *WorkflowNodeContext) DefaultPayloadToMap() (map[string]string, error) { - if c == nil { - return nil, fmt.Errorf("Workflow node context is nil") - } - if c.DefaultPayload == nil { - return map[string]string{}, nil - } - dumper := dump.NewDefaultEncoder() - dumper.ExtraFields.DetailedMap = false - dumper.ExtraFields.DetailedStruct = false - dumper.ExtraFields.Len = false - dumper.ExtraFields.Type = false - return dumper.ToStringMap(c.DefaultPayload) -} - //WorkflowNodeContextDefaultPayloadVCS represents a default payload when a workflow is attached to a repository Webhook type WorkflowNodeContextDefaultPayloadVCS struct { GitBranch string `json:"git.branch" db:"-"` @@ -1554,46 +222,6 @@ type WorkflowNodeContextDefaultPayloadVCS struct { GitMessage string `json:"git.message" db:"-"` } -// IsWorkflowNodeContextDefaultPayloadVCS checks with several way if the workflow node context has a default vcs payloas -func IsWorkflowNodeContextDefaultPayloadVCS(i interface{}) bool { - _, ok := i.(WorkflowNodeContextDefaultPayloadVCS) - if ok { - return true - } - - dumper := dump.NewDefaultEncoder() - dumper.ExtraFields.DetailedMap = false - dumper.ExtraFields.DetailedStruct = false - dumper.ExtraFields.Len = false - dumper.ExtraFields.Type = false - - mI, _ := dumper.ToMap(i) - mD, _ := dumper.ToMap(WorkflowNodeContextDefaultPayloadVCS{}) - - // compare interface keys with default payload keys - hasKey := func(s string) bool { - _, has := mI[s] - return has - } - - if len(mI) == len(mD) { - for k := range mD { - if !hasKey(k) { - goto checkGitKey - } - } - return true - } - -checkGitKey: - return hasKey("git.branch") && - hasKey("git.hash") && - hasKey("git.author") && - hasKey("git.hash.before") && - hasKey("git.repository") && - hasKey("git.message") -} - // WorkflowNodeJobRunCount return nb workflow run job since 'since' type WorkflowNodeJobRunCount struct { Count int64 `json:"version"` diff --git a/sdk/workflow_data.go b/sdk/workflow_data.go index 37d2c07a29..8ac08647d3 100644 --- a/sdk/workflow_data.go +++ b/sdk/workflow_data.go @@ -21,13 +21,14 @@ func (w *WorkflowData) AncestorsNames(n Node) []string { } // GetHooks returns the list of all hooks in the workflow tree -func (w *WorkflowData) GetHooks() map[string]NodeHook { +func (w *WorkflowData) GetHooks() map[string]*NodeHook { if w == nil { return nil } - res := map[string]NodeHook{} + res := map[string]*NodeHook{} for _, n := range w.Array() { - for _, h := range n.Hooks { + for i := range n.Hooks { + h := &n.Hooks[i] res[h.UUID] = h } } diff --git a/sdk/workflow_hook.go b/sdk/workflow_hook.go index 46c1c5dc52..c134b55814 100644 --- a/sdk/workflow_hook.go +++ b/sdk/workflow_hook.go @@ -8,136 +8,23 @@ const ( GerritIcon = "git" ) -// FilterHooksConfig filter all hooks configuration and remove some configuration key -func (w *Workflow) FilterHooksConfig(s ...string) { - if w.Root == nil { - return - } - - w.Root.FilterHooksConfig(s...) - for i := range w.Joins { - for j := range w.Joins[i].Triggers { - w.Joins[i].Triggers[j].WorkflowDestNode.FilterHooksConfig(s...) - } - } -} - -// GetHooks returns the list of all hooks in the workflow tree -func (w *Workflow) GetHooks() map[string]WorkflowNodeHook { - if w == nil { - return nil - } - - if w.Root == nil { - return nil - } - - res := map[string]WorkflowNodeHook{} - - a := w.Root.GetHooks() - for k, v := range a { - res[k] = v - } - - for _, j := range w.Joins { - for _, t := range j.Triggers { - b := t.WorkflowDestNode.GetHooks() - for k, v := range b { - res[k] = v - } - } - } - - return res -} - -// WorkflowNodeOutgoingHook represents a outgoing hook -type WorkflowNodeOutgoingHook struct { - ID int64 `json:"id" db:"id"` - Name string `json:"name" db:"name"` - Ref string `json:"ref" db:"-"` - WorkflowNodeID int64 `json:"workflow_node_id" db:"workflow_node_id"` - WorkflowHookModelID int64 `json:"workflow_hook_model_id" db:"workflow_hook_model_id"` - WorkflowHookModel WorkflowHookModel `json:"model" db:"-"` - Config WorkflowNodeHookConfig `json:"config" db:"-"` - Triggers []WorkflowNodeOutgoingHookTrigger `json:"triggers,omitempty" db:"-"` -} - -func (h WorkflowNodeOutgoingHook) migrate(withID bool) Node { - newNode := Node{ - Name: h.Name, - Ref: h.Ref, - Type: NodeTypeOutGoingHook, - OutGoingHookContext: &NodeOutGoingHook{ - Config: h.Config, - HookModelID: h.WorkflowHookModelID, - }, - Triggers: make([]NodeTrigger, 0, len(h.Triggers)), - } - if withID { - newNode.ID = h.ID - } - if h.Ref == "" { - h.Ref = h.Name - } - for _, t := range h.Triggers { - child := t.WorkflowDestNode.migrate(withID) - newNode.Triggers = append(newNode.Triggers, NodeTrigger{ - ParentNodeName: newNode.Name, - ChildNode: child, - }) - } - return newNode -} - -//WorkflowNodeFork represents a hook which cann trigger the workflow from a given node -type WorkflowNodeFork struct { - ID int64 `json:"id" db:"id"` - Name string `json:"name" db:"name"` - WorkflowNodeID int64 `json:"workflow_node_id" db:"workflow_node_id"` - Triggers []WorkflowNodeForkTrigger `json:"triggers,omitempty" db:"-"` -} - -func (f WorkflowNodeFork) migrate(withID bool) Node { - newNode := Node{ - Name: f.Name, - Ref: f.Name, - Type: NodeTypeFork, - Triggers: make([]NodeTrigger, 0, len(f.Triggers)), - } - if withID { - newNode.ID = f.ID - } - if newNode.Ref == "" { - newNode.Ref = newNode.Name - } - for _, t := range f.Triggers { - child := t.WorkflowDestNode.migrate(withID) - newNode.Triggers = append(newNode.Triggers, NodeTrigger{ - ParentNodeName: newNode.Name, - ChildNode: child, - }) - } - return newNode -} - -//WorkflowNodeHook represents a hook which cann trigger the workflow from a given node -type WorkflowNodeHook struct { - ID int64 `json:"id" db:"id"` - UUID string `json:"uuid" db:"uuid"` - Ref string `json:"ref" db:"ref"` - WorkflowNodeID int64 `json:"workflow_node_id" db:"workflow_node_id"` - WorkflowHookModelID int64 `json:"workflow_hook_model_id" db:"workflow_hook_model_id"` - WorkflowHookModel WorkflowHookModel `json:"model" db:"-"` - Config WorkflowNodeHookConfig `json:"config" db:"-"` +//NodeHook represents a hook which cann trigger the workflow from a given node +type NodeHook struct { + ID int64 `json:"id" db:"id"` + UUID string `json:"uuid" db:"uuid"` + Ref string `json:"ref" db:"ref"` + NodeID int64 `json:"node_id" db:"node_id"` + HookModelID int64 `json:"hook_model_id" db:"hook_model_id"` + HookModelName string `json:"hook_model_name" db:"-"` + Config WorkflowNodeHookConfig `json:"config" db:"-"` } //Equals checks functionnal equality between two hooks -func (h WorkflowNodeHook) Equals(h1 WorkflowNodeHook) bool { +func (h NodeHook) Equals(h1 NodeHook) bool { if h.UUID != h1.UUID { return false } - if h.WorkflowHookModelID != h1.WorkflowHookModelID { + if h.HookModelID != h1.HookModelID { return false } for k, cfg := range h.Config { @@ -161,6 +48,18 @@ func (h WorkflowNodeHook) Equals(h1 WorkflowNodeHook) bool { return true } +// FilterHooksConfig filter all hooks configuration and remove some configuration key +func (w *Workflow) FilterHooksConfig(s ...string) { + if w.WorkflowData == nil { + return + } + + w.WorkflowData.Node.FilterHooksConfig(s...) + for i := range w.WorkflowData.Joins { + w.WorkflowData.Joins[i].FilterHooksConfig(s...) + } +} + // WorkflowHookModelBuiltin is a constant for the builtin hook models const WorkflowHookModelBuiltin = "builtin" @@ -243,39 +142,3 @@ type WorkflowHookModel struct { DefaultConfig WorkflowNodeHookConfig `json:"default_config" db:"-"` Disabled bool `json:"disabled" db:"disabled"` } - -// FilterHooksConfig filter all hooks configuration and remove somme configuration key -func (n *WorkflowNode) FilterHooksConfig(s ...string) { - if n == nil { - return - } - - for _, h := range n.Hooks { - for i := range s { - for k := range h.Config { - if k == s[i] { - delete(h.Config, k) - break - } - } - } - } -} - -//GetHooks returns all hooks for the node and its children -func (n *WorkflowNode) GetHooks() map[string]WorkflowNodeHook { - res := map[string]WorkflowNodeHook{} - - for _, h := range n.Hooks { - res[h.UUID] = h - } - - for _, t := range n.Triggers { - b := t.WorkflowDestNode.GetHooks() - for k, v := range b { - res[k] = v - } - } - - return res -} diff --git a/sdk/workflow_node.go b/sdk/workflow_node.go index f5b8d5bb98..59433c512d 100644 --- a/sdk/workflow_node.go +++ b/sdk/workflow_node.go @@ -30,17 +30,6 @@ type Node struct { Groups []GroupPermission `json:"groups,omitempty" db:"-"` } -//NodeHook represents a hook which cann trigger the workflow from a given node -type NodeHook struct { - ID int64 `json:"id" db:"id"` - UUID string `json:"uuid" db:"uuid"` - Ref string `json:"ref" db:"ref"` - NodeID int64 `json:"node_id" db:"node_id"` - HookModelID int64 `json:"hook_model_id" db:"hook_model_id"` - HookModelName string `json:"-" db:"-"` - Config WorkflowNodeHookConfig `json:"config" db:"-"` -} - // NodeContext represents a node linked to a pipeline type NodeContext struct { ID int64 `json:"id" db:"id"` @@ -59,6 +48,24 @@ type NodeContext struct { Mutex bool `json:"mutex" db:"mutex"` } +// FilterHooksConfig filter all hooks configuration and remove somme configuration key +func (n *Node) FilterHooksConfig(s ...string) { + if n == nil { + return + } + + for _, h := range n.Hooks { + for i := range s { + for k := range h.Config { + if k == s[i] { + delete(h.Config, k) + break + } + } + } + } +} + //AddTrigger adds a trigger to the destination node from the node found by its name func (n *Node) AddTrigger(name string, dest Node) { if n.Name == name { diff --git a/sdk/workflow_test.go b/sdk/workflow_test.go deleted file mode 100644 index 0ea52ddd60..0000000000 --- a/sdk/workflow_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package sdk - -import ( - "testing" - - "fmt" - - "github.com/stretchr/testify/assert" -) - -func TestWorkflowNode_AncestorsWithTriggers(t *testing.T) { - w := Workflow{ - ID: 1, - } - w.Root = &WorkflowNode{ - ID: 1, - } - - node4 := WorkflowNode{ - ID: 4, - } - - w.Root.Triggers = []WorkflowNodeTrigger{ - { - WorkflowDestNode: WorkflowNode{ - ID: 2, - Triggers: []WorkflowNodeTrigger{ - { - WorkflowDestNode: WorkflowNode{ - ID: 3, - Triggers: []WorkflowNodeTrigger{ - { - WorkflowDestNode: node4, - }, - }, - }, - }, - }, - }, - }, - } - - ids := node4.Ancestors(&w, true) - t.Logf("Deep node4.Ancestors: %v\n", ids) - assert.Equal(t, 3, len(ids)) - - ids = node4.Ancestors(&w, false) - t.Logf("Not deep node4.Ancestors: %v\n", ids) - assert.Equal(t, 1, len(ids)) - assert.Equal(t, int64(3), ids[0]) -} - -func TestWorkflowNode_AncestorsDirectAfterJoin(t *testing.T) { - w := Workflow{ - ID: 1, - } - w.Root = &WorkflowNode{ - ID: 1, - Triggers: []WorkflowNodeTrigger{ - { - WorkflowDestNode: WorkflowNode{ - ID: 2, - }, - }, - { - WorkflowDestNode: WorkflowNode{ - ID: 3, - }, - }, - }, - } - - node4 := WorkflowNode{ - ID: 4, - } - - w.Joins = []WorkflowNodeJoin{ - { - SourceNodeIDs: []int64{2, 3}, - Triggers: []WorkflowNodeJoinTrigger{ - { - WorkflowDestNode: node4, - }, - }, - }, - } - - ids := node4.Ancestors(&w, true) - t.Logf("Deep node4.Ancestors: %v\n", ids) - assert.Equal(t, 3, len(ids)) - - ids = node4.Ancestors(&w, false) - t.Logf("Not deep node4.Ancestors: %v\n", ids) - assert.Equal(t, 2, len(ids)) - - if !((ids[0] == 2 && ids[1] == 3) || (ids[0] == 3 && ids[1] == 2)) { - assert.Error(t, fmt.Errorf("Wrong parent ID")) - } -} - -func TestWorkflowNode_AncestorsAfterJoin(t *testing.T) { - w := Workflow{ - ID: 1, - } - w.Root = &WorkflowNode{ - ID: 1, - Triggers: []WorkflowNodeTrigger{ - { - WorkflowDestNode: WorkflowNode{ - ID: 2, - }, - }, - { - WorkflowDestNode: WorkflowNode{ - ID: 3, - }, - }, - }, - } - - node5 := WorkflowNode{ - ID: 5, - } - - w.Joins = []WorkflowNodeJoin{ - { - SourceNodeIDs: []int64{2, 3}, - Triggers: []WorkflowNodeJoinTrigger{ - { - WorkflowDestNode: WorkflowNode{ - ID: 4, - Triggers: []WorkflowNodeTrigger{ - { - WorkflowDestNode: node5, - }, - }, - }, - }, - }, - }, - } - - ids := node5.Ancestors(&w, true) - t.Logf("Deep node5.Ancestors: %v\n", ids) - assert.Equal(t, 4, len(ids)) - - ids = node5.Ancestors(&w, false) - t.Logf("Not deep node5.Ancestors: %v\n", ids) - assert.Equal(t, 1, len(ids)) - assert.Equal(t, int64(4), ids[0]) -} diff --git a/tests/clictl_template_bulk.yml b/tests/clictl_template_bulk.yml index fee9c90fd0..8705f4c8c3 100644 --- a/tests/clictl_template_bulk.yml +++ b/tests/clictl_template_bulk.yml @@ -9,8 +9,8 @@ testcases: - name: prepare test steps: - - script: {{.cds.build.cdsctl}} project delete --force ITCLIPRJ - - script: {{.cds.build.cdsctl}} project add ITCLIPRJ "Test Project" + - script: {{.cds.build.cdsctl}} project delete --force ITCLIPRJBULK + - script: {{.cds.build.cdsctl}} project add ITCLIPRJBULK "Test Project" - name: sendTemplateBulkRequest steps: diff --git a/tests/fixtures/template/bulk_request.yml b/tests/fixtures/template/bulk_request.yml index 2eed069fdb..e5e786a73c 100644 --- a/tests/fixtures/template/bulk_request.yml +++ b/tests/fixtures/template/bulk_request.yml @@ -1,14 +1,14 @@ template_path: shared.infra/example-simple instances: -- workflow_path: ITCLIPRJ/one +- workflow_path: ITCLIPRJBULK/one parameters: - withDeploy=true - deployWhen=success -- workflow_path: ITCLIPRJ/two +- workflow_path: ITCLIPRJBULK/two parameters: - withDeploy=false - data=["one", "two"] -- workflow_path: ITCLIPRJ/three +- workflow_path: ITCLIPRJBULK/three parameters: - withDeploy=true - deployWhen=ok diff --git a/ui/src/app/model/workflow.model.ts b/ui/src/app/model/workflow.model.ts index 59d5a5153f..5329be27cd 100644 --- a/ui/src/app/model/workflow.model.ts +++ b/ui/src/app/model/workflow.model.ts @@ -396,7 +396,7 @@ export class WorkflowTriggerConditionCache { export class WorkflowNotification { id: number; - source_node_id: Array; + node_id: Array; source_node_ref: Array; type: string; settings: UserNotificationSettings; @@ -405,7 +405,7 @@ export class WorkflowNotification { this.type = notificationTypes[0]; this.settings = new UserNotificationSettings(); this.source_node_ref = new Array(); - this.source_node_id = new Array(); + this.node_id = new Array(); } } diff --git a/ui/src/app/shared/table/data-table.component.ts b/ui/src/app/shared/table/data-table.component.ts index 263f762daa..d5f4b16f17 100644 --- a/ui/src/app/shared/table/data-table.component.ts +++ b/ui/src/app/shared/table/data-table.component.ts @@ -158,11 +158,10 @@ export class DataTableComponent extends Table implements O getData(): Array { this.filteredData = this.data; - if (this.filter && this.filterFunc) { - this.filteredData = this.data.filter(this.filterFunc(this.filter)); - } - if (this.filteredData) { + if (this.filter && this.filterFunc) { + this.filteredData = this.data.filter(this.filterFunc(this.filter)); + } this.dataChange.emit(this.filteredData.length); } diff --git a/ui/src/app/shared/workflow/wizard/context/wizard.context.component.ts b/ui/src/app/shared/workflow/wizard/context/wizard.context.component.ts index 38f72cf3d1..c352964866 100644 --- a/ui/src/app/shared/workflow/wizard/context/wizard.context.component.ts +++ b/ui/src/app/shared/workflow/wizard/context/wizard.context.component.ts @@ -146,7 +146,26 @@ export class WorkflowWizardNodeContextComponent implements OnInit { n.context.environment_id = this.editableNode.context.environment_id; n.context.project_integration_id = this.editableNode.context.project_integration_id; n.context.mutex = this.editableNode.context.mutex; + + let previousName = n.name; n.name = this.editableNode.name; + + if (previousName !== n.name) { + if (clonedWorkflow.notifications) { + clonedWorkflow.notifications.forEach(notif => { + if (notif.source_node_ref) { + notif.source_node_ref = notif.source_node_ref.map(ref => { + if (ref === previousName) { + return n.name; + } + return ref; + }); + } + + }); + } + } + this._store.dispatch(new UpdateWorkflow({ projectKey: this.workflow.project_key, workflowName: this.workflow.name, diff --git a/ui/src/app/views/workflow/show/notification/list/workflow.notification.list.component.ts b/ui/src/app/views/workflow/show/notification/list/workflow.notification.list.component.ts index a7d9130cae..960b79862c 100644 --- a/ui/src/app/views/workflow/show/notification/list/workflow.notification.list.component.ts +++ b/ui/src/app/views/workflow/show/notification/list/workflow.notification.list.component.ts @@ -122,7 +122,7 @@ export class WorkflowNotificationListComponent { if (this.workflow.notifications) { this.workflow.notifications.forEach(n => { let listNodes = new Array(); - n.source_node_id.forEach(id => { + n.node_id.forEach(id => { let node = mapNodes.get(id); if (node) { listNodes.push(node.name);