Skip to content

refactor: simplify webhook deployment api(breaking) #7545

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
d4x1 merged 1 commit into from
May 30, 2024
Merged

refactor: simplify webhook deployment api(breaking) #7545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
300 changes: 146 additions & 154 deletions backend/plugins/webhook/api/deployments.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,13 @@ package api
import (
"crypto/md5"
"fmt"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/log"
"net/http"
"strings"
"time"

"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/log"

"github.com/apache/incubator-devlake/helpers/dbhelper"
"github.com/go-playground/validator/v10"

Expand All @@ -37,44 +38,92 @@ import (
"github.com/apache/incubator-devlake/plugins/webhook/models"
)

type WebhookDeployTaskRequest struct {
DisplayTitle string `mapstructure:"display_title"`
PipelineId string `mapstructure:"pipeline_id" validate:"required"`
// RepoUrl should be unique string, fill url or other unique data
RepoId string `mapstructure:"repo_id"`
Result string `mapstructure:"result"`
// start_time and end_time is more readable for users,
// StartedDate and FinishedDate is same as columns in db.
// So they all keep.
CreatedDate *time.Time `mapstructure:"create_time"`
// QueuedDate *time.Time `mapstructure:"queue_time"`
StartedDate *time.Time `mapstructure:"start_time" validate:"required"`
FinishedDate *time.Time `mapstructure:"end_time"`
RepoUrl string `mapstructure:"repo_url"`
Environment string `validate:"omitempty,oneof=PRODUCTION STAGING TESTING DEVELOPMENT"`
Name string `mapstructure:"name"`
RefName string `mapstructure:"ref_name"`
CommitSha string `mapstructure:"commit_sha"`
CommitMsg string `mapstructure:"commit_msg"`
type WebhookDeploymentReq struct {
Id string `mapstructure:"id"`
DisplayTitle string `mapstructure:"displayTitle"`
PipelineId string `mapstructure:"pipelineId" validate:"required"`
Result string `mapstructure:"result"`
Environment string `validate:"omitempty,oneof=PRODUCTION STAGING TESTING DEVELOPMENT"`
Name string `mapstructure:"name"`
// DeploymentCommits is used for multiple commits in one deployment
DeploymentCommits []DeploymentCommit `mapstructure:"deployment_commits" validate:"omitempty,dive"`
DeploymentCommits []WebhookDeploymentCommitReq `mapstructure:"deploymentCommits" validate:"omitempty,dive"`
CreatedDate *time.Time `mapstructure:"createdTime"`
// QueuedDate *time.Time `mapstructure:"queue_time"`
StartedDate *time.Time `mapstructure:"startedTime" validate:"required"`
FinishedDate *time.Time `mapstructure:"endedTime"`
}

type DeploymentCommit struct {
DisplayTitle string `mapstructure:"display_title"`
RepoUrl string `mapstructure:"repo_url" validate:"required"`
Name string `mapstructure:"name"`
RefName string `mapstructure:"ref_name"`
CommitSha string `mapstructure:"commit_sha" validate:"required"`
CommitMsg string `mapstructure:"commit_msg"`
type WebhookDeploymentCommitReq struct {
DisplayTitle string `mapstructure:"displayTitle"`
RepoId string `mapstructure:"repoId"`
RepoUrl string `mapstructure:"repoUrl" validate:"required"`
Name string `mapstructure:"name"`
RefName string `mapstructure:"refName"`
CommitSha string `mapstructure:"commitSha" validate:"required"`
CommitMsg string `mapstructure:"commitMsg"`
Result string `mapstructure:"result"`
Status string `mapstructure:"status"`
CreatedDate *time.Time `mapstructure:"createdTime"`
// QueuedDate *time.Time `mapstructure:"queue_time"`
StartedDate *time.Time `mapstructure:"startedTime"`
FinishedDate *time.Time `mapstructure:"endedTime"`
}

func GenerateDeploymentCommitId(connectionId uint64, pipelineId string, repoUrl string, commitSha string) string {
urlHash16 := fmt.Sprintf("%x", md5.Sum([]byte(repoUrl)))[:16]
return fmt.Sprintf("%s:%d:%s:%s:%s", "webhook", connectionId, pipelineId, urlHash16, commitSha)
// PostDeployments
// @Summary create deployment by webhook
// @Description Create deployment pipeline by webhook.<br/>
// @Description example1: {"repo_url":"devlake","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d","start_time":"2020-01-01T12:00:00+00:00","end_time":"2020-01-01T12:59:59+00:00","environment":"PRODUCTION"}<br/>
// @Description So we suggest request before task after deployment pipeline finish.
// @Description Both cicd_pipeline and cicd_task will be created
// @Tags plugins/webhook
// @Param body body WebhookDeployTaskRequest true "json body"
// @Success 200
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 403 {string} errcode.Error "Forbidden"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/webhook/connections/:connectionId/deployments [POST]
func PostDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
connection := &models.WebhookConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
// get request
request := &WebhookDeploymentReq{}
err = api.DecodeMapStruct(input.Body, request, true)
if err != nil {
return &plugin.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
}
// validate
vld = validator.New()
err = errors.Convert(vld.Struct(request))
if err != nil {
return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`)
}
txHelper := dbhelper.NewTxHelper(basicRes, &err)
defer txHelper.End()
tx := txHelper.Begin()
if err := CreateDeploymentAndDeploymentCommits(connection, request, tx, logger); err != nil {
logger.Error(err, "create deployments")
return nil, err
}

return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}

func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, request *WebhookDeployTaskRequest, tx dal.Transaction, logger log.Logger) errors.Error {
func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, request *WebhookDeploymentReq, tx dal.Transaction, logger log.Logger) errors.Error {
// validation
if request == nil {
return errors.BadInput.New("request body is nil")
}
if len(request.DeploymentCommits) == 0 {
return errors.BadInput.New("deployment_commits is empty")
}
// set default values for optional fields
deploymentId := request.Id
if deploymentId == "" {
deploymentId = request.PipelineId
}
scopeId := fmt.Sprintf("%s:%d", "webhook", connection.ID)
if request.CreatedDate == nil {
request.CreatedDate = request.StartedDate
Expand All @@ -92,147 +141,90 @@ func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection,
duration := float64(request.FinishedDate.Sub(*request.StartedDate).Milliseconds() / 1e3)
name := request.Name
if name == "" {
if request.DeploymentCommits == nil {
name = fmt.Sprintf(`deployment for %s`, request.CommitSha)
} else {
var commitShaList []string
for _, commit := range request.DeploymentCommits {
commitShaList = append(commitShaList, commit.CommitSha)
}
name = fmt.Sprintf(`deployment for %s`, strings.Join(commitShaList, ","))
var commitShaList []string
for _, commit := range request.DeploymentCommits {
commitShaList = append(commitShaList, commit.CommitSha)
}
name = fmt.Sprintf(`deploy %s to %s`, strings.Join(commitShaList, ","), request.Environment)
}
createdDate := time.Now()
if request.CreatedDate != nil {
createdDate = *request.CreatedDate
} else if request.StartedDate != nil {
createdDate = *request.StartedDate
}
dateInfo := devops.TaskDatesInfo{
CreatedDate: createdDate,
// QueuedDate: request.QueuedDate,
StartedDate: request.StartedDate,
FinishedDate: request.FinishedDate,
}

// prepare deploymentCommits and deployment records
// queuedDuration := dateInfo.CalculateQueueDuration()
if request.DeploymentCommits == nil {
if request.CommitSha == "" || request.RepoUrl == "" {
return errors.Convert(fmt.Errorf("commit_sha or repo_url is required"))
deploymentCommits := make([]*devops.CicdDeploymentCommit, len(request.DeploymentCommits))
for i, commit := range request.DeploymentCommits {
if commit.Result == "" {
commit.Result = devops.RESULT_SUCCESS
}
if commit.Status == "" {
commit.Status = devops.STATUS_DONE
}
if commit.Name == "" {
commit.Name = fmt.Sprintf(`deployment for %s`, commit.CommitSha)
}
if commit.CreatedDate == nil {
commit.CreatedDate = &createdDate
}
if commit.StartedDate == nil {
commit.StartedDate = request.StartedDate
}
if commit.FinishedDate == nil {
commit.FinishedDate = request.FinishedDate
}
// create a deployment_commit record
deploymentCommit := &devops.CicdDeploymentCommit{
deploymentCommits[i] = &devops.CicdDeploymentCommit{
DomainEntity: domainlayer.DomainEntity{
Id: GenerateDeploymentCommitId(connection.ID, request.PipelineId, request.RepoUrl, request.CommitSha),
Id: GenerateDeploymentCommitId(connection.ID, deploymentId, commit.RepoUrl, commit.CommitSha),
},
CicdDeploymentId: request.PipelineId,
CicdDeploymentId: deploymentId,
CicdScopeId: scopeId,
Name: name,
DisplayTitle: request.DisplayTitle,
Result: request.Result,
Status: devops.STATUS_DONE,
OriginalResult: request.Result,
OriginalStatus: devops.STATUS_DONE,
TaskDatesInfo: dateInfo,
DurationSec: &duration,
//QueuedDurationSec: queuedDuration,
RepoId: request.RepoId,
RepoUrl: request.RepoUrl,
Result: commit.Result,
Status: commit.Status,
OriginalResult: commit.Result,
OriginalStatus: commit.Status,
TaskDatesInfo: devops.TaskDatesInfo{
CreatedDate: *commit.CreatedDate,
StartedDate: commit.StartedDate,
FinishedDate: commit.FinishedDate,
},
DurationSec: &duration,
RepoId: commit.RepoId,
Name: commit.Name,
DisplayTitle: commit.DisplayTitle,
RepoUrl: commit.RepoUrl,
Environment: request.Environment,
OriginalEnvironment: request.Environment,
RefName: request.RefName,
CommitSha: request.CommitSha,
CommitMsg: request.CommitMsg,
}
if err := tx.CreateOrUpdate(deploymentCommit); err != nil {
logger.Error(err, "create deployment commit")
return err
}
// create a deployment record
if err := tx.CreateOrUpdate(deploymentCommit.ToDeploymentWithCustomDisplayTitle(request.DisplayTitle)); err != nil {
logger.Error(err, "create deployment")
return err
}
} else {
for _, commit := range request.DeploymentCommits {
// create a deployment_commit record
deploymentCommit := &devops.CicdDeploymentCommit{
DomainEntity: domainlayer.DomainEntity{
Id: GenerateDeploymentCommitId(connection.ID, request.PipelineId, commit.RepoUrl, commit.CommitSha),
},
CicdDeploymentId: request.PipelineId,
CicdScopeId: scopeId,
Result: request.Result,
Status: devops.STATUS_DONE,
OriginalResult: request.Result,
OriginalStatus: devops.STATUS_DONE,
TaskDatesInfo: dateInfo,
DurationSec: &duration,
//QueuedDurationSec: queuedDuration,
RepoId: request.RepoId,
Name: fmt.Sprintf(`deployment for %s`, commit.CommitSha),
DisplayTitle: commit.DisplayTitle,
RepoUrl: commit.RepoUrl,
Environment: request.Environment,
OriginalEnvironment: request.Environment,
RefName: commit.RefName,
CommitSha: commit.CommitSha,
CommitMsg: commit.CommitMsg,
}

if err := tx.CreateOrUpdate(deploymentCommit); err != nil {
logger.Error(err, "create deployment commit")
return err
}

// create a deployment record
deploymentCommit.Name = name
if err := tx.CreateOrUpdate(deploymentCommit.ToDeploymentWithCustomDisplayTitle(request.DisplayTitle)); err != nil {
logger.Error(err, "create deployment")
return err
}
RefName: commit.RefName,
CommitSha: commit.CommitSha,
CommitMsg: commit.CommitMsg,
//QueuedDurationSec: queuedDuration,
}
}
return nil
}

// PostDeploymentCicdTask
// @Summary create deployment by webhook
// @Description Create deployment pipeline by webhook.<br/>
// @Description example1: {"repo_url":"devlake","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d","start_time":"2020-01-01T12:00:00+00:00","end_time":"2020-01-01T12:59:59+00:00","environment":"PRODUCTION"}<br/>
// @Description So we suggest request before task after deployment pipeline finish.
// @Description Both cicd_pipeline and cicd_task will be created
// @Tags plugins/webhook
// @Param body body WebhookDeployTaskRequest true "json body"
// @Success 200
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 403 {string} errcode.Error "Forbidden"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/webhook/connections/:connectionId/deployments [POST]
func PostDeploymentCicdTask(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
connection := &models.WebhookConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
// get request
request := &WebhookDeployTaskRequest{}
err = api.DecodeMapStruct(input.Body, request, true)
if err != nil {
return &plugin.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
if err := tx.CreateOrUpdate(deploymentCommits); err != nil {
logger.Error(err, "failed to save deployment commits")
return err
}
// validate
vld = validator.New()
err = errors.Convert(vld.Struct(request))
if err != nil {
return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`)
}
txHelper := dbhelper.NewTxHelper(basicRes, &err)
defer txHelper.End()
tx := txHelper.Begin()
if err := CreateDeploymentAndDeploymentCommits(connection, request, tx, logger); err != nil {
logger.Error(err, "create deployments")
return nil, err

// create a deployment record
deployment := deploymentCommits[0].ToDeploymentWithCustomDisplayTitle(request.DisplayTitle)
deployment.Name = name
deployment.CreatedDate = createdDate
deployment.StartedDate = request.StartedDate
deployment.FinishedDate = request.FinishedDate
if err := tx.CreateOrUpdate(deployment); err != nil {
logger.Error(err, "failed to save deployment")
return err
}
return nil
}

return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
func GenerateDeploymentCommitId(connectionId uint64, deploymentId string, repoUrl string, commitSha string) string {
urlHash16 := fmt.Sprintf("%x", md5.Sum([]byte(repoUrl)))[:16]
return fmt.Sprintf("%s:%d:%s:%s:%s", "webhook", connectionId, deploymentId, urlHash16, commitSha)
}
4 changes: 2 additions & 2 deletions backend/plugins/webhook/impl/impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler
"DELETE": api.DeleteConnection,
},
"connections/:connectionId/deployments": {
"POST": api.PostDeploymentCicdTask,
"POST": api.PostDeployments,
},
"connections/:connectionId/issues": {
"POST": api.PostIssue,
Expand All @@ -97,7 +97,7 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler
"POST": api.CloseIssue,
},
":connectionId/deployments": {
"POST": api.PostDeploymentCicdTask,
"POST": api.PostDeployments,
},
":connectionId/issues": {
"POST": api.PostIssue,
Expand Down