Skip to content

Commit

Permalink
refactor: simplify webhook deployment api(breaking)
Browse files Browse the repository at this point in the history
  • Loading branch information
@klesh
klesh committed May 30, 2024
1 parent 5e38416 commit 05f1d34
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 156 deletions.
300 changes: 146 additions & 154 deletions backend/plugins/webhook/api/deployments.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,13 @@ package api
import (
"crypto/md5"
"fmt"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/log"
"net/http"
"strings"
"time"

"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/log"

"github.com/apache/incubator-devlake/helpers/dbhelper"
"github.com/go-playground/validator/v10"

Expand All @@ -37,44 +38,92 @@ import (
"github.com/apache/incubator-devlake/plugins/webhook/models"
)

type WebhookDeployTaskRequest struct {
DisplayTitle string `mapstructure:"display_title"`
PipelineId string `mapstructure:"pipeline_id" validate:"required"`
// RepoUrl should be unique string, fill url or other unique data
RepoId string `mapstructure:"repo_id"`
Result string `mapstructure:"result"`
// start_time and end_time is more readable for users,
// StartedDate and FinishedDate is same as columns in db.
// So they all keep.
CreatedDate *time.Time `mapstructure:"create_time"`
// QueuedDate *time.Time `mapstructure:"queue_time"`
StartedDate *time.Time `mapstructure:"start_time" validate:"required"`
FinishedDate *time.Time `mapstructure:"end_time"`
RepoUrl string `mapstructure:"repo_url"`
Environment string `validate:"omitempty,oneof=PRODUCTION STAGING TESTING DEVELOPMENT"`
Name string `mapstructure:"name"`
RefName string `mapstructure:"ref_name"`
CommitSha string `mapstructure:"commit_sha"`
CommitMsg string `mapstructure:"commit_msg"`
type WebhookDeploymentReq struct {
Id string `mapstructure:"id"`
DisplayTitle string `mapstructure:"displayTitle"`
PipelineId string `mapstructure:"pipelineId" validate:"required"`
Result string `mapstructure:"result"`
Environment string `validate:"omitempty,oneof=PRODUCTION STAGING TESTING DEVELOPMENT"`
Name string `mapstructure:"name"`
// DeploymentCommits is used for multiple commits in one deployment
DeploymentCommits []DeploymentCommit `mapstructure:"deployment_commits" validate:"omitempty,dive"`
DeploymentCommits []WebhookDeploymentCommitReq `mapstructure:"deploymentCommits" validate:"omitempty,dive"`
CreatedDate *time.Time `mapstructure:"createdTime"`
// QueuedDate *time.Time `mapstructure:"queue_time"`
StartedDate *time.Time `mapstructure:"startedTime" validate:"required"`
FinishedDate *time.Time `mapstructure:"endedTime"`
}

type DeploymentCommit struct {
DisplayTitle string `mapstructure:"display_title"`
RepoUrl string `mapstructure:"repo_url" validate:"required"`
Name string `mapstructure:"name"`
RefName string `mapstructure:"ref_name"`
CommitSha string `mapstructure:"commit_sha" validate:"required"`
CommitMsg string `mapstructure:"commit_msg"`
type WebhookDeploymentCommitReq struct {
DisplayTitle string `mapstructure:"displayTitle"`
RepoId string `mapstructure:"repoId"`
RepoUrl string `mapstructure:"repoUrl" validate:"required"`
Name string `mapstructure:"name"`
RefName string `mapstructure:"refName"`
CommitSha string `mapstructure:"commitSha" validate:"required"`
CommitMsg string `mapstructure:"commitMsg"`
Result string `mapstructure:"result"`
Status string `mapstructure:"status"`
CreatedDate *time.Time `mapstructure:"createdTime"`
// QueuedDate *time.Time `mapstructure:"queue_time"`
StartedDate *time.Time `mapstructure:"startedTime"`
FinishedDate *time.Time `mapstructure:"endedTime"`
}

func GenerateDeploymentCommitId(connectionId uint64, pipelineId string, repoUrl string, commitSha string) string {
urlHash16 := fmt.Sprintf("%x", md5.Sum([]byte(repoUrl)))[:16]
return fmt.Sprintf("%s:%d:%s:%s:%s", "webhook", connectionId, pipelineId, urlHash16, commitSha)
// PostDeployments
// @Summary create deployment by webhook
// @Description Create deployment pipeline by webhook.<br/>
// @Description example1: {"repo_url":"devlake","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d","start_time":"2020-01-01T12:00:00+00:00","end_time":"2020-01-01T12:59:59+00:00","environment":"PRODUCTION"}<br/>
// @Description So we suggest request before task after deployment pipeline finish.
// @Description Both cicd_pipeline and cicd_task will be created
// @Tags plugins/webhook
// @Param body body WebhookDeployTaskRequest true "json body"
// @Success 200
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 403 {string} errcode.Error "Forbidden"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/webhook/connections/:connectionId/deployments [POST]
func PostDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
connection := &models.WebhookConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
// get request
request := &WebhookDeploymentReq{}
err = api.DecodeMapStruct(input.Body, request, true)
if err != nil {
return &plugin.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
}
// validate
vld = validator.New()
err = errors.Convert(vld.Struct(request))
if err != nil {
return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`)
}
txHelper := dbhelper.NewTxHelper(basicRes, &err)
defer txHelper.End()
tx := txHelper.Begin()
if err := CreateDeploymentAndDeploymentCommits(connection, request, tx, logger); err != nil {
logger.Error(err, "create deployments")
return nil, err
}

return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}

func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, request *WebhookDeployTaskRequest, tx dal.Transaction, logger log.Logger) errors.Error {
func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection, request *WebhookDeploymentReq, tx dal.Transaction, logger log.Logger) errors.Error {
// validation
if request == nil {
return errors.BadInput.New("request body is nil")
}
if len(request.DeploymentCommits) == 0 {
return errors.BadInput.New("deployment_commits is empty")
}
// set default values for optional fields
deploymentId := request.Id
if deploymentId == "" {
deploymentId = request.PipelineId
}
scopeId := fmt.Sprintf("%s:%d", "webhook", connection.ID)
if request.CreatedDate == nil {
request.CreatedDate = request.StartedDate
Expand All @@ -92,147 +141,90 @@ func CreateDeploymentAndDeploymentCommits(connection *models.WebhookConnection,
duration := float64(request.FinishedDate.Sub(*request.StartedDate).Milliseconds() / 1e3)
name := request.Name
if name == "" {
if request.DeploymentCommits == nil {
name = fmt.Sprintf(`deployment for %s`, request.CommitSha)
} else {
var commitShaList []string
for _, commit := range request.DeploymentCommits {
commitShaList = append(commitShaList, commit.CommitSha)
}
name = fmt.Sprintf(`deployment for %s`, strings.Join(commitShaList, ","))
var commitShaList []string
for _, commit := range request.DeploymentCommits {
commitShaList = append(commitShaList, commit.CommitSha)
}
name = fmt.Sprintf(`deploy %s to %s`, strings.Join(commitShaList, ","), request.Environment)
}
createdDate := time.Now()
if request.CreatedDate != nil {
createdDate = *request.CreatedDate
} else if request.StartedDate != nil {
createdDate = *request.StartedDate
}
dateInfo := devops.TaskDatesInfo{
CreatedDate: createdDate,
// QueuedDate: request.QueuedDate,
StartedDate: request.StartedDate,
FinishedDate: request.FinishedDate,
}

// prepare deploymentCommits and deployment records
// queuedDuration := dateInfo.CalculateQueueDuration()
if request.DeploymentCommits == nil {
if request.CommitSha == "" || request.RepoUrl == "" {
return errors.Convert(fmt.Errorf("commit_sha or repo_url is required"))
deploymentCommits := make([]*devops.CicdDeploymentCommit, len(request.DeploymentCommits))
for i, commit := range request.DeploymentCommits {
if commit.Result == "" {
commit.Result = devops.RESULT_SUCCESS
}
if commit.Status == "" {
commit.Status = devops.STATUS_DONE
}
if commit.Name == "" {
commit.Name = fmt.Sprintf(`deployment for %s`, commit.CommitSha)
}
if commit.CreatedDate == nil {
commit.CreatedDate = &createdDate
}
if commit.StartedDate == nil {
commit.StartedDate = request.StartedDate
}
if commit.FinishedDate == nil {
commit.FinishedDate = request.FinishedDate
}
// create a deployment_commit record
deploymentCommit := &devops.CicdDeploymentCommit{
deploymentCommits[i] = &devops.CicdDeploymentCommit{
DomainEntity: domainlayer.DomainEntity{
Id: GenerateDeploymentCommitId(connection.ID, request.PipelineId, request.RepoUrl, request.CommitSha),
Id: GenerateDeploymentCommitId(connection.ID, deploymentId, commit.RepoUrl, commit.CommitSha),
},
CicdDeploymentId: request.PipelineId,
CicdDeploymentId: deploymentId,
CicdScopeId: scopeId,
Name: name,
DisplayTitle: request.DisplayTitle,
Result: request.Result,
Status: devops.STATUS_DONE,
OriginalResult: request.Result,
OriginalStatus: devops.STATUS_DONE,
TaskDatesInfo: dateInfo,
DurationSec: &duration,
//QueuedDurationSec: queuedDuration,
RepoId: request.RepoId,
RepoUrl: request.RepoUrl,
Result: commit.Result,
Status: commit.Status,
OriginalResult: commit.Result,
OriginalStatus: commit.Status,
TaskDatesInfo: devops.TaskDatesInfo{
CreatedDate: *commit.CreatedDate,
StartedDate: commit.StartedDate,
FinishedDate: commit.FinishedDate,
},
DurationSec: &duration,
RepoId: commit.RepoId,
Name: commit.Name,
DisplayTitle: commit.DisplayTitle,
RepoUrl: commit.RepoUrl,
Environment: request.Environment,
OriginalEnvironment: request.Environment,
RefName: request.RefName,
CommitSha: request.CommitSha,
CommitMsg: request.CommitMsg,
}
if err := tx.CreateOrUpdate(deploymentCommit); err != nil {
logger.Error(err, "create deployment commit")
return err
}
// create a deployment record
if err := tx.CreateOrUpdate(deploymentCommit.ToDeploymentWithCustomDisplayTitle(request.DisplayTitle)); err != nil {
logger.Error(err, "create deployment")
return err
}
} else {
for _, commit := range request.DeploymentCommits {
// create a deployment_commit record
deploymentCommit := &devops.CicdDeploymentCommit{
DomainEntity: domainlayer.DomainEntity{
Id: GenerateDeploymentCommitId(connection.ID, request.PipelineId, commit.RepoUrl, commit.CommitSha),
},
CicdDeploymentId: request.PipelineId,
CicdScopeId: scopeId,
Result: request.Result,
Status: devops.STATUS_DONE,
OriginalResult: request.Result,
OriginalStatus: devops.STATUS_DONE,
TaskDatesInfo: dateInfo,
DurationSec: &duration,
//QueuedDurationSec: queuedDuration,
RepoId: request.RepoId,
Name: fmt.Sprintf(`deployment for %s`, commit.CommitSha),
DisplayTitle: commit.DisplayTitle,
RepoUrl: commit.RepoUrl,
Environment: request.Environment,
OriginalEnvironment: request.Environment,
RefName: commit.RefName,
CommitSha: commit.CommitSha,
CommitMsg: commit.CommitMsg,
}

if err := tx.CreateOrUpdate(deploymentCommit); err != nil {
logger.Error(err, "create deployment commit")
return err
}

// create a deployment record
deploymentCommit.Name = name
if err := tx.CreateOrUpdate(deploymentCommit.ToDeploymentWithCustomDisplayTitle(request.DisplayTitle)); err != nil {
logger.Error(err, "create deployment")
return err
}
RefName: commit.RefName,
CommitSha: commit.CommitSha,
CommitMsg: commit.CommitMsg,
//QueuedDurationSec: queuedDuration,
}
}
return nil
}

// PostDeploymentCicdTask
// @Summary create deployment by webhook
// @Description Create deployment pipeline by webhook.<br/>
// @Description example1: {"repo_url":"devlake","commit_sha":"015e3d3b480e417aede5a1293bd61de9b0fd051d","start_time":"2020-01-01T12:00:00+00:00","end_time":"2020-01-01T12:59:59+00:00","environment":"PRODUCTION"}<br/>
// @Description So we suggest request before task after deployment pipeline finish.
// @Description Both cicd_pipeline and cicd_task will be created
// @Tags plugins/webhook
// @Param body body WebhookDeployTaskRequest true "json body"
// @Success 200
// @Failure 400 {string} errcode.Error "Bad Request"
// @Failure 403 {string} errcode.Error "Forbidden"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/webhook/connections/:connectionId/deployments [POST]
func PostDeploymentCicdTask(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
connection := &models.WebhookConnection{}
err := connectionHelper.First(connection, input.Params)
if err != nil {
return nil, err
}
// get request
request := &WebhookDeployTaskRequest{}
err = api.DecodeMapStruct(input.Body, request, true)
if err != nil {
return &plugin.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil
if err := tx.CreateOrUpdate(deploymentCommits); err != nil {
logger.Error(err, "failed to save deployment commits")
return err
}
// validate
vld = validator.New()
err = errors.Convert(vld.Struct(request))
if err != nil {
return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`)
}
txHelper := dbhelper.NewTxHelper(basicRes, &err)
defer txHelper.End()
tx := txHelper.Begin()
if err := CreateDeploymentAndDeploymentCommits(connection, request, tx, logger); err != nil {
logger.Error(err, "create deployments")
return nil, err

// create a deployment record
deployment := deploymentCommits[0].ToDeploymentWithCustomDisplayTitle(request.DisplayTitle)
deployment.Name = name
deployment.CreatedDate = createdDate
deployment.StartedDate = request.StartedDate
deployment.FinishedDate = request.FinishedDate
if err := tx.CreateOrUpdate(deployment); err != nil {
logger.Error(err, "failed to save deployment")
return err
}
return nil
}

return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
func GenerateDeploymentCommitId(connectionId uint64, deploymentId string, repoUrl string, commitSha string) string {
urlHash16 := fmt.Sprintf("%x", md5.Sum([]byte(repoUrl)))[:16]
return fmt.Sprintf("%s:%d:%s:%s:%s", "webhook", connectionId, deploymentId, urlHash16, commitSha)
}
4 changes: 2 additions & 2 deletions backend/plugins/webhook/impl/impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler
"DELETE": api.DeleteConnection,
},
"connections/:connectionId/deployments": {
"POST": api.PostDeploymentCicdTask,
"POST": api.PostDeployments,
},
"connections/:connectionId/issues": {
"POST": api.PostIssue,
Expand All @@ -97,7 +97,7 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler
"POST": api.CloseIssue,
},
":connectionId/deployments": {
"POST": api.PostDeploymentCicdTask,
"POST": api.PostDeployments,
},
":connectionId/issues": {
"POST": api.PostIssue,
Expand Down

0 comments on commit 05f1d34

Please sign in to comment.