Skip to content

Commit

Permalink
feat: add deployment metadata to the table of the deployment frequenc…
Browse files Browse the repository at this point in the history 
…y details dashboard (#7325)

* feat: add deployment metadata to the table of the 'DORA details - deployment frequency' dashboard

* fix: e2e test

* fix: e2e test

* fix: e2e test and unit test

* fix: DORADetails-DeploymentFrequency

* fix: adjust pg

* feat: jenkins plugin add display_title and url

* fix: jenkins e2e

* fix: bamboo and github

* fix: github bamboo and dashboard url add link

* fix: dashboard, bamboo, jenkins

* fix: bamboo + jenkins

* fix: jenkins
  • Loading branch information
@abeizn
abeizn committed Apr 18, 2024
1 parent 55fd2cc commit 470669d
Show file tree
Hide file tree
Showing 78 changed files with 1,320 additions and 808 deletions.
2 changes: 2 additions & 0 deletions backend/core/models/domainlayer/devops/cicd_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ type CICDDeployment struct {
domainlayer.DomainEntity
CicdScopeId string `gorm:"index;type:varchar(255)"`
Name string `gorm:"type:varchar(255)"`
DisplayTitle string
Url string
Result string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
Expand Down
4 changes: 4 additions & 0 deletions backend/core/models/domainlayer/devops/cicd_deployment_commit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ type CicdDeploymentCommit struct {
CicdScopeId string `gorm:"index;type:varchar(255)"`
CicdDeploymentId string `gorm:"type:varchar(255)"` // if it is converted from a cicd_pipeline_commit
Name string `gorm:"type:varchar(255)"`
DisplayTitle string
Url string
Result string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
Expand Down Expand Up @@ -70,5 +72,7 @@ func (cicdDeploymentCommit CicdDeploymentCommit) ToDeployment() *CICDDeployment
},
QueuedDurationSec: cicdDeploymentCommit.QueuedDurationSec,
DurationSec: cicdDeploymentCommit.DurationSec,
DisplayTitle: cicdDeploymentCommit.DisplayTitle,
Url: cicdDeploymentCommit.Url,
}
}
5 changes: 4 additions & 1 deletion backend/core/models/domainlayer/devops/cicd_pipeline.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@ limitations under the License.
package devops

import (
"github.com/spf13/cast"
"strings"

"github.com/spf13/cast"

"github.com/apache/incubator-devlake/core/models/domainlayer"
)

type CICDPipeline struct {
domainlayer.DomainEntity
Name string `gorm:"type:varchar(255)"`
DisplayTitle string
Url string
Result string `gorm:"type:varchar(100)"`
Status string `gorm:"type:varchar(100)"`
OriginalStatus string `gorm:"type:varchar(100)"`
Expand Down
14 changes: 8 additions & 6 deletions backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ import (

type CiCDPipelineCommit struct {
common.NoPKModel
PipelineId string `gorm:"primaryKey;type:varchar(255)"`
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
CommitMsg string
Branch string `gorm:"type:varchar(255)"`
RepoId string `gorm:"index;type:varchar(255)"`
RepoUrl string
PipelineId string `gorm:"primaryKey;type:varchar(255)"`
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
CommitMsg string
DisplayTitle string
Url string
Branch string `gorm:"type:varchar(255)"`
RepoId string `gorm:"index;type:varchar(255)"`
RepoUrl string
}

func (CiCDPipelineCommit) TableName() string {
Expand Down
77 changes: 77 additions & 0 deletions backend/core/models/migrationscripts/20240410_add_display_title_and_url_to_deployment.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
)

var _ plugin.MigrationScript = (*addDisplayTitleAndUrl)(nil)

type cicdDeployment20240410 struct {
DisplayTitle string
Url string
}

func (cicdDeployment20240410) TableName() string {
return "cicd_deployments"
}

type cicdDeploymentCommit20240410 struct {
DisplayTitle string
Url string
}

func (cicdDeploymentCommit20240410) TableName() string {
return "cicd_deployment_commits"
}

type cicdPipeline20240410 struct {
DisplayTitle string
Url string
}

func (cicdPipeline20240410) TableName() string {
return "cicd_pipelines"
}

type cicdPipelineCommit20240410 struct {
DisplayTitle string
Url string
}

func (cicdPipelineCommit20240410) TableName() string {
return "cicd_pipeline_commits"
}

type addDisplayTitleAndUrl struct{}

func (*addDisplayTitleAndUrl) Up(basicRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(basicRes, &cicdDeployment20240410{}, &cicdDeploymentCommit20240410{}, &cicdPipeline20240410{}, &cicdPipelineCommit20240410{})
}

func (*addDisplayTitleAndUrl) Version() uint64 {
return 20240410111248
}

func (*addDisplayTitleAndUrl) Name() string {
return "add display title and url to deployments and pipelines"
}
1 change: 1 addition & 0 deletions backend/core/models/migrationscripts/register.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,5 +111,6 @@ func All() []plugin.MigrationScript {
new(addSubtabknameToDeployment),
new(addStore),
new(addSubtaskField),
new(addDisplayTitleAndUrl),
}
}
4 changes: 2 additions & 2 deletions backend/plugins/bamboo/e2e/deploy_build_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,11 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_deployment_commits.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
IgnoreFields: []string{},
IgnoreFields: []string{"created_date", "queued_date", "started_date", "finished_date"},
})
dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/cicd_deployments.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
IgnoreFields: []string{},
IgnoreFields: []string{"created_date", "queued_date", "started_date", "finished_date"},
})
}
30 changes: 15 additions & 15 deletions backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,original_status,original_result,environment,original_environment,created_date,queued_date,started_date,finished_date,duration_sec,queued_duration_sec,commit_msg,ref_name,repo_id,repo_url,prev_success_deployment_commit_id
bamboo:deployBuildWithVcsRevision:1:130001:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130001:622595,test_project2 - test_plan/release-1,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130002:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130002:622595,test_project2 - test_plan/release-1,FAILURE,IN_PROGRESS,IN_PROGRESS,FAILED,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130003:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130003:622595,test_project2 - test_plan/release-1,,IN_PROGRESS,PENDING,REPLACED,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130004:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130004:622595,test_project2 - test_plan/release-1,,IN_PROGRESS,QUEUED,SKIPPED,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130005:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130005:622595,test_project2 - test_plan/release-1,,OTHER,NOT_BUILT,NEVER,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130006:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130006:622595,test_project2 - test_plan/release-1,,OTHER,NOT_BUILT,QUEUED,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130007:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130007:622595,test_project2 - test_plan/release-1,,OTHER,NOT_BUILT,IN PROGRESS,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:130008:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130008:622595,test_project2 - test_plan/release-1,,OTHER,NOT_BUILT,NOT BUILT,dev,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:1540100:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540100:622595,test_project2 - test_plan/release-1,FAILURE,DONE,FINISHED,FAILED,dev,dev,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:1540101:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540101:622595,test_project2 - test_plan/release-2,FAILURE,DONE,FINISHED,FAILED,dev,dev,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:1540102:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540102:622595,test_project2 - test_plan/release-2,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:1540105:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540105:622595,test_project2 - test_plan/release-2,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:1540106:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540106:622595,test_project2 - test_plan/release-2,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
bamboo:deployBuildWithVcsRevision:1:1540117:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540117:622595,test_project2 - test_plan/release-3,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,0,,,,622595,fake://127.0.0.1:8080/repos/622595,
id,commit_sha,cicd_scope_id,cicd_deployment_id,name,display_title,url,result,status,original_status,original_result,environment,original_environment,duration_sec,queued_duration_sec,commit_msg,ref_name,repo_id,repo_url,prev_success_deployment_commit_id,subtask_name
bamboo:deployBuildWithVcsRevision:1:130001:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130001:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130002:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130002:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,FAILURE,IN_PROGRESS,IN_PROGRESS,FAILED,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130003:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130003:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,,IN_PROGRESS,PENDING,REPLACED,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130004:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130004:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,,IN_PROGRESS,QUEUED,SKIPPED,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130005:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130005:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,,OTHER,NOT_BUILT,NEVER,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130006:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130006:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,,OTHER,NOT_BUILT,QUEUED,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130007:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130007:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,,OTHER,NOT_BUILT,IN PROGRESS,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:130008:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:130008:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,,OTHER,NOT_BUILT,NOT BUILT,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:1540100:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540100:622595,test_project2 - test_plan/release-1,test_project2 - test_plan/release-1,,FAILURE,DONE,FINISHED,FAILED,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:1540101:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540101:622595,test_project2 - test_plan/release-2,test_project2 - test_plan/release-2,,FAILURE,DONE,FINISHED,FAILED,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:1540102:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540102:622595,test_project2 - test_plan/release-2,test_project2 - test_plan/release-2,,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:1540105:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540105:622595,test_project2 - test_plan/release-2,test_project2 - test_plan/release-2,,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:1540106:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540106:622595,test_project2 - test_plan/release-2,test_project2 - test_plan/release-2,,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
bamboo:deployBuildWithVcsRevision:1:1540117:622595,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:deployBuildWithVcsRevision:1:1540117:622595,test_project2 - test_plan/release-3,test_project2 - test_plan/release-3,,SUCCESS,DONE,FINISHED,SUCCESS,dev,dev,0,,,,622595,fake://127.0.0.1:8080/repos/622595,,
Loading

0 comments on commit 470669d

Please sign in to comment.