From 8bedf3c344475a0d17ce272a98e982e810c3751b Mon Sep 17 00:00:00 2001 From: Lynwee Date: Mon, 25 Sep 2023 14:43:04 +0800 Subject: [PATCH 1/4] fix(bamboo): set domain layer's repo_url field with an acceptable value --- .../plugins/bamboo/e2e/deploy_build_test.go | 17 +++++++++ backend/plugins/bamboo/e2e/deploy_test.go | 1 + backend/plugins/bamboo/e2e/job_build_test.go | 1 + backend/plugins/bamboo/e2e/job_test.go | 1 + .../bamboo/e2e/plan_build_commits_test.go | 1 + backend/plugins/bamboo/e2e/plan_build_test.go | 1 + backend/plugins/bamboo/e2e/plan_test.go | 1 + .../cicd_deployment_commits.csv | 16 ++++----- .../snapshot_tables/cicd_pipeline_commits.csv | 22 ++++++------ .../bamboo/tasks/deploy_build_convertor.go | 7 ++++ .../bamboo/tasks/plan_commit_convertor.go | 8 ++++- backend/plugins/bamboo/tasks/shared.go | 12 +++++++ backend/plugins/bamboo/tasks/shared_test.go | 35 +++++++++++++++++++ 13 files changed, 103 insertions(+), 20 deletions(-) diff --git a/backend/plugins/bamboo/e2e/deploy_build_test.go b/backend/plugins/bamboo/e2e/deploy_build_test.go index 3534395ff80..2a76dd479b2 100644 --- a/backend/plugins/bamboo/e2e/deploy_build_test.go +++ b/backend/plugins/bamboo/e2e/deploy_build_test.go @@ -18,7 +18,10 @@ limitations under the License. package e2e import ( + gocontext "context" + "github.com/apache/incubator-devlake/core/runner" "testing" + "time" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -29,6 +32,19 @@ import ( "github.com/apache/incubator-devlake/plugins/bamboo/tasks" ) +var basicContext = runner.CreateAppBasicRes() + +func getFakeAPIClient() *helper.ApiAsyncClient { + client, _ := helper.NewApiClient(gocontext.Background(), + "https://zentao.demo.haogs.cn/api.php/v1/", + nil, time.Second*5, "", + basicContext, + ) + return &helper.ApiAsyncClient{ + ApiClient: client, + } +} + func TestBambooDeployBuildDataFlow(t *testing.T) { var bamboo impl.Bamboo dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo) @@ -42,6 +58,7 @@ func TestBambooDeployBuildDataFlow(t *testing.T) { }, }, RegexEnricher: helper.NewRegexEnricher(), + ApiClient: getFakeAPIClient(), } taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern) taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern) diff --git a/backend/plugins/bamboo/e2e/deploy_test.go b/backend/plugins/bamboo/e2e/deploy_test.go index 62c7efc9bc9..54bf4a4a7ef 100644 --- a/backend/plugins/bamboo/e2e/deploy_test.go +++ b/backend/plugins/bamboo/e2e/deploy_test.go @@ -35,6 +35,7 @@ func TestBambooDeployDataFlow(t *testing.T) { PlanKey: "TEST1", BambooScopeConfig: new(models.BambooScopeConfig), }, + ApiClient: getFakeAPIClient(), } // import raw data table diff --git a/backend/plugins/bamboo/e2e/job_build_test.go b/backend/plugins/bamboo/e2e/job_build_test.go index 2f69c196361..5db603638f1 100644 --- a/backend/plugins/bamboo/e2e/job_build_test.go +++ b/backend/plugins/bamboo/e2e/job_build_test.go @@ -43,6 +43,7 @@ func TestBambooJobBuildDataFlow(t *testing.T) { }, }, RegexEnricher: helper.NewRegexEnricher(), + ApiClient: getFakeAPIClient(), } taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern) // import raw data table diff --git a/backend/plugins/bamboo/e2e/job_test.go b/backend/plugins/bamboo/e2e/job_test.go index 419b062f0f5..dc89148e7a3 100644 --- a/backend/plugins/bamboo/e2e/job_test.go +++ b/backend/plugins/bamboo/e2e/job_test.go @@ -40,6 +40,7 @@ func TestBambooJobDataFlow(t *testing.T) { ProductionPattern: "(?i)compile", }, }, + ApiClient: getFakeAPIClient(), } // import raw data table // SELECT * FROM _raw_bamboo_api_job INTO OUTFILE "/tmp/_raw_bamboo_api_jobs.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n'; diff --git a/backend/plugins/bamboo/e2e/plan_build_commits_test.go b/backend/plugins/bamboo/e2e/plan_build_commits_test.go index f52b894c89d..ed853348136 100644 --- a/backend/plugins/bamboo/e2e/plan_build_commits_test.go +++ b/backend/plugins/bamboo/e2e/plan_build_commits_test.go @@ -42,6 +42,7 @@ func TestBambooPlanBuildCommitsDataFlow(t *testing.T) { ProductionPattern: "(?i)compile", }, }, + ApiClient: getFakeAPIClient(), } dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_build_commits.csv", &models.BambooPlanBuildVcsRevision{}) dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_builds.csv", &models.BambooPlanBuild{}) diff --git a/backend/plugins/bamboo/e2e/plan_build_test.go b/backend/plugins/bamboo/e2e/plan_build_test.go index e83654e80b0..ca9755d351e 100644 --- a/backend/plugins/bamboo/e2e/plan_build_test.go +++ b/backend/plugins/bamboo/e2e/plan_build_test.go @@ -44,6 +44,7 @@ func TestBambooPlanBuildDataFlow(t *testing.T) { }, }, RegexEnricher: helper.NewRegexEnricher(), + ApiClient: getFakeAPIClient(), } taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern) taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern) diff --git a/backend/plugins/bamboo/e2e/plan_test.go b/backend/plugins/bamboo/e2e/plan_test.go index a8757081ada..09987c8b10d 100644 --- a/backend/plugins/bamboo/e2e/plan_test.go +++ b/backend/plugins/bamboo/e2e/plan_test.go @@ -39,6 +39,7 @@ func TestBambooPlanDataFlow(t *testing.T) { PlanKey: "TEST1", BambooScopeConfig: new(models.BambooScopeConfig), }, + ApiClient: getFakeAPIClient(), } dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plans.csv", models.BambooPlan{}) diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv index 9aa1fc2193f..b32bb639a9b 100644 --- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv +++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv @@ -1,8 +1,8 @@ -id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,created_date,started_date,finished_date,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id -bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,2023-07-31T02:16:41.000+00:00,2023-07-31T02:16:41.000+00:00,2023-07-31T02:16:41.000+00:00,0,,622595,, -bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,2023-07-31T03:50:10.000+00:00,2023-07-31T03:50:10.000+00:00,2023-07-31T03:50:10.000+00:00,0,,622595,, -bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,2023-07-31T03:51:14.000+00:00,2023-07-31T03:51:14.000+00:00,2023-07-31T03:51:14.000+00:00,0,,622595,, -bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,2023-07-31T03:52:32.000+00:00,2023-07-31T03:52:32.000+00:00,2023-07-31T03:52:32.000+00:00,0,,622595,, -bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,2023-08-01T01:31:53.000+00:00,2023-08-01T01:31:53.000+00:00,2023-08-01T01:31:53.000+00:00,0,,622595,, -bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,2023-08-01T01:32:00.000+00:00,2023-08-01T01:32:00.000+00:00,2023-08-01T01:32:00.000+00:00,0,,622595,, -bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,2023-08-03T01:49:07.000+00:00,2023-08-03T01:49:07.000+00:00,2023-08-03T01:49:07.000+00:00,0,,622595,, +id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id +bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv index 1a18d25dd50..55108471c9e 100644 --- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv +++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv @@ -1,12 +1,12 @@ pipeline_id,commit_sha,branch,repo_id,repo_url -bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis -bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis +bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_convertor.go index f8f39f37632..40d73d120df 100644 --- a/backend/plugins/bamboo/tasks/deploy_build_convertor.go +++ b/backend/plugins/bamboo/tasks/deploy_build_convertor.go @@ -49,6 +49,7 @@ type deployBuildWithVcsRevision struct { func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() + logger := taskCtx.GetLogger() rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_BUILD_TABLE) cursor, err := db.Cursor( dal.Select("db.*, pbc.repository_id, pbc.repository_name, pbc.vcs_revision_key"), @@ -109,6 +110,12 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error { duration := uint64(input.FinishedDate.Sub(*input.StartedDate).Seconds()) deploymentCommit.DurationSec = &duration } + fakeRepoUrl, err := generateFakeRepoUrl(data.ApiClient.GetEndpoint(), input.RepositoryId) + if err != nil { + logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), input.RepositoryId) + } else { + deploymentCommit.RepoId = fakeRepoUrl + } return []interface{}{deploymentCommit}, nil }, diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go b/backend/plugins/bamboo/tasks/plan_commit_convertor.go index a6d891eb9f0..b5336f033f5 100644 --- a/backend/plugins/bamboo/tasks/plan_commit_convertor.go +++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go @@ -40,6 +40,7 @@ var ConvertPlanVcsMeta = plugin.SubTaskMeta{ func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() + logger := taskCtx.GetLogger() rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_BUILD_TABLE) cursor, err := db.Cursor( dal.From(&models.BambooPlanBuildVcsRevision{}), @@ -61,9 +62,14 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error { domainPlanVcs := &devops.CiCDPipelineCommit{ PipelineId: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey), CommitSha: line.VcsRevisionKey, - RepoUrl: line.RepositoryName, } domainPlanVcs.RepoId = repoMap[line.RepositoryId] + fakeRepoUrl, err := generateFakeRepoUrl(data.ApiClient.GetEndpoint(), line.RepositoryId) + if err != nil { + logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), line.RepositoryId) + } else { + domainPlanVcs.RepoId = fakeRepoUrl + } return []interface{}{ domainPlanVcs, }, nil diff --git a/backend/plugins/bamboo/tasks/shared.go b/backend/plugins/bamboo/tasks/shared.go index f74af27a1ce..0d834a0f0d1 100644 --- a/backend/plugins/bamboo/tasks/shared.go +++ b/backend/plugins/bamboo/tasks/shared.go @@ -103,3 +103,15 @@ func getBambooHomePage(endpoint string) (string, error) { return fmt.Sprintf("%s://%s%s", protocol, host, bambooPath), nil } } + +// generateFakeRepoUrl will return a fake url for repo url field. +func generateFakeRepoUrl(endpoint string, repoId int) (string, error) { + if endpoint == "" { + return "", errors.Default.New("empty endpoint") + } + endpointURL, err := url.Parse(endpoint) + if err != nil { + return "", err + } + return fmt.Sprintf("fake://%s/repos/%d", endpointURL.Host, repoId), nil +} diff --git a/backend/plugins/bamboo/tasks/shared_test.go b/backend/plugins/bamboo/tasks/shared_test.go index 462408cd22e..bd03ab07d52 100644 --- a/backend/plugins/bamboo/tasks/shared_test.go +++ b/backend/plugins/bamboo/tasks/shared_test.go @@ -79,3 +79,38 @@ func Test_getBambooWebURL(t *testing.T) { }) } } + +func Test_generateFakeRepoURL(t *testing.T) { + type args struct { + endpoint string + repoID uint + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "t-1", + args: args{ + endpoint: "http://127.0.0.1:8080/abc", + repoID: 123, + }, + want: "fake://127.0.0.1:8080/repos/123", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := generateFakeRepoUrl(tt.args.endpoint, tt.args.repoID) + if (err != nil) != tt.wantErr { + t.Errorf("generateFakeRepoUrl() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("generateFakeRepoUrl() got = %v, want %v", got, tt.want) + } + }) + } +} From c06b6ed4732e9f75d7f9505864c04f0468d627dc Mon Sep 17 00:00:00 2001 From: Lynwee Date: Mon, 25 Sep 2023 14:54:44 +0800 Subject: [PATCH 2/4] fix(bamboo): fix e2e test --- .../plugins/bamboo/e2e/deploy_build_test.go | 16 ++++---------- .../cicd_deployment_commits.csv | 14 ++++++------ .../snapshot_tables/cicd_pipeline_commits.csv | 22 +++++++++---------- .../bamboo/tasks/deploy_build_convertor.go | 2 +- .../bamboo/tasks/plan_commit_convertor.go | 2 +- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/backend/plugins/bamboo/e2e/deploy_build_test.go b/backend/plugins/bamboo/e2e/deploy_build_test.go index 2a76dd479b2..bd95f8c0f72 100644 --- a/backend/plugins/bamboo/e2e/deploy_build_test.go +++ b/backend/plugins/bamboo/e2e/deploy_build_test.go @@ -18,11 +18,6 @@ limitations under the License. package e2e import ( - gocontext "context" - "github.com/apache/incubator-devlake/core/runner" - "testing" - "time" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/helpers/e2ehelper" @@ -30,16 +25,13 @@ import ( "github.com/apache/incubator-devlake/plugins/bamboo/impl" "github.com/apache/incubator-devlake/plugins/bamboo/models" "github.com/apache/incubator-devlake/plugins/bamboo/tasks" + "testing" + "time" ) -var basicContext = runner.CreateAppBasicRes() - func getFakeAPIClient() *helper.ApiAsyncClient { - client, _ := helper.NewApiClient(gocontext.Background(), - "https://zentao.demo.haogs.cn/api.php/v1/", - nil, time.Second*5, "", - basicContext, - ) + client := &helper.ApiClient{} + client.Setup("http://127.0.0.1:8080/bamboo/", nil, time.Second) return &helper.ApiAsyncClient{ ApiClient: client, } diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv index b32bb639a9b..5ff05bccd4d 100644 --- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv +++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv @@ -1,8 +1,8 @@ id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id -bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, -bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, -bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, -bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, -bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, -bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, -bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,0,,fake://zentao.demo.haogs.cn/repos/622595,, +bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv index 55108471c9e..dafc2e66e8b 100644 --- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv +++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv @@ -1,12 +1,12 @@ pipeline_id,commit_sha,branch,repo_id,repo_url -bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, -bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,fake://zentao.demo.haogs.cn/repos/622595, +bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 +bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595 diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_convertor.go index 40d73d120df..00beeb37895 100644 --- a/backend/plugins/bamboo/tasks/deploy_build_convertor.go +++ b/backend/plugins/bamboo/tasks/deploy_build_convertor.go @@ -114,7 +114,7 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), input.RepositoryId) } else { - deploymentCommit.RepoId = fakeRepoUrl + deploymentCommit.RepoUrl = fakeRepoUrl } return []interface{}{deploymentCommit}, nil diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go b/backend/plugins/bamboo/tasks/plan_commit_convertor.go index b5336f033f5..ec9384b49f6 100644 --- a/backend/plugins/bamboo/tasks/plan_commit_convertor.go +++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go @@ -68,7 +68,7 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), line.RepositoryId) } else { - domainPlanVcs.RepoId = fakeRepoUrl + domainPlanVcs.RepoUrl = fakeRepoUrl } return []interface{}{ domainPlanVcs, From b1bc1e474aa29f4b729c4d05e540889a108d108a Mon Sep 17 00:00:00 2001 From: Lynwee Date: Mon, 25 Sep 2023 14:56:22 +0800 Subject: [PATCH 3/4] fix(bamboo): fix conflicts --- backend/plugins/bamboo/e2e/deploy_build_test.go | 2 +- .../snapshot_tables/cicd_deployment_commits.csv | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/plugins/bamboo/e2e/deploy_build_test.go b/backend/plugins/bamboo/e2e/deploy_build_test.go index bd95f8c0f72..c659f849f37 100644 --- a/backend/plugins/bamboo/e2e/deploy_build_test.go +++ b/backend/plugins/bamboo/e2e/deploy_build_test.go @@ -97,6 +97,6 @@ func TestBambooDeployBuildDataFlow(t *testing.T) { dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{}, e2ehelper.TableOptions{ CSVRelPath: "./snapshot_tables/cicd_deployment_commits.csv", IgnoreTypes: []interface{}{common.NoPKModel{}}, - IgnoreFields: []string{"created_date", "started_date", "finished_date"}, + IgnoreFields: []string{}, }) } diff --git a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv index 5ff05bccd4d..20182662ef4 100644 --- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv +++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv @@ -1,8 +1,8 @@ -id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id -bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, -bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, -bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, -bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, -bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, -bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, -bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,0,,622595,fake://127.0.0.1:8080/repos/622595, +id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,created_date,started_date,finished_date,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id +bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, +bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595, From 6eeca4771e37d1fb5ab885eeb1bb108a894cf3e5 Mon Sep 17 00:00:00 2001 From: Lynwee Date: Mon, 25 Sep 2023 15:03:35 +0800 Subject: [PATCH 4/4] fix(test): fix test case --- backend/plugins/bamboo/tasks/shared_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/plugins/bamboo/tasks/shared_test.go b/backend/plugins/bamboo/tasks/shared_test.go index bd03ab07d52..941643c4842 100644 --- a/backend/plugins/bamboo/tasks/shared_test.go +++ b/backend/plugins/bamboo/tasks/shared_test.go @@ -83,7 +83,7 @@ func Test_getBambooWebURL(t *testing.T) { func Test_generateFakeRepoURL(t *testing.T) { type args struct { endpoint string - repoID uint + repoID int } tests := []struct { name string