cherry-pick fix(bamboo): set domain layer's repo_url field with an acceptable value to v0.19

backend/plugins/bamboo/e2e/deploy_build_test.go

@@ -18,17 +18,25 @@ limitations under the License.
 package e2e
 
 import (
-	"testing"
-
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/helpers/e2ehelper"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/bamboo/impl"
 	"github.com/apache/incubator-devlake/plugins/bamboo/models"
 	"github.com/apache/incubator-devlake/plugins/bamboo/tasks"
+	"testing"
+	"time"
 )
 
+func getFakeAPIClient() *helper.ApiAsyncClient {
+	client := &helper.ApiClient{}
+	client.Setup("http://127.0.0.1:8080/bamboo/", nil, time.Second)
+	return &helper.ApiAsyncClient{
+		ApiClient: client,
+	}
+}
+
 func TestBambooDeployBuildDataFlow(t *testing.T) {
 	var bamboo impl.Bamboo
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
@@ -42,6 +50,7 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
 			},
 		},
 		RegexEnricher: helper.NewRegexEnricher(),
+		ApiClient:     getFakeAPIClient(),
 	}
 	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern)
 	taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern)
@@ -89,6 +98,6 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
 	dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{}, e2ehelper.TableOptions{
 		CSVRelPath:   "./snapshot_tables/cicd_deployment_commits.csv",
 		IgnoreTypes:  []interface{}{common.NoPKModel{}},
-		IgnoreFields: []string{"created_date", "started_date", "finished_date"},
+		IgnoreFields: []string{},
 	})
 }

backend/plugins/bamboo/e2e/deploy_test.go

@@ -35,6 +35,7 @@ func TestBambooDeployDataFlow(t *testing.T) {
 			PlanKey:           "TEST1",
 			BambooScopeConfig: new(models.BambooScopeConfig),
 		},
+		ApiClient: getFakeAPIClient(),
 	}
 
 	// import raw data table

backend/plugins/bamboo/e2e/job_build_test.go

@@ -43,6 +43,7 @@ func TestBambooJobBuildDataFlow(t *testing.T) {
 			},
 		},
 		RegexEnricher: helper.NewRegexEnricher(),
+		ApiClient:     getFakeAPIClient(),
 	}
 	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern)
 	// import raw data table

backend/plugins/bamboo/e2e/job_test.go

@@ -40,6 +40,7 @@ func TestBambooJobDataFlow(t *testing.T) {
 				ProductionPattern: "(?i)compile",
 			},
 		},
+		ApiClient: getFakeAPIClient(),
 	}
 	// import raw data table
 	// SELECT * FROM _raw_bamboo_api_job INTO OUTFILE "/tmp/_raw_bamboo_api_jobs.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';

backend/plugins/bamboo/e2e/plan_build_commits_test.go

@@ -42,6 +42,7 @@ func TestBambooPlanBuildCommitsDataFlow(t *testing.T) {
 				ProductionPattern: "(?i)compile",
 			},
 		},
+		ApiClient: getFakeAPIClient(),
 	}
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_build_commits.csv", &models.BambooPlanBuildVcsRevision{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_builds.csv", &models.BambooPlanBuild{})

backend/plugins/bamboo/e2e/plan_build_test.go

@@ -44,6 +44,7 @@ func TestBambooPlanBuildDataFlow(t *testing.T) {
 			},
 		},
 		RegexEnricher: helper.NewRegexEnricher(),
+		ApiClient:     getFakeAPIClient(),
 	}
 	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern)
 	taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern)

backend/plugins/bamboo/e2e/plan_test.go

@@ -39,6 +39,7 @@ func TestBambooPlanDataFlow(t *testing.T) {
 			PlanKey:           "TEST1",
 			BambooScopeConfig: new(models.BambooScopeConfig),
 		},
+		ApiClient: getFakeAPIClient(),
 	}
 
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plans.csv", models.BambooPlan{})

backend/plugins/bamboo/e2e/snapshot_tables/cicd_deployment_commits.csv

@@ -1,8 +1,8 @@
-id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id
-bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,0,,622595,,
-bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,0,,622595,,
+id,commit_sha,cicd_scope_id,cicd_deployment_id,name,result,status,environment,created_date,started_date,finished_date,duration_sec,ref_name,repo_id,repo_url,prev_success_deployment_commit_id
+bamboo:BambooDeployBuild:1:1540097,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540097,release-1,FAILURE,DONE,dev,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,2023-07-31T10:16:41.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540100,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540100,release-1,FAILURE,DONE,dev,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,2023-07-31T11:50:10.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540101,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540101,release-2,FAILURE,DONE,dev,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,2023-07-31T11:51:14.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540102,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540102,release-2,,DONE,dev,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,2023-07-31T11:52:32.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540105,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540105,release-2,,DONE,dev,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,2023-08-01T09:31:53.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540106,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540106,release-2,,DONE,dev,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,2023-08-01T09:32:00.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,
+bamboo:BambooDeployBuild:1:1540117,79b062bd53af15c701193c90b543386557cb7a3a,bamboo:BambooPlan:1:TEST-PLA2,bamboo:BambooDeployBuild:1:1540117,release-3,,DONE,dev,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,2023-08-03T09:49:07.000+00:00,0,,622595,fake://127.0.0.1:8080/repos/622595,

backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv

@@ -1,12 +1,12 @@
 pipeline_id,commit_sha,branch,repo_id,repo_url
-bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
-bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,,devlake-louis
+bamboo:BambooPlanBuild:1:TEST-PLA2-10,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-11,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-12,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-2,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-3,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-4,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-5,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-6,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-7,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-8,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595
+bamboo:BambooPlanBuild:1:TEST-PLA2-9,79b062bd53af15c701193c90b543386557cb7a3a,,,fake://127.0.0.1:8080/repos/622595

backend/plugins/bamboo/tasks/deploy_build_convertor.go

@@ -59,6 +59,7 @@ func (deployBuildWithVcsRevision deployBuildWithVcsRevision) GenerateCICDDeploym
 
 func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error {
 	db := taskCtx.GetDal()
+	logger := taskCtx.GetLogger()
 	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_BUILD_TABLE)
 	cursor, err := db.Cursor(
 		dal.Select("db.*, pbc.repository_id, pbc.repository_name, pbc.vcs_revision_key, p.name as project_plan_name, p.project_name"),
@@ -120,6 +121,12 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error {
 				duration := uint64(input.FinishedDate.Sub(*input.StartedDate).Seconds())
 				deploymentCommit.DurationSec = &duration
 			}
+			fakeRepoUrl, err := generateFakeRepoUrl(data.ApiClient.GetEndpoint(), input.RepositoryId)
+			if err != nil {
+				logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), input.RepositoryId)
+			} else {
+				deploymentCommit.RepoUrl = fakeRepoUrl
+			}
 
 			return []interface{}{deploymentCommit}, nil
 		},

backend/plugins/bamboo/tasks/plan_commit_convertor.go

@@ -40,6 +40,7 @@ var ConvertPlanVcsMeta = plugin.SubTaskMeta{
 
 func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error {
 	db := taskCtx.GetDal()
+	logger := taskCtx.GetLogger()
 	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PLAN_BUILD_TABLE)
 	cursor, err := db.Cursor(
 		dal.From(&models.BambooPlanBuildVcsRevision{}),
@@ -61,9 +62,14 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error {
 			domainPlanVcs := &devops.CiCDPipelineCommit{
 				PipelineId: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey),
 				CommitSha:  line.VcsRevisionKey,
-				RepoUrl:    line.RepositoryName,
 			}
 			domainPlanVcs.RepoId = repoMap[line.RepositoryId]
+			fakeRepoUrl, err := generateFakeRepoUrl(data.ApiClient.GetEndpoint(), line.RepositoryId)
+			if err != nil {
+				logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), line.RepositoryId)
+			} else {
+				domainPlanVcs.RepoUrl = fakeRepoUrl
+			}
 			return []interface{}{
 				domainPlanVcs,
 			}, nil

backend/plugins/bamboo/tasks/shared.go

@@ -103,3 +103,15 @@ func getBambooHomePage(endpoint string) (string, error) {
 		return fmt.Sprintf("%s://%s%s", protocol, host, bambooPath), nil
 	}
 }
+
+// generateFakeRepoUrl will return a fake url for repo url field.
+func generateFakeRepoUrl(endpoint string, repoId int) (string, error) {
+	if endpoint == "" {
+		return "", errors.Default.New("empty endpoint")
+	}
+	endpointURL, err := url.Parse(endpoint)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("fake://%s/repos/%d", endpointURL.Host, repoId), nil
+}

backend/plugins/bamboo/tasks/shared_test.go

@@ -79,3 +79,38 @@ func Test_getBambooWebURL(t *testing.T) {
 		})
 	}
 }
+
+func Test_generateFakeRepoURL(t *testing.T) {
+	type args struct {
+		endpoint string
+		repoID   int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "t-1",
+			args: args{
+				endpoint: "http://127.0.0.1:8080/abc",
+				repoID:   123,
+			},
+			want:    "fake://127.0.0.1:8080/repos/123",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := generateFakeRepoUrl(tt.args.endpoint, tt.args.repoID)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("generateFakeRepoUrl() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("generateFakeRepoUrl() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}