Merge pull request #8510 from tangruotian/issue_8505

主集群和子集群间构建制品网关与流量网关的配置优化 #8505

src/agent/src/pkg/job/build.go

@@ -276,7 +276,8 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error {
 		"PROJECT_ID":            buildInfo.ProjectId,           //deprecated
 		"BUILD_ID":              buildInfo.BuildId,             //deprecated
 		"VM_SEQ_ID":             buildInfo.VmSeqId,             //deprecated
-
+		"DEVOPS_FILE_GATEWAY":   config.GAgentConfig.FileGateway,
+		"DEVOPS_GATEWAY":        config.GetGateWay(),
 	}
 	if config.GEnvVars != nil {
 		for k, v := range config.GEnvVars {

src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdPartyAgent/OpThirdPartyAgentResource.kt

@@ -29,6 +29,7 @@ package com.tencent.devops.environment.api.thirdPartyAgent
 
 import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID
 import com.tencent.devops.common.api.pojo.Result
+import com.tencent.devops.environment.pojo.slave.SlaveGateway
 import com.tencent.devops.environment.pojo.thirdPartyAgent.UpdateAgentRequest
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineCreate
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineResponse
@@ -37,6 +38,7 @@ import io.swagger.annotations.Api
 import io.swagger.annotations.ApiOperation
 import io.swagger.annotations.ApiParam
 import javax.ws.rs.Consumes
+import javax.ws.rs.DELETE
 import javax.ws.rs.GET
 import javax.ws.rs.HeaderParam
 import javax.ws.rs.POST
@@ -109,4 +111,34 @@ interface OpThirdPartyAgentResource {
         @ApiParam("内容", required = false)
         updateAgentRequest: UpdateAgentRequest
     ): Result<Boolean>
+
+    @ApiOperation("查询agent下载网关")
+    @GET
+    @Path("/gateways")
+    fun getGateways(): Result<List<SlaveGateway>>
+
+    @ApiOperation("新增agent下载网关")
+    @POST
+    @Path("/gateways")
+    fun addGateway(
+        @ApiParam("gateway", required = true)
+        gateway: SlaveGateway
+    ): Result<Boolean>
+
+    @ApiOperation("修改agent下载网关")
+    @PUT
+    @Path("/gateways")
+    fun updateGateway(
+        @ApiParam("gateway", required = true)
+        gateway: SlaveGateway
+    ): Result<Boolean>
+
+    @ApiOperation("删除agent下载网关")
+    @DELETE
+    @Path("/gateways/{zoneName}")
+    fun deleteGateway(
+        @ApiParam("zoneName", required = true)
+        @PathParam("zoneName")
+        zoneName: String
+    ): Result<Boolean>
 }

src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdPartyAgent/ServiceThirdPartyAgentResource.kt

@@ -35,6 +35,7 @@ import com.tencent.devops.common.api.pojo.Result
 import com.tencent.devops.common.api.pojo.agent.UpgradeItem
 import com.tencent.devops.common.web.annotation.BkField
 import com.tencent.devops.environment.pojo.AgentPipelineRefRequest
+import com.tencent.devops.environment.pojo.slave.SlaveGateway
 import com.tencent.devops.environment.pojo.thirdPartyAgent.AgentPipelineRef
 import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgent
 import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgentDetail
@@ -292,6 +293,11 @@ interface ServiceThirdPartyAgentResource {
         agentHashId: String
     ): Result<ThirdPartyAgentDetail?>
 
+    @ApiOperation("获取Gateway列表")
+    @GET
+    @Path("/gateways")
+    fun getGateways(): Result<List<SlaveGateway>>
+
     @ApiOperation("获取构建机详情(by node id)")
     @GET
     @Path("/projects/{projectId}/agent_detail_by_node_id")

src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdPartyAgent/UserThirdPartyAgentResource.kt

@@ -104,7 +104,10 @@ interface UserThirdPartyAgentResource {
         projectId: String,
         @ApiParam("操作系统", required = true)
         @PathParam("os")
-        os: OS
+        os: OS,
+        @ApiParam("可见性", required = false)
+        @QueryParam("visibility")
+        visibility: Boolean?
     ): Result<List<SlaveGateway>>
 
     @ApiOperation("查看Agent安装链接")

src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/slave/SlaveGateway.kt

@@ -37,5 +37,9 @@ data class SlaveGateway(
     @ApiModelProperty("展示名称")
     val showName: String,
     @ApiModelProperty("网关地址")
-    val gateway: String
+    val gateway: String,
+    @ApiModelProperty("文件网关地址")
+    val fileGateway: String?,
+    @ApiModelProperty("可见性")
+    val visibility: Boolean
 )

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/slave/SlaveGatewayDao.kt

@@ -27,6 +27,7 @@
 
 package com.tencent.devops.environment.dao.slave
 
+import com.tencent.devops.environment.pojo.slave.SlaveGateway
 import com.tencent.devops.model.environment.tables.TEnvironmentSlaveGateway
 import com.tencent.devops.model.environment.tables.records.TEnvironmentSlaveGatewayRecord
 import org.jooq.DSLContext
@@ -60,4 +61,30 @@ class SlaveGatewayDao {
             }
         }
     }
+
+    fun add(dslContext: DSLContext, gateWay: SlaveGateway): Boolean {
+        with(TEnvironmentSlaveGateway.T_ENVIRONMENT_SLAVE_GATEWAY) {
+            return dslContext.insertInto(this, NAME, SHOW_NAME, GATEWAY, FILE_GATEWAY, VISIBILITY)
+                .values(gateWay.zoneName, gateWay.showName, gateWay.gateway, gateWay.fileGateway, gateWay.visibility)
+                .execute() > 0
+        }
+    }
+
+    fun update(dslContext: DSLContext, gateWay: SlaveGateway): Boolean {
+        with(TEnvironmentSlaveGateway.T_ENVIRONMENT_SLAVE_GATEWAY) {
+            return dslContext.update(this)
+                .set(SHOW_NAME, gateWay.showName)
+                .set(GATEWAY, gateWay.gateway)
+                .set(FILE_GATEWAY, gateWay.fileGateway)
+                .set(VISIBILITY, gateWay.visibility)
+                .where(NAME.eq(gateWay.zoneName))
+                .execute() > 0
+        }
+    }
+
+    fun delete(dslContext: DSLContext, name: String): Boolean {
+        with(TEnvironmentSlaveGateway.T_ENVIRONMENT_SLAVE_GATEWAY) {
+            return dslContext.delete(this).where(NAME.eq(name)).execute() > 0
+        }
+    }
 }

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/OpThirdPartyAgentResourceImpl.kt

@@ -30,18 +30,21 @@ package com.tencent.devops.environment.resources.thirdPartyAgent
 import com.tencent.devops.common.api.pojo.Result
 import com.tencent.devops.common.web.RestResource
 import com.tencent.devops.environment.api.thirdPartyAgent.OpThirdPartyAgentResource
+import com.tencent.devops.environment.pojo.slave.SlaveGateway
 import com.tencent.devops.environment.pojo.thirdPartyAgent.UpdateAgentRequest
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineCreate
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineResponse
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineSeqId
+import com.tencent.devops.environment.service.slave.SlaveGatewayService
 import com.tencent.devops.environment.service.thirdPartyAgent.ThirdPartyAgentMgrService
 import com.tencent.devops.environment.service.thirdPartyAgent.ThirdPartyAgentPipelineService
 import org.springframework.beans.factory.annotation.Autowired
 
 @RestResource
 class OpThirdPartyAgentResourceImpl @Autowired constructor(
     private val thirdPartyAgentService: ThirdPartyAgentMgrService,
-    private val thirdPartyAgentPipelineService: ThirdPartyAgentPipelineService
+    private val thirdPartyAgentPipelineService: ThirdPartyAgentPipelineService,
+    private val slaveGatewayService: SlaveGatewayService
 ) : OpThirdPartyAgentResource {
 
     override fun listEnableProjects(): Result<List<String>> {
@@ -70,4 +73,20 @@ class OpThirdPartyAgentResourceImpl @Autowired constructor(
         thirdPartyAgentService.updateAgentGateway(updateAgentRequest)
         return Result(true)
     }
+
+    override fun getGateways(): Result<List<SlaveGateway>> {
+        return Result(slaveGatewayService.getGatewayNoCache())
+    }
+
+    override fun addGateway(gateway: SlaveGateway): Result<Boolean> {
+        return Result(slaveGatewayService.addGateway(gateway))
+    }
+
+    override fun updateGateway(gateway: SlaveGateway): Result<Boolean> {
+        return Result(slaveGatewayService.updateGateway(gateway))
+    }
+
+    override fun deleteGateway(zoneName: String): Result<Boolean> {
+        return Result(slaveGatewayService.deleteGateway(zoneName))
+    }
 }

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ServiceThirdPartyAgentResourceImpl.kt

@@ -39,6 +39,7 @@ import com.tencent.devops.environment.constant.EnvironmentMessageCode
 import com.tencent.devops.environment.constant.EnvironmentMessageCode.ERROR_NODE_NO_VIEW_PERMISSSION
 import com.tencent.devops.environment.permission.EnvironmentPermissionService
 import com.tencent.devops.environment.pojo.AgentPipelineRefRequest
+import com.tencent.devops.environment.pojo.slave.SlaveGateway
 import com.tencent.devops.environment.pojo.enums.NodeType
 import com.tencent.devops.environment.pojo.thirdPartyAgent.AgentPipelineRef
 import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgent
@@ -48,6 +49,7 @@ import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgentUpgrad
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineCreate
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineResponse
 import com.tencent.devops.environment.pojo.thirdPartyAgent.pipeline.PipelineSeqId
+import com.tencent.devops.environment.service.slave.SlaveGatewayService
 import com.tencent.devops.environment.service.NodeService
 import com.tencent.devops.environment.service.thirdPartyAgent.AgentPipelineService
 import com.tencent.devops.environment.service.thirdPartyAgent.ThirdPartyAgentMgrService
@@ -61,6 +63,7 @@ class ServiceThirdPartyAgentResourceImpl @Autowired constructor(
     private val upgradeService: UpgradeService,
     private val thirdPartyAgentPipelineService: ThirdPartyAgentPipelineService,
     private val agentPipelineService: AgentPipelineService,
+    private val slaveGatewayService: SlaveGatewayService,
     private val permissionService: EnvironmentPermissionService,
     private val nodeService: NodeService
 ) : ServiceThirdPartyAgentResource {
@@ -172,6 +175,10 @@ class ServiceThirdPartyAgentResourceImpl @Autowired constructor(
         return Result(thirdPartyAgentService.getAgentDetailById(userId, projectId, agentHashId = agentHashId))
     }
 
+    override fun getGateways(): Result<List<SlaveGateway>> {
+        return Result(slaveGatewayService.getGateway())
+    }
+
     override fun getNodeDetail(
         userId: String,
         projectId: String,

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/UserThirdPartyAgentResourceImpl.kt

@@ -69,10 +69,15 @@ class UserThirdPartyAgentResourceImpl @Autowired constructor(
         return Result(thirdPartyAgentService.generateAgent(userId, projectId, os, zoneName))
     }
 
-    override fun getGateway(userId: String, projectId: String, os: OS): Result<List<SlaveGateway>> {
+    override fun getGateway(
+        userId: String,
+        projectId: String,
+        os: OS,
+        visibility: Boolean?
+    ): Result<List<SlaveGateway>> {
         checkUserId(userId)
         checkProjectId(projectId)
-        return Result(slaveGatewayService.getGateway())
+        return Result(slaveGatewayService.getGateway().filter { it.visibility == (visibility ?: true) })
     }
 
     override fun getLink(userId: String, projectId: String, nodeId: String): Result<ThirdPartyAgentLink> {

src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/slave/SlaveGatewayService.kt

@@ -71,28 +71,34 @@ class SlaveGatewayService @Autowired constructor(
             val defaultFileGateway = agentPropsScope.getDefaultFileGateway()
             if (defaultFileGateway.isNotBlank()) return defaultFileGateway
         }
-        return getConfigGateway(zoneName)
+        val (gateway, fileGateWay) = getConfigGateway(zoneName)
+        return fileGateWay ?: gateway
     }
 
     fun getGateway(zoneName: String?): String? {
         if (agentPropsScope.useDefaultGateway()) {
             val defaultGateway = agentPropsScope.getDefaultGateway()
             if (defaultGateway.isNotBlank()) return defaultGateway
         }
-        return getConfigGateway(zoneName)
+        val (gateway, _) = getConfigGateway(zoneName)
+        return gateway
     }
 
-    private fun getConfigGateway(zoneName: String?): String {
+    // @return gateway,filegateway
+    private fun getConfigGateway(zoneName: String?): Pair<String, String?> {
         if (zoneName.isNullOrBlank()) {
-            return agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!)
+            return Pair(agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!), null)
         }
         val gateways = getGateway()
         gateways.forEach {
             if (it.zoneName == zoneName) {
-                return agentUrlService.fixGateway(it.gateway)
+                return Pair(
+                    agentUrlService.fixGateway(it.gateway),
+                    agentUrlService.fixGateway(it.fileGateway ?: it.gateway)
+                )
             }
         }
-        return agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!)
+        return Pair(agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!), null)
     }
 
     fun getGateway(): List<SlaveGateway> {
@@ -105,7 +111,15 @@ class SlaveGatewayService @Autowired constructor(
                         val records = slaveGatewayDao.list(dslContext)
                         if (records.isNotEmpty()) {
                             records.forEach {
-                                cache.add(SlaveGateway(it.name, it.showName, it.gateway))
+                                cache.add(
+                                    SlaveGateway(
+                                        zoneName = it.name,
+                                        showName = it.showName,
+                                        gateway = it.gateway,
+                                        fileGateway = it.fileGateway,
+                                        visibility = it.visibility ?: false
+                                    )
+                                )
                             }
                         }
                     } finally {
@@ -118,6 +132,40 @@ class SlaveGatewayService @Autowired constructor(
         return cache
     }
 
+    // 不走缓存的接口，给op接口使用
+    fun getGatewayNoCache(): List<SlaveGateway> {
+        val result = ArrayList<SlaveGateway>()
+
+        val records = slaveGatewayDao.list(dslContext)
+        if (records.isNotEmpty()) {
+            records.forEach {
+                result.add(
+                    SlaveGateway(
+                        zoneName = it.name,
+                        showName = it.showName,
+                        gateway = it.gateway,
+                        fileGateway = it.fileGateway,
+                        visibility = it.visibility ?: false
+                    )
+                )
+            }
+        }
+
+        return result
+    }
+
+    fun addGateway(gateway: SlaveGateway): Boolean {
+        return slaveGatewayDao.add(dslContext, gateway)
+    }
+
+    fun updateGateway(gateway: SlaveGateway): Boolean {
+        return slaveGatewayDao.update(dslContext, gateway)
+    }
+
+    fun deleteGateway(zoneName: String): Boolean {
+        return slaveGatewayDao.delete(dslContext, zoneName)
+    }
+
     private fun need2Refresh() =
         System.currentTimeMillis() - lastUpdate >= TimeUnit.MINUTES.toMillis(1)
 

support-files/sql/1001_ci_environment_ddl_mysql.sql

@@ -52,6 +52,8 @@ CREATE TABLE IF NOT EXISTS `T_ENVIRONMENT_SLAVE_GATEWAY` (
   `NAME` varchar(32) NOT NULL COMMENT '名称',
   `SHOW_NAME` varchar(32) NOT NULL COMMENT '展示名称',
   `GATEWAY` varchar(127) DEFAULT '' COMMENT '网关地址',
+  `FILE_GATEWAY` varchar(127) DEFAULT NULL COMMENT '文件网关地址',
+  `VISIBILITY` bit(1) DEFAULT b'0' COMMENT '是否在界面可见',
   PRIMARY KEY (`ID`),
   UNIQUE KEY `NAME` (`NAME`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='';

support-files/sql/2002_v1.x/2010_ci_environment-update_v1.10_mysql.sql

@@ -37,6 +37,26 @@ BEGIN
         END IF;
     END IF;
 
+    IF NOT EXISTS(SELECT 1
+                  FROM information_schema.COLUMNS
+                  WHERE TABLE_SCHEMA = db
+                    AND TABLE_NAME = 'T_ENVIRONMENT_SLAVE_GATEWAY'
+                    AND COLUMN_NAME = 'FILE_GATEWAY') THEN
+
+        ALTER TABLE `T_ENVIRONMENT_SLAVE_GATEWAY`
+            ADD COLUMN `FILE_GATEWAY` varchar(127) DEFAULT NULL COMMENT '文件网关地址';
+    END IF;
+
+    IF NOT EXISTS(SELECT 1
+                  FROM information_schema.COLUMNS
+                  WHERE TABLE_SCHEMA = db
+                    AND TABLE_NAME = 'T_ENVIRONMENT_SLAVE_GATEWAY'
+                    AND COLUMN_NAME = 'VISIBILITY') THEN
+
+        ALTER TABLE `T_ENVIRONMENT_SLAVE_GATEWAY`
+            ADD COLUMN `VISIBILITY` bit(1) DEFAULT b'0' COMMENT '是否在界面可见';
+    END IF;
+
     COMMIT;
 
 END <CI_UBF>