Merge pull request #1434 from sawyersong2/master

feat: 构建机定时清理时做一下延迟判断 #1428

src/backend/ci/core/dispatch/api-dispatch-docker/src/main/kotlin/com/tencent/devops/dispatch/api/op/OPDispatchDockerResource.kt

@@ -98,4 +98,15 @@ interface OPDispatchDockerResource {
         @ApiParam("创建IDC构建机所需信息", required = true)
         dockerHostLoadConfigMap: Map<String, DockerHostLoadConfig>
     ): Result<Boolean>
+
+    @POST
+    @Path("/docker/threshold/update")
+    @ApiOperation("更新docker漂移负载阈值")
+    fun updateDockerDriftThreshold(
+        @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE)
+        @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID)
+        userId: String,
+        @ApiParam("阈值", required = true)
+        threshold: Int
+    ): Result<Boolean>
 }

src/backend/ci/core/dispatch/biz-dispatch-docker/src/main/kotlin/com/tencent/devops/dispatch/controller/OPDispatchDockerResourceImpl.kt

@@ -44,4 +44,8 @@ class OPDispatchDockerResourceImpl constructor(
     ): Result<Boolean> {
         return Result(dispatchDockerService.createDockerHostLoadConfig(userId, dockerHostLoadConfigMap))
     }
+
+    override fun updateDockerDriftThreshold(userId: String, threshold: Int): Result<Boolean> {
+        return Result(dispatchDockerService.updateDockerDriftThreshold(userId, threshold))
+    }
 }

src/backend/ci/core/dispatch/biz-dispatch-docker/src/main/kotlin/com/tencent/devops/dispatch/service/DispatchDockerService.kt

@@ -184,4 +184,19 @@ class DispatchDockerService @Autowired constructor(
             throw RuntimeException("OP dispatcheDocker create dockerhost loadConfig error.")
         }
     }
+
+    fun updateDockerDriftThreshold(userId: String, threshold: Int): Boolean {
+        logger.info("$userId updateDockerDriftThreshold $threshold")
+        if (threshold < 0 || threshold > 100) {
+            throw RuntimeException("Parameter threshold must in (0-100).")
+        }
+
+        try {
+            dockerHostUtils.updateDockerDriftThreshold(threshold)
+            return true
+        } catch (e: Exception) {
+            logger.error("OP dispatcheDocker update Docker DriftThreshold error.", e)
+            throw RuntimeException("OP dispatcheDocker update Docker DriftThreshold error.")
+        }
+    }
 }

src/backend/ci/core/dispatch/biz-dispatch-docker/src/main/kotlin/com/tencent/devops/dispatch/utils/DockerHostUtils.kt

@@ -69,6 +69,7 @@ class DockerHostUtils @Autowired constructor(
 ) {
     companion object {
         private const val LOAD_CONFIG_KEY = "dockerhost-load-config"
+        private const val DOCKER_DRIFT_THRESHOLD_KEY = "docker-drift-threshold-spKyQ86qdYhAkDDR"
         private const val DOCKER_IP_COUNT_MAX = 8
         private const val BUILD_POOL_SIZE = 100 // 单个流水线可同时执行的任务数量
 
@@ -178,9 +179,9 @@ class DockerHostUtils @Autowired constructor(
         pipelineId: String,
         vmSeq: String
     ): Int {
-        val lock = RedisLock(redisOperation, "DISPATCH_DEVCLOUD_LOCK_CONTAINER_${pipelineId}_$vmSeq", 30)
+        val lock = RedisLock(redisOperation, "DISPATCH_DOCKER_LOCK_CONTAINER_${pipelineId}_$vmSeq", 30)
         try {
-            lock.tryLock()
+            lock.lock()
             for (i in 1..BUILD_POOL_SIZE) {
                 logger.info("poolNo is $i")
                 val poolNo = pipelineDockerPoolDao.getPoolNoStatus(dslContext, pipelineId, vmSeq, i)
@@ -265,7 +266,8 @@ class DockerHostUtils @Autowired constructor(
         }
 
         // 查看当前IP负载情况，当前IP不可用或者负载超额或者设置为专机独享或者是否灰度已被切换，重新选择构建机
-        if (!dockerIpInfo.enable || dockerIpInfo.diskLoad > 90 || dockerIpInfo.memLoad > 95 || dockerIpInfo.specialOn || (dockerIpInfo.grayEnv != gray.isGray())) {
+        val threshold = getDockerDriftThreshold()
+        if (!dockerIpInfo.enable || dockerIpInfo.diskLoad > threshold || dockerIpInfo.memLoad > threshold || dockerIpInfo.specialOn || (dockerIpInfo.grayEnv != gray.isGray())) {
             val pair = getAvailableDockerIpWithSpecialIps(
                 event.projectId,
                 event.pipelineId,
@@ -342,6 +344,19 @@ class DockerHostUtils @Autowired constructor(
         )
     }
 
+    fun updateDockerDriftThreshold(threshold: Int) {
+        redisOperation.set(DOCKER_DRIFT_THRESHOLD_KEY, threshold.toString())
+    }
+
+    fun getDockerDriftThreshold(): Int {
+        val thresholdStr = redisOperation.get(DOCKER_DRIFT_THRESHOLD_KEY)
+        return if (thresholdStr != null && thresholdStr.isNotEmpty()) {
+            thresholdStr.toInt()
+        } else {
+            90
+        }
+    }
+
     private fun selectAvailableDockerIp(
         dockerIpList: List<TDispatchPipelineDockerIpInfoRecord>,
         unAvailableIpList: Set<String> = setOf()

src/backend/ci/core/dockerhost/biz-dockerhost/src/main/kotlin/com/tencent/devops/dockerhost/services/DockerHostBuildService.kt

@@ -72,7 +72,9 @@ import org.springframework.stereotype.Component
 import java.io.File
 import java.io.IOException
 import java.nio.file.Paths
+import java.text.SimpleDateFormat
 import java.util.Date
+import java.util.TimeZone
 import javax.annotation.PostConstruct
 
 @Component
@@ -601,8 +603,16 @@ class DockerHostBuildService(
     fun clearContainers() {
         val containerInfo = dockerCli.listContainersCmd().withStatusFilter(setOf("exited")).exec()
         for (container in containerInfo) {
-            logger.info("Clear container, containerId: ${container.id}")
-            dockerCli.removeContainerCmd(container.id).exec()
+            try {
+                val finishTime = dockerCli.inspectContainerCmd(container.id).exec().state.finishedAt
+                // 是否已退出30分钟
+                if (checkFinishTime(finishTime)) {
+                    logger.info("Clear container, containerId: ${container.id}")
+                    dockerCli.removeContainerCmd(container.id).exec()
+                }
+            } catch (e: Exception) {
+                logger.error("Clear container failed, containerId: ${container.id}", e)
+            }
         }
     }
 
@@ -725,6 +735,22 @@ class DockerHostBuildService(
         }
     }
 
+    private fun checkFinishTime(utcTime: String?): Boolean {
+        if (utcTime != null && utcTime.isNotEmpty()) {
+            val array = utcTime.split(".")
+            val utcTimeLocal = array[0] + "Z"
+            val sdf = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'")
+            sdf.timeZone = TimeZone.getTimeZone("UTC")
+
+            val date = sdf.parse(utcTimeLocal)
+            val finishTimestamp = date.time
+            val nowTimestamp = System.currentTimeMillis()
+            return (nowTimestamp - finishTimestamp) > (30 * 60 * 1000)
+        }
+
+        return true
+    }
+
     inner class MyBuildImageResultCallback internal constructor(
         private val buildId: String,
         private val elementId: String?,