From 7ae243664a69864e81cd7578a2eb77355b40f4df Mon Sep 17 00:00:00 2001 From: royalhuang Date: Wed, 24 May 2023 13:25:53 +0800 Subject: [PATCH 1/3] =?UTF-8?q?perf:=20=E6=97=A5=E5=BF=97=E5=8D=95?= =?UTF-8?q?=E8=A1=8C=E8=BE=93=E5=87=BA=E8=B6=85=E8=BF=87=E9=99=90=E5=88=B6?= =?UTF-8?q?=EF=BC=8816KB=EF=BC=89=E8=87=AA=E5=8A=A8=E5=88=86=E8=A1=8C=20#8?= =?UTF-8?q?856?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../worker/common/logger/LoggerService.kt | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt index 1962f4efcfd..588222a7260 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt @@ -32,10 +32,8 @@ import com.tencent.devops.common.log.pojo.TaskBuildLogProperty import com.tencent.devops.common.log.pojo.enums.LogStorageMode import com.tencent.devops.common.log.pojo.enums.LogType import com.tencent.devops.common.log.pojo.message.LogMessage -import com.tencent.devops.common.service.utils.CommonUtils import com.tencent.devops.common.service.utils.ZipUtil import com.tencent.devops.common.util.HttpRetryUtils -import com.tencent.devops.log.meta.Ansi import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.worker.common.LOG_DEBUG_FLAG @@ -265,9 +263,16 @@ object LoggerService { try { if (currentTaskLineNo <= LOG_TASK_LINE_LIMIT) { + var offset = 0 // 上报前做长度等内容限制 - fixUploadMessage(logMessage) - this.uploadQueue.put(logMessage) + while (offset < logMessage.message.length) { + val chunk = logMessage.message.substring( + offset, minOf(offset + LOG_MESSAGE_LENGTH_LIMIT, logMessage.message.length) + ) + this.uploadQueue.put(logMessage.copy(message = chunk)) + offset += LOG_MESSAGE_LENGTH_LIMIT + } + } else if (elementId2LogProperty[elementId]?.logStorageMode != LogStorageMode.LOCAL) { logger.warn( "The number of Task[$elementId] log lines exceeds the limit, " + @@ -471,19 +476,6 @@ object LoggerService { } } - private fun fixUploadMessage(logMessage: LogMessage) { - // 字符数超过32766时analyzer索引分析将失效，同时为保护系统稳定性，若配置值为空或负数则限制为32KB - if (logMessage.message.length > LOG_MESSAGE_LENGTH_LIMIT) { - logMessage.message = Ansi().bold().fgYellow() - .a("[Length exceeds limit]") - .reset().toString() - } - logMessage.message = CommonUtils.interceptStringInLength( - string = logMessage.message, - length = LOG_MESSAGE_LENGTH_LIMIT - ) ?: "" - } - private fun disableLogUpload() { // 将已有任务的日志模式都设为本地保存 elementId2LogProperty.forEach { (elementId, property) -> From b77e615cbdf472229453ce9f0705ad6380248bd8 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Wed, 24 May 2023 14:17:09 +0800 Subject: [PATCH 2/3] =?UTF-8?q?perf:=20=E6=97=A5=E5=BF=97=E5=8D=95?= =?UTF-8?q?=E8=A1=8C=E8=BE=93=E5=87=BA=E8=B6=85=E8=BF=87=E9=99=90=E5=88=B6?= =?UTF-8?q?=EF=BC=8816KB=EF=BC=89=E8=87=AA=E5=8A=A8=E5=88=86=E8=A1=8C=20#8?= =?UTF-8?q?856?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/worker/common/Constants.kt | 2 +- .../com/tencent/devops/worker/common/logger/LoggerService.kt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt index 467ab8957f5..b33b121d15a 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt @@ -65,7 +65,7 @@ const val LOG_SUBTAG_FINISH_FLAG = "##subTagFinish##" const val LOG_UPLOAD_BUFFER_SIZE = 200 -const val LOG_MESSAGE_LENGTH_LIMIT = 32000 +const val LOG_MESSAGE_LENGTH_LIMIT = 16 * 1024 // 16KB const val LOG_TASK_LINE_LIMIT = 1000000 diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt index 588222a7260..d0deea3d342 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt @@ -272,7 +272,6 @@ object LoggerService { this.uploadQueue.put(logMessage.copy(message = chunk)) offset += LOG_MESSAGE_LENGTH_LIMIT } - } else if (elementId2LogProperty[elementId]?.logStorageMode != LogStorageMode.LOCAL) { logger.warn( "The number of Task[$elementId] log lines exceeds the limit, " + From b482216470a1396c197449178dcc470102fd7ec4 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Wed, 24 May 2023 16:28:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?perf:=20=E6=97=A5=E5=BF=97=E5=8D=95?= =?UTF-8?q?=E8=A1=8C=E8=BE=93=E5=87=BA=E8=B6=85=E8=BF=87=E9=99=90=E5=88=B6?= =?UTF-8?q?=EF=BC=8816KB=EF=BC=89=E8=87=AA=E5=8A=A8=E5=88=86=E8=A1=8C=20#8?= =?UTF-8?q?856?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/worker/common/logger/LoggerService.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt index d0deea3d342..c364da0dcff 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt @@ -65,7 +65,7 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.locks.ReentrantLock -@Suppress("MagicNumber", "TooManyFunctions", "ComplexMethod") +@Suppress("MagicNumber", "TooManyFunctions", "ComplexMethod", "LongMethod") object LoggerService { private val logResourceApi = ApiFactory.create(LogSDKApi::class) @@ -181,8 +181,8 @@ object LoggerService { Runtime.getRuntime().addShutdownHook(object : Thread() { override fun run() = loggerService.stop() }) - } catch (t: Throwable) { - logger.warn("Fail to add shutdown hook", t) + } catch (ignore: Throwable) { + logger.warn("Fail to add shutdown hook", ignore) } }