diff --git a/pom.xml b/pom.xml index d02fd66a69..3c417e946a 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ taier-data-develop taier-worker taier-datasource + taier-metrics diff --git a/taier-common/src/main/java/com/dtstack/taier/common/client/ClientProxy.java b/taier-common/src/main/java/com/dtstack/taier/common/client/ClientProxy.java index 4d8983b390..f96d4a32a0 100644 --- a/taier-common/src/main/java/com/dtstack/taier/common/client/ClientProxy.java +++ b/taier-common/src/main/java/com/dtstack/taier/common/client/ClientProxy.java @@ -20,7 +20,7 @@ import com.dtstack.taier.common.exception.LimitResourceException; import com.dtstack.taier.common.exception.TaierDefineException; -import com.dtstack.taier.pluginapi.CustomThreadFactory; +import com.dtstack.taier.metrics.collect.em.QueueTypeEnum; import com.dtstack.taier.pluginapi.JobClient; import com.dtstack.taier.pluginapi.JobIdentifier; import com.dtstack.taier.pluginapi.callback.CallBack; @@ -29,9 +29,9 @@ import com.dtstack.taier.pluginapi.enums.TaskStatus; import com.dtstack.taier.pluginapi.exception.ClientArgumentException; import com.dtstack.taier.pluginapi.exception.ExceptionUtil; +import com.dtstack.taier.pluginapi.metrics.DynamicMetricsThreadPoolUtil; import com.dtstack.taier.pluginapi.pojo.CheckResult; import com.dtstack.taier.pluginapi.pojo.ComponentTestResult; -import com.dtstack.taier.pluginapi.pojo.FileResult; import com.dtstack.taier.pluginapi.pojo.JobResult; import com.dtstack.taier.pluginapi.pojo.JudgeResult; @@ -41,8 +41,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -64,8 +62,12 @@ public class ClientProxy implements IClient { public ClientProxy(IClient targetClient) { this.targetClient = targetClient; - executorService = new ThreadPoolExecutor(1, 10, 0L, TimeUnit.MILLISECONDS, - new SynchronousQueue<>(), new CustomThreadFactory(targetClient.getClass().getSimpleName() + "_" + this.getClass().getSimpleName())); + + executorService = DynamicMetricsThreadPoolUtil.buildDynamicThreadPool( + 1, 10, 0L, TimeUnit.MILLISECONDS, + "ClientProxy", QueueTypeEnum.SYNCHRONOUS_QUEUE.getName(), null, false, this.timeout, + targetClient.getClass().getSimpleName() + "_" + this.getClass().getSimpleName(), + null, false); } @Override diff --git a/taier-common/src/main/java/com/dtstack/taier/common/env/EnvironmentContext.java b/taier-common/src/main/java/com/dtstack/taier/common/env/EnvironmentContext.java index c4f735ea89..70af802b7d 100644 --- a/taier-common/src/main/java/com/dtstack/taier/common/env/EnvironmentContext.java +++ b/taier-common/src/main/java/com/dtstack/taier/common/env/EnvironmentContext.java @@ -35,7 +35,11 @@ import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** @@ -271,6 +275,12 @@ public class EnvironmentContext implements InitializingBean { @Value("${schedule.scanning.cycle.day:1}") private Integer scanningCycleJobDay; + @Value("${engine.monitor.metrics.interval:5}") + private Long monitorMetricsInterval; + + @Value("${engine.monitor.metrics.delay:30}") + private Long monitorMetricsDelay; + @Override public void afterPropertiesSet() throws Exception { // 读取全局配置并初始化 datasource @@ -621,4 +631,63 @@ public Integer getTempSelectExpireTime() { public int getScanningCycleJobDay() { return scanningCycleJobDay; } + + public long getMonitorMetricsInterval() { + return monitorMetricsInterval; + } + + public long getMonitorMetricsDelay() { + return monitorMetricsDelay; + } + + /** + * 默认支持的监控指标推送类型 + * 平台内置的支持两种: micrometer, logging, output 默认logging + */ + public List getSupportMonitorMetricsType() { + String property = environment.getProperty("taier.monitor.metrics.support.type", "logging"); + if (StringUtils.isEmpty(property)) { + return Collections.emptyList(); + } + return Arrays.stream(StringUtils.split(property, ",")).map(String::valueOf) + .collect(Collectors.toList()); + } + + /** + */ + public boolean getIsMonitorMetrics() { + return Boolean.parseBoolean(environment.getProperty("taier.monitor.metrics.enabled", "true")); + } + + /** + * push gateway prometheus 默认url + */ + public String getPrometheusPushGatewayUrl() { + return environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.url", ""); + } + + /** + * push gateway prometheus 默认search参数 + */ + public String getPrometheusPushGatewaySearch() { + return environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.search", ""); + } + + /** + * push gateway 超时时间, 单位s 默认5s + */ + public long getPrometheusPushGatewayTimeout() { + return Long.parseLong(environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.timeout", "5")); + } + + /** + * push gateway 默认间隔时间,单位s, 默认60s + */ + public long getPrometheusPushGatewayInterval() { + return Long.parseLong(environment.getProperty("taier.monitor.metrics.prometheus.pushgateway.interval", "60")); + } + + + + } diff --git a/taier-metrics/pom.xml b/taier-metrics/pom.xml new file mode 100644 index 0000000000..f948dc44be --- /dev/null +++ b/taier-metrics/pom.xml @@ -0,0 +1,134 @@ + + + 4.0.0 + + com.dtstack.taier + taier + 1.0.0 + + + taier-metrics + + + 8 + 8 + UTF-8 + + + + + + io.micrometer + micrometer-registry-prometheus + 1.10.12 + + + + io.prometheus + simpleclient_pushgateway + 0.16.0 + + + + io.dropwizard.metrics + metrics-core + 4.2.37 + + + + + + + + org.projectlombok + lombok + 1.18.36 + + + + + com.google.guava + guava + + + + org.apache.commons + commons-lang3 + + + + commons-logging + commons-logging + 1.2 + true + + + + commons-collections + commons-collections + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + + + org.projectlombok + lombok + 1.18.36 + + + + + + + maven-source-plugin + 3.0.1 + + true + + + + compile + + jar + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.0.1 + + ../javadocs + taier-api-client + + -Xdoclint:none + + + + + + + diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/ExecutorWrapper.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/ExecutorWrapper.java new file mode 100644 index 0000000000..33334f162b --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/ExecutorWrapper.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics; + +import com.dtstack.taier.metrics.adapter.ExecutorAdapter; +import com.dtstack.taier.metrics.adapter.ThreadPoolExecutorAdapter; +import com.dtstack.taier.metrics.aware.AwareManager; +import com.dtstack.taier.metrics.aware.RejectHandlerAware; +import com.dtstack.taier.metrics.aware.TaskEnhanceAware; +import com.dtstack.taier.metrics.executor.EngineExecutor; +import com.dtstack.taier.metrics.rejects.RejectHandlerGetter; +import com.dtstack.taier.metrics.wrapper.TaskWrapper; +import lombok.Data; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Data +public class ExecutorWrapper { + + /** + * Thread pool name. + */ + private String threadPoolName; + + /** + * Thread pool alias name. + */ + private String threadPoolAliasName; + + /** + * Executor. + */ + private ExecutorAdapter executor; + + /** + * Notify platform ids. + */ + private List platformIds; + + /** + * Whether to enable notification. + */ + private boolean notifyEnabled = true; + + /** + * If enhance reject. + */ + private boolean rejectEnhanced = true; + + /** + * Aware names + */ + private Set awareNames = new HashSet<>(); + + /** + * Whether to wait for scheduled tasks to complete on shutdown, + * not interrupting running tasks and executing all tasks in the queue. + */ + protected boolean waitForTasksToCompleteOnShutdown = false; + + /** + * The maximum number of seconds that this executor is supposed to block + * on shutdown in order to wait for remaining tasks to complete their execution + * before the rest of the container continues to shut down. + */ + protected int awaitTerminationSeconds = 0; + + /** + * Thread pool stat provider + */ + private ThreadPoolStatProvider threadPoolStatProvider; + + private ExecutorWrapper() { + } + + /** + * Instantiates a new Executor wrapper. + * + * @param executor the DtpExecutor + */ + public ExecutorWrapper(EngineExecutor executor) { + this.executor = executor; + this.threadPoolName = executor.getThreadPoolName(); + this.threadPoolAliasName = executor.getThreadPoolAliasName(); + this.notifyEnabled = executor.isNotifyEnabled(); + this.platformIds = executor.getPlatformIds(); + this.awareNames = executor.getAwareNames(); + this.rejectEnhanced = executor.isRejectEnhanced(); + this.waitForTasksToCompleteOnShutdown = executor.isWaitForTasksToCompleteOnShutdown(); + this.awaitTerminationSeconds = executor.getAwaitTerminationSeconds(); + this.threadPoolStatProvider = ThreadPoolStatProvider.of(this); + } + + /** + * Instantiates a new Executor wrapper. + * + * @param threadPoolName the thread pool name + * @param executor the executor + */ + public ExecutorWrapper(String threadPoolName, Executor executor) { + this.threadPoolName = threadPoolName; + if (executor instanceof ThreadPoolExecutor) { + this.executor = new ThreadPoolExecutorAdapter((ThreadPoolExecutor) executor); + } else if (executor instanceof ExecutorAdapter) { + this.executor = (ExecutorAdapter) executor; + } else { + throw new IllegalArgumentException("unsupported Executor type !"); + } + this.threadPoolStatProvider = ThreadPoolStatProvider.of(this); + } + + /** + * Create executor wrapper. + * + * @param executor the executor + * @return the executor wrapper + */ + public static ExecutorWrapper of(EngineExecutor executor) { + return new ExecutorWrapper(executor); + } + + /** + * Initialize. + */ + public void initialize() { + if (isDtpExecutor()) { + ((EngineExecutor) getExecutor()).initialize(); + AwareManager.register(this); + } else if (isThreadPoolExecutor()) { + AwareManager.register(this); + } + } + + /** + * whether is DtpExecutor + * + * @return boolean + */ + public boolean isDtpExecutor() { + return this.executor instanceof EngineExecutor; + } + + public boolean isExecutorService() { + return this.executor.getOriginal() instanceof ExecutorService; + } + + /** + * whether is ThreadPoolExecutor + * + * @return boolean + */ + public boolean isThreadPoolExecutor() { + return this.executor instanceof ThreadPoolExecutorAdapter; + } + + /** + * set taskWrappers + * + * @param taskWrappers taskWrappers + */ + public void setTaskWrappers(List taskWrappers) { + if (executor.getOriginal() instanceof TaskEnhanceAware) { + ((TaskEnhanceAware) executor.getOriginal()).setTaskWrappers(taskWrappers); + } + } + + public void setRejectHandler(RejectedExecutionHandler handler) { + String rejectHandlerType = handler.getClass().getSimpleName(); + if (executor.getOriginal() instanceof RejectHandlerAware) { + ((RejectHandlerAware) executor.getOriginal()).setRejectHandlerType(rejectHandlerType); + } + if (isRejectEnhanced()) { + executor.setRejectedExecutionHandler(RejectHandlerGetter.getProxy(handler)); + } else { + executor.setRejectedExecutionHandler(handler); + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/NamedRunnable.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/NamedRunnable.java new file mode 100644 index 0000000000..1a2c949af8 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/NamedRunnable.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics; + +import org.apache.commons.lang3.StringUtils; + +import java.util.UUID; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class NamedRunnable implements Runnable { + + private final Runnable runnable; + + private final String name; + + public NamedRunnable(Runnable runnable, String name) { + this.runnable = runnable; + this.name = name; + } + + @Override + public void run() { + this.runnable.run(); + } + + public String getName() { + return name; + } + + public static NamedRunnable of(Runnable runnable, String name) { + if (StringUtils.isBlank(name)) { + name = runnable.getClass().getSimpleName() + "-" + UUID.randomUUID(); + } + return new NamedRunnable(runnable, name); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/PerformanceProvider.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/PerformanceProvider.java new file mode 100644 index 0000000000..b0be0c1683 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/PerformanceProvider.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics; + +import com.dtstack.taier.metrics.metrice.MMAPCounter; +import lombok.Getter; +import lombok.val; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.concurrent.atomic.AtomicLong; + +/** + * PerformanceProvider related + * + * @author kyao + * @since 1.1.5 + */ +public class PerformanceProvider { + + /** + * last refresh timestamp + */ + private final AtomicLong lastRefreshMillis = new AtomicLong(System.currentTimeMillis()); + + private final MMAPCounter mmapCounter = new MMAPCounter(); + + public void completeTask(long rt) { + mmapCounter.add(rt); + } + + public PerformanceSnapshot getSnapshotAndReset() { + long currentMillis = System.currentTimeMillis(); + int intervalTs = (int) (currentMillis - lastRefreshMillis.get()) / 1000; + PerformanceSnapshot performanceSnapshot = new PerformanceSnapshot(mmapCounter, intervalTs); + reset(currentMillis); + return performanceSnapshot; + } + + private void reset(long currentMillis) { + mmapCounter.reset(); + lastRefreshMillis.compareAndSet(lastRefreshMillis.get(), currentMillis); + } + + @Getter + public static class PerformanceSnapshot { + + private final double tps; + + private final long maxRt; + + private final long minRt; + + private final double avg; + + private final double tp50; + + private final double tp75; + + private final double tp90; + + private final double tp95; + + private final double tp99; + + private final double tp999; + + public PerformanceSnapshot(MMAPCounter mmapCounter, int monitorInterval) { + tps = BigDecimal.valueOf(mmapCounter.getMmaCounter().getCount()) + .divide(BigDecimal.valueOf(Math.max(monitorInterval, 1)), 1, RoundingMode.HALF_UP) + .doubleValue(); + + maxRt = mmapCounter.getMmaCounter().getMax(); + minRt = mmapCounter.getMmaCounter().getMin(); + avg = mmapCounter.getMmaCounter().getAvg(); + + tp50 = mmapCounter.getSnapshot().getMedian(); + tp75 = mmapCounter.getSnapshot().get75thPercentile(); + tp90 = mmapCounter.getSnapshot().getValue(0.9); + tp95 = mmapCounter.getSnapshot().get95thPercentile(); + tp99 = mmapCounter.getSnapshot().get99thPercentile(); + tp999 = mmapCounter.getSnapshot().get999thPercentile(); + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/Summary.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/Summary.java new file mode 100644 index 0000000000..5734de419a --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/Summary.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics; + +import com.dtstack.taier.metrics.metrice.Meter; + +/** + * Summary related + * + * @author yanhom + * @since 1.1.5 + */ +public interface Summary extends Meter { + + /** + * Add value. + * + * @param value current value + */ + void add(long value); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/ThreadPoolStatProvider.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/ThreadPoolStatProvider.java new file mode 100644 index 0000000000..1d43e8a187 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/ThreadPoolStatProvider.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics; + +import com.dtstack.taier.metrics.executor.EngineExecutor; +import com.dtstack.taier.metrics.timer.HashedWheelTimer; +import com.dtstack.taier.metrics.timer.HashedWheelTimerFactory; +import com.dtstack.taier.metrics.timer.QueueTimeoutTimerTask; +import com.dtstack.taier.metrics.timer.RunTimeoutTimerTask; +import com.dtstack.taier.metrics.timer.Timeout; + +import java.lang.ref.SoftReference; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class ThreadPoolStatProvider { + + private final ExecutorWrapper executorWrapper; + + /** + * Task execute timeout, unit (ms), just for statistics. + */ + private long runTimeout = 0; + + /** + * Try interrupt task when timeout. + */ + private boolean tryInterrupt = false; + + /** + * Task queue wait timeout, unit (ms), just for statistics. + */ + private long queueTimeout = 0; + + /** + * Total reject count. + */ + private final LongAdder rejectCount = new LongAdder(); + + /** + * Count run timeout tasks. + */ + private final LongAdder runTimeoutCount = new LongAdder(); + + /** + * Count queue wait timeout tasks. + */ + private final LongAdder queueTimeoutCount = new LongAdder(); + + /** + * runTimeoutMap key -> Runnable value -> Timeout + */ + private final Map> runTimeoutMap = new ConcurrentHashMap<>(); + + /** + * queueTimeoutMap key -> Runnable value -> Timeout + */ + private final Map> queueTimeoutMap = new ConcurrentHashMap<>(); + + /** + * stopWatchMap key -> Runnable value -> millis + */ + private final Map stopWatchMap = new ConcurrentHashMap<>(); + + /** + * performance provider + */ + private final PerformanceProvider performanceProvider = new PerformanceProvider(); + + private ThreadPoolStatProvider(ExecutorWrapper executorWrapper) { + this.executorWrapper = executorWrapper; + } + + public static ThreadPoolStatProvider of(ExecutorWrapper executorWrapper) { + ThreadPoolStatProvider provider = new ThreadPoolStatProvider(executorWrapper); + if (executorWrapper.isDtpExecutor()) { + EngineExecutor dtpExecutor = (EngineExecutor) executorWrapper.getExecutor(); + provider.setRunTimeout(dtpExecutor.getRunTimeout()); + provider.setQueueTimeout(dtpExecutor.getQueueTimeout()); + provider.setTryInterrupt(dtpExecutor.isTryInterrupt()); + } + return provider; + } + + public ExecutorWrapper getExecutorWrapper() { + return executorWrapper; + } + + public long getRunTimeout() { + return runTimeout; + } + + public void setRunTimeout(long runTimeout) { + this.runTimeout = runTimeout; + } + + public boolean isTryInterrupt() { + return tryInterrupt; + } + + public void setTryInterrupt(boolean tryInterrupt) { + this.tryInterrupt = tryInterrupt; + } + + public long getQueueTimeout() { + return queueTimeout; + } + + public void setQueueTimeout(long queueTimeout) { + this.queueTimeout = queueTimeout; + } + + public long getRejectedTaskCount() { + return rejectCount.sum(); + } + + public void incRejectCount(int count) { + rejectCount.add(count); + } + + public long getRunTimeoutCount() { + return runTimeoutCount.sum(); + } + + public void incRunTimeoutCount(int count) { + runTimeoutCount.add(count); + } + + public long getQueueTimeoutCount() { + return queueTimeoutCount.sum(); + } + + public void incQueueTimeoutCount(int count) { + queueTimeoutCount.add(count); + } + + public void startQueueTimeoutTask(Runnable r) { + if (queueTimeout <= 0) { + return; + } + HashedWheelTimer timer = HashedWheelTimerFactory.holderHashedWheelTimer(); + QueueTimeoutTimerTask timerTask = new QueueTimeoutTimerTask(executorWrapper, r); + queueTimeoutMap.put(r, new SoftReference<>(timer.newTimeout(timerTask, queueTimeout, TimeUnit.MILLISECONDS))); + } + + public void cancelQueueTimeoutTask(Runnable r) { + Optional.ofNullable(queueTimeoutMap.remove(r)) + .map(SoftReference::get) + .ifPresent(Timeout::cancel); + } + + public void startRunTimeoutTask(Thread t, Runnable r) { + if (runTimeout <= 0) { + return; + } + HashedWheelTimer timer = HashedWheelTimerFactory.holderHashedWheelTimer(); + RunTimeoutTimerTask timerTask = new RunTimeoutTimerTask(executorWrapper, r, t); + runTimeoutMap.put(r, new SoftReference<>(timer.newTimeout(timerTask, runTimeout, TimeUnit.MILLISECONDS))); + } + + public void cancelRunTimeoutTask(Runnable r) { + Optional.ofNullable(runTimeoutMap.remove(r)) + .map(SoftReference::get) + .ifPresent(Timeout::cancel); + } + + public void startTask(Runnable r) { + stopWatchMap.put(r, System.currentTimeMillis()); + } + + public void completeTask(Runnable r) { + Optional.ofNullable(stopWatchMap.remove(r)) + .ifPresent(millis -> { + long rt = System.currentTimeMillis() - millis; + performanceProvider.completeTask(rt); + }); + } + + public PerformanceProvider getPerformanceProvider() { + return this.performanceProvider; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/adapter/ExecutorAdapter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/adapter/ExecutorAdapter.java new file mode 100644 index 0000000000..a08c1a3e7e --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/adapter/ExecutorAdapter.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.adapter; + +import java.util.AbstractQueue; +import java.util.Collection; +import java.util.Iterator; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.TimeUnit; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public interface ExecutorAdapter extends Executor { + + /** + * Get the original executor + * + * @return the original executor + */ + E getOriginal(); + + /** + * Execute the task + * + * @param command the runnable task + */ + @Override + default void execute(Runnable command) { + getOriginal().execute(command); + } + + /** + * Get the core pool size + * + * @return the core pool size + */ + int getCorePoolSize(); + + /** + * Set the core pool size + * + * @param corePoolSize the core pool size + */ + void setCorePoolSize(int corePoolSize); + + /** + * Get the maximum pool size + * + * @return the maximum pool size + */ + int getMaximumPoolSize(); + + /** + * Set the maximum pool size + * + * @param maximumPoolSize the maximum pool size + */ + void setMaximumPoolSize(int maximumPoolSize); + + /** + * Get the pool size + * + * @return the pool size + */ + int getPoolSize(); + + /** + * Get the active count + * + * @return the active count + */ + int getActiveCount(); + + /** + * Get the largest pool size + * + * @return the largest pool size + */ + default int getLargestPoolSize() { + // default unsupported + return -1; + } + + /** + * Get the task count + * + * @return the task count + */ + default long getTaskCount() { + // default unsupported + return -1; + } + + /** + * Get the completed task count + * + * @return the completed task count + */ + default long getCompletedTaskCount() { + // default unsupported + return -1; + } + + /** + * Get the queue + * + * @return the queue + */ + default BlockingQueue getQueue() { + return new UnsupportedBlockingQueue(); + } + + /** + * Get the queue type + * + * @return the queue type + */ + default String getQueueType() { + return getQueue().getClass().getSimpleName(); + } + + /** + * Get the queue size + * + * @return the queue size + */ + default int getQueueSize() { + return getQueue().size(); + } + + /** + * Get the queue remaining capacity + * + * @return the queue remaining capacity + */ + default int getQueueRemainingCapacity() { + return getQueue().remainingCapacity(); + } + + /** + * Get the queue capacity + * + * @return the queue capacity + */ + default int getQueueCapacity() { + int capacity = getQueueSize() + getQueueRemainingCapacity(); + return capacity < 0 ? Integer.MAX_VALUE : capacity; + } + + /** + * On refresh queue capacity. + * + * @param capacity the queue capacity + */ + default void onRefreshQueueCapacity(int capacity) { + // default do nothing + } + + /** + * Get the rejected execution handler + * + * @return the rejected execution handler + */ + default RejectedExecutionHandler getRejectedExecutionHandler() { + // default unsupported + return null; + } + + /** + * Set the rejected execution handler + * + * @param handler the rejected execution handler + */ + default void setRejectedExecutionHandler(RejectedExecutionHandler handler) { + // default unsupported + } + + /** + * Get the reject handler type + * + * @return the reject handler type + */ + default String getRejectHandlerType() { + return Optional.ofNullable(getRejectedExecutionHandler()) + .map(h -> h.getClass().getSimpleName()) + .orElse("unknown"); + } + + /** + * If allow core thread time out + * + * @return if allow core thread time out + */ + default boolean allowsCoreThreadTimeOut() { + // default unsupported + return false; + } + + /** + * Allow core thread time out + * + * @param value if allow core thread time out + */ + default void allowCoreThreadTimeOut(boolean value) { + // default unsupported + } + + /** + * Pre start all core threads + */ + default void preStartAllCoreThreads() { + // default unsupported + } + + /** + * Get the keep alive time + * + * @param unit the time unit + * @return the keep alive time + */ + default long getKeepAliveTime(TimeUnit unit) { + // default unsupported + return -1; + } + + /** + * Set the keep alive time + * + * @param time the keep alive time + * @param unit the time unit + */ + default void setKeepAliveTime(long time, TimeUnit unit) { + // default unsupported + } + + /** + * is shutdown + * @return boolean + */ + default boolean isShutdown() { + // default unsupported + return false; + } + + /** + * is terminated + * @return boolean + */ + default boolean isTerminated() { + // default unsupported + return false; + } + + /** + * is terminating + * @return boolean + */ + default boolean isTerminating() { + // default unsupported + return false; + } + + class UnsupportedBlockingQueue extends AbstractQueue implements BlockingQueue { + + @Override + public Iterator iterator() { + throw new UnsupportedOperationException(); + } + + @Override + public int size() { + return 0; + } + + @Override + public void put(Runnable runnable) throws InterruptedException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean offer(Runnable runnable, long timeout, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + + @Override + public Runnable take() { + throw new UnsupportedOperationException(); + } + + @Override + public Runnable poll(long timeout, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + + @Override + public int remainingCapacity() { + return 0; + } + + @Override + public int drainTo(Collection c) { + throw new UnsupportedOperationException(); + } + + @Override + public int drainTo(Collection c, int maxElements) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean offer(Runnable runnable) { + throw new UnsupportedOperationException(); + } + + @Override + public Runnable poll() { + throw new UnsupportedOperationException(); + } + + @Override + public Runnable peek() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/adapter/ThreadPoolExecutorAdapter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/adapter/ThreadPoolExecutorAdapter.java new file mode 100644 index 0000000000..0b4af54a5c --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/adapter/ThreadPoolExecutorAdapter.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.adapter; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class ThreadPoolExecutorAdapter implements ExecutorAdapter { + + private final ThreadPoolExecutor executor; + + public ThreadPoolExecutorAdapter(ThreadPoolExecutor executor) { + this.executor = executor; + } + + @Override + public ThreadPoolExecutor getOriginal() { + return this.executor; + } + + @Override + public void execute(Runnable command) { + this.executor.execute(command); + } + + @Override + public int getCorePoolSize() { + return this.executor.getCorePoolSize(); + } + + @Override + public void setCorePoolSize(int corePoolSize) { + this.executor.setCorePoolSize(corePoolSize); + } + + @Override + public int getMaximumPoolSize() { + return this.executor.getMaximumPoolSize(); + } + + @Override + public void setMaximumPoolSize(int maximumPoolSize) { + this.executor.setMaximumPoolSize(maximumPoolSize); + } + + @Override + public int getPoolSize() { + return this.executor.getPoolSize(); + } + + @Override + public int getActiveCount() { + return this.executor.getActiveCount(); + } + + @Override + public int getLargestPoolSize() { + return this.executor.getLargestPoolSize(); + } + + @Override + public long getTaskCount() { + return this.executor.getTaskCount(); + } + + @Override + public long getCompletedTaskCount() { + return this.executor.getCompletedTaskCount(); + } + + @Override + public BlockingQueue getQueue() { + return this.executor.getQueue(); + } + + @Override + public RejectedExecutionHandler getRejectedExecutionHandler() { + return this.executor.getRejectedExecutionHandler(); + } + + @Override + public String getRejectHandlerType() { + return getRejectedExecutionHandler().getClass().getSimpleName(); + } + + @Override + public void setRejectedExecutionHandler(RejectedExecutionHandler handler) { + this.executor.setRejectedExecutionHandler(handler); + } + + @Override + public boolean allowsCoreThreadTimeOut() { + return this.executor.allowsCoreThreadTimeOut(); + } + + @Override + public void allowCoreThreadTimeOut(boolean value) { + this.executor.allowCoreThreadTimeOut(value); + } + + @Override + public void preStartAllCoreThreads() { + this.executor.prestartAllCoreThreads(); + } + + @Override + public long getKeepAliveTime(TimeUnit unit) { + return this.executor.getKeepAliveTime(unit); + } + + @Override + public void setKeepAliveTime(long time, TimeUnit unit) { + this.executor.setKeepAliveTime(time, unit); + } + + @Override + public boolean isShutdown() { + return this.executor.isShutdown(); + } + + @Override + public boolean isTerminated() { + return this.executor.isTerminated(); + } + + @Override + public boolean isTerminating() { + return this.executor.isTerminating(); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/Aware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/Aware.java new file mode 100644 index 0000000000..928cbad9dd --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/Aware.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class Aware { +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/AwareManager.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/AwareManager.java new file mode 100644 index 0000000000..0b6b9df69e --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/AwareManager.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.collect.util.ExtensionServiceLoader; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.Executor; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +public class AwareManager { + + private static final List EXECUTOR_AWARE_LIST = new ArrayList<>(); + + private AwareManager() { + } + + static { + EXECUTOR_AWARE_LIST.add(new PerformanceMonitorAware()); + EXECUTOR_AWARE_LIST.add(new TaskTimeoutAware()); + EXECUTOR_AWARE_LIST.add(new TaskRejectAware()); + + List serviceLoader = ExtensionServiceLoader.get(ExecutorAware.class); + EXECUTOR_AWARE_LIST.addAll(serviceLoader); + EXECUTOR_AWARE_LIST.sort(Comparator.comparingInt(ExecutorAware::getOrder)); + } + + public static void add(ExecutorAware aware) { + for (ExecutorAware executorAware : EXECUTOR_AWARE_LIST) { + if (executorAware.getName().equalsIgnoreCase(aware.getName())) { + return; + } + } + EXECUTOR_AWARE_LIST.add(aware); + EXECUTOR_AWARE_LIST.sort(Comparator.comparingInt(ExecutorAware::getOrder)); + } + + public static void register(ExecutorWrapper executorWrapper) { + for (ExecutorAware executorAware : EXECUTOR_AWARE_LIST) { + val awareNames = executorWrapper.getAwareNames(); + // if awareNames is empty, register all + if (CollectionUtils.isEmpty(awareNames) || awareNames.contains(executorAware.getName())) { + executorAware.register(executorWrapper); + } + } + } + + public static void execute(Executor executor, Runnable r) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + aware.execute(executor, r); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance execute error.", aware.getName(), e); + } + } + } + + public static void beforeExecute(Executor executor, Thread t, Runnable r) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + r = aware.beforeExecuteWrap(executor, t, r); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance beforeExecute error.", aware.getName(), e); + } + } + } + + public static void afterExecute(Executor executor, Runnable r, Throwable t) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + r = aware.afterExecuteWrap(executor, r, t); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance afterExecute error.", aware.getName(), e); + } + } + } + + public static void shutdown(Executor executor) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + aware.shutdown(executor); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance shutdown error.", aware.getName(), e); + } + } + } + + public static void shutdownNow(Executor executor, List tasks) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + aware.shutdownNow(executor, tasks); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance shutdownNow error.", aware.getName(), e); + } + } + } + + public static void terminated(Executor executor) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + aware.terminated(executor); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance terminated error.", aware.getName(), e); + } + } + } + + public static void beforeReject(Runnable r, Executor executor) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + r = aware.beforeRejectWrap(r, executor); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance beforeReject error.", aware.getName(), e); + } + } + } + + public static void afterReject(Runnable r, Executor executor) { + for (ExecutorAware aware : EXECUTOR_AWARE_LIST) { + try { + r = aware.afterRejectWrap(r, executor); + } catch (Exception e) { + log.error("DynamicTp aware [{}], enhance afterReject error.", aware.getName(), e); + } + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/AwareTypeEnum.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/AwareTypeEnum.java new file mode 100644 index 0000000000..a181ca949a --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/AwareTypeEnum.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * AwareType Enum + * + * @author kyao + * @since 1.1.4 + */ +@Getter +@AllArgsConstructor +public enum AwareTypeEnum { + + /** + * PerformanceMonitorAware + */ + PERFORMANCE_MONITOR_AWARE(1, "monitor"), + + /** + * TaskTimeoutAware + */ + TASK_TIMEOUT_AWARE(2, "timeout"), + + /** + * TaskRejectAware + */ + TASK_REJECT_AWARE(3, "reject"); + + private final int order; + + private final String name; + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/ExecutorAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/ExecutorAware.java new file mode 100644 index 0000000000..c0f9e6d83c --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/ExecutorAware.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import com.dtstack.taier.metrics.ExecutorWrapper; + +import java.util.List; +import java.util.concurrent.Executor; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public interface ExecutorAware { + + /** + * aware order + * + * @return order + */ + int getOrder(); + + /** + * aware name + * + * @return name + */ + String getName(); + + /** + * register Executor + * + * @param wrapper executor wrapper + */ + default void register(ExecutorWrapper wrapper) { + } + + /** + * remove Executor + * + * @param wrapper executor wrapper + */ + default void remove(ExecutorWrapper wrapper) { + } + + /** + * enhance execute + * + * @param executor executor + * @param r runnable + */ + default void execute(Executor executor, Runnable r) { + // default no Operation + } + + /** + * enhance beforeExecute + * + * @param executor executor + * @param t thread + * @param r runnable + */ + default void beforeExecute(Executor executor, Thread t, Runnable r) { + // default no Operation + } + + default Runnable beforeExecuteWrap(Executor executor, Thread t, Runnable r) { + beforeExecute(executor, t, r); + return r; + } + + /** + * enhance afterExecute + * + * @param executor executor + * @param r runnable + * @param t throwable + */ + default void afterExecute(Executor executor, Runnable r, Throwable t) { + // default no Operation + } + + default Runnable afterExecuteWrap(Executor executor, Runnable r, Throwable t) { + afterExecute(executor, r, t); + return r; + } + + /** + * enhance shutdown + * + * @param executor executor + */ + default void shutdown(Executor executor) { + // default no Operation + } + + /** + * enhance shutdownNow + * + * @param executor executor + * @param tasks tasks + */ + default void shutdownNow(Executor executor, List tasks) { + // default no Operation + } + + /** + * enhance terminated + * + * @param executor executor + */ + default void terminated(Executor executor) { + // default no Operation + } + + /** + * enhance before reject + * @param r runnable + * @param executor executor + */ + default void beforeReject(Runnable r, Executor executor) { + // default no Operation + } + + default Runnable beforeRejectWrap(Runnable r, Executor executor) { + beforeReject(r, executor); + return r; + } + + /** + * enhance after reject + * @param r runnable + * @param executor executor + */ + default void afterReject(Runnable r, Executor executor) { + // default no Operation + } + + default Runnable afterRejectWrap(Runnable r, Executor executor) { + afterReject(r, executor); + return r; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/PerformanceMonitorAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/PerformanceMonitorAware.java new file mode 100644 index 0000000000..f2d6293e12 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/PerformanceMonitorAware.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * PerformanceMonitorAware related + * + * @author kyao + * @since 1.1.5 + */ +public class PerformanceMonitorAware extends TaskStatAware { + + @Override + public int getOrder() { + return AwareTypeEnum.PERFORMANCE_MONITOR_AWARE.getOrder(); + } + + @Override + public String getName() { + return AwareTypeEnum.PERFORMANCE_MONITOR_AWARE.getName(); + } + + @Override + public void execute(Executor executor, Runnable r) { + if ("true".equals(System.getProperty("dtp.execute.enhanced", "true"))) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> p.startTask(r)); + } + } + + @Override + public void afterExecute(Executor executor, Runnable r, Throwable t) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> p.completeTask(r)); + } + + @Override + public void afterReject(Runnable r, Executor executor) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> p.completeTask(r)); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/RejectHandlerAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/RejectHandlerAware.java new file mode 100644 index 0000000000..e625e457a3 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/RejectHandlerAware.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +/** + * RejectHandlerAware related + **/ +public interface RejectHandlerAware { + + /** + * Get reject handler type. + * + * @return reject handler type + */ + String getRejectHandlerType(); + + /** + * Set reject handler type. + * + * @param rejectHandlerType reject handler type + */ + default void setRejectHandlerType(String rejectHandlerType) { + + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskEnhanceAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskEnhanceAware.java new file mode 100644 index 0000000000..2fcc9c34a9 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskEnhanceAware.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import com.dtstack.taier.metrics.NamedRunnable; +import com.dtstack.taier.metrics.runnable.DtpRunnable; +import com.dtstack.taier.metrics.wrapper.TaskWrapper; +import org.apache.commons.collections.CollectionUtils; + +import java.util.List; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public interface TaskEnhanceAware { + + /** + * Enhance task + * + * @param command command + * @param taskWrappers task wrappers + * @return enhanced task + */ + default Runnable getEnhancedTask(Runnable command, List taskWrappers) { + Runnable wrapRunnable = command; + String taskName = (wrapRunnable instanceof NamedRunnable) ? ((NamedRunnable) wrapRunnable).getName() : null; + if (CollectionUtils.isNotEmpty(taskWrappers)) { + for (TaskWrapper t : taskWrappers) { + wrapRunnable = t.wrap(wrapRunnable); + } + } + return new DtpRunnable(command, wrapRunnable, taskName); + } + + /** + * Enhance task + * + * @param command command + * @return enhanced task + */ + default Runnable getEnhancedTask(Runnable command) { + return getEnhancedTask(command, getTaskWrappers()); + } + + /** + * Get task wrappers + * + * @return task wrappers + */ + List getTaskWrappers(); + + /** + * Set task wrappers + * + * @param taskWrappers task wrappers + */ + void setTaskWrappers(List taskWrappers); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskRejectAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskRejectAware.java new file mode 100644 index 0000000000..b7fdf9e23f --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskRejectAware.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import com.dtstack.taier.metrics.ThreadPoolStatProvider; +import com.dtstack.taier.metrics.adapter.ExecutorAdapter; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.MDC; + +import java.util.Objects; +import java.util.concurrent.Executor; + +/** + * TaskRejectAware related + */ +@Slf4j +public class TaskRejectAware extends TaskStatAware { + + @Override + public int getOrder() { + return AwareTypeEnum.TASK_REJECT_AWARE.getOrder(); + } + + @Override + public String getName() { + return AwareTypeEnum.TASK_REJECT_AWARE.getName(); + } + + @Override + public void beforeReject(Runnable runnable, Executor executor) { + ThreadPoolStatProvider statProvider = statProviders.get(executor); + if (Objects.isNull(statProvider)) { + return; + } + + statProvider.incRejectCount(1); + ExecutorAdapter executorAdapter = statProvider.getExecutorWrapper().getExecutor(); + String logMsg = String.format("DynamicTp execute, thread pool is exhausted, tpName: %s, traceId: %s, " + + "poolSize: %s (active: %s, core: %s, max: %s, largest: %s), " + + "task: %s (completed: %s), queueCapacity: %s (currSize: %s, remaining: %s) ," + + "executorStatus: (isShutdown: %s, isTerminated: %s, isTerminating: %s)", + statProvider.getExecutorWrapper().getThreadPoolName(), MDC.get("traceId"), + executorAdapter.getPoolSize(), + executorAdapter.getActiveCount(), executorAdapter.getCorePoolSize(), + executorAdapter.getMaximumPoolSize(), + executorAdapter.getLargestPoolSize(), executorAdapter.getTaskCount(), + executorAdapter.getCompletedTaskCount(), + statProvider.getExecutorWrapper().getExecutor().getQueueCapacity(), executorAdapter.getQueue().size(), + executorAdapter.getQueue().remainingCapacity(), + executorAdapter.isShutdown(), executorAdapter.isTerminated(), executorAdapter.isTerminating()); + // log.warn(logMsg); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskStatAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskStatAware.java new file mode 100644 index 0000000000..0da555de95 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskStatAware.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.ThreadPoolStatProvider; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public abstract class TaskStatAware implements ExecutorAware { + + protected final Map statProviders = new ConcurrentHashMap<>(); + + @Override + public void register(ExecutorWrapper wrapper) { + ThreadPoolStatProvider statProvider = wrapper.getThreadPoolStatProvider(); + statProviders.put(wrapper.getExecutor(), statProvider); + statProviders.put(wrapper.getExecutor().getOriginal(), statProvider); + } + + @Override + public void remove(ExecutorWrapper wrapper) { + statProviders.remove(wrapper.getExecutor()); + statProviders.remove(wrapper.getExecutor().getOriginal()); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskTimeoutAware.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskTimeoutAware.java new file mode 100644 index 0000000000..1fba287ca5 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/aware/TaskTimeoutAware.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.aware; + +import lombok.extern.slf4j.Slf4j; + +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * TaskTimeoutAware related + * + * @author kyao + * @since 1.1.4 + */ +@Slf4j +public class TaskTimeoutAware extends TaskStatAware { + + @Override + public int getOrder() { + return AwareTypeEnum.TASK_TIMEOUT_AWARE.getOrder(); + } + + @Override + public String getName() { + return AwareTypeEnum.TASK_TIMEOUT_AWARE.getName(); + } + + @Override + public void execute(Executor executor, Runnable r) { + if ("true".equals(System.getProperty("dtp.execute.enhanced", "true"))) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> p.startQueueTimeoutTask(r)); + } + } + + @Override + public void beforeExecute(Executor executor, Thread t, Runnable r) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> { + p.cancelQueueTimeoutTask(r); + p.startRunTimeoutTask(t, r); + }); + } + + @Override + public void afterExecute(Executor executor, Runnable r, Throwable t) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> p.cancelRunTimeoutTask(r)); + } + + @Override + public void beforeReject(Runnable r, Executor executor) { + Optional.ofNullable(statProviders.get(executor)).ifPresent(p -> p.cancelQueueTimeoutTask(r)); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/builder/ThreadPoolBuilder.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/builder/ThreadPoolBuilder.java new file mode 100644 index 0000000000..9e82f0438e --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/builder/ThreadPoolBuilder.java @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.builder; + +import com.dtstack.taier.metrics.collect.em.QueueTypeEnum; +import com.dtstack.taier.metrics.executor.EagerEngineExecutor; +import com.dtstack.taier.metrics.executor.EngineExecutor; +import com.dtstack.taier.metrics.queue.TaskQueue; +import com.dtstack.taier.metrics.queue.VariableLinkedBlockingQueue; +import com.dtstack.taier.metrics.rejects.RejectHandlerGetter; +import com.dtstack.taier.metrics.wrapper.TaskWrapper; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import io.micrometer.core.instrument.util.NamedThreadFactory; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class ThreadPoolBuilder { + + /** + * Name of Dynamic ThreadPool. + */ + private String threadPoolName = "ThreadPool"; + + /** + * CoreSize of ThreadPool. + */ + private int corePoolSize = 1; + + /** + * MaxSize of ThreadPool. + */ + private int maximumPoolSize = Runtime.getRuntime().availableProcessors(); + + /** + * When the number of threads is greater than the core, + * this is the maximum time that excess idle threads + * will wait for new tasks before terminating + */ + private long keepAliveTime = 60; + + /** + * Timeout unit. + */ + private TimeUnit timeUnit = TimeUnit.SECONDS; + + /** + * Blocking queue, see {@link QueueTypeEnum} + */ + private BlockingQueue workQueue = new VariableLinkedBlockingQueue<>(1024); + + /** + * Queue capacity + */ + private int queueCapacity = 1024; + + /** + * Max free memory for MemorySafeLBQ, unit M + */ + private int maxFreeMemory = 16; + + /** + * RejectedExecutionHandler + */ + private RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.AbortPolicy(); + + /** + * Default inner thread factory. + */ + private ThreadFactory threadFactory = new NamedThreadFactory("dtp"); + + /** + * If allow core thread timeout. + */ + private boolean allowCoreThreadTimeOut = false; + + /** + * Dynamic or common. + */ + private boolean dynamic = true; + + /** + * Whether to wait for scheduled tasks to complete on shutdown, + * not interrupting running tasks and executing all tasks in the queue. + */ + private boolean waitForTasksToCompleteOnShutdown = true; + + /** + * The maximum number of seconds that this executor is supposed to block + * on shutdown in order to wait for remaining tasks to complete their execution + * before the rest of the container continues to shut down. + */ + private int awaitTerminationSeconds = 3; + + /** + * If io intensive thread pool. + * default false, true indicate cpu intensive thread pool. + */ + private boolean eager = false; + + /** + * If ordered thread pool. + * default false, true ordered thread pool. + */ + private boolean ordered = false; + + /** + * If scheduled executor, default false. + */ + private boolean scheduled = false; + + /** + * If priority thread pool. + * default false, true priority thread pool. + */ + private boolean priority = false; + + /** + * If pre start all core threads. + */ + private boolean preStartAllCoreThreads = false; + + /** + * If enhance reject. + */ + private boolean rejectEnhanced = true; + + /** + * If enable notify. + */ + private boolean notifyEnabled = true; + + /** + * Task execute timeout, unit (ms). + */ + private long runTimeout = 0; + + /** + * If try interrupt thread when task run timeout. + */ + private boolean tryInterrupt = false; + + /** + * Task queue wait timeout, unit (ms), just for statistics. + */ + private long queueTimeout = 0; + + /** + * Task wrappers. + */ + private final List taskWrappers = Lists.newArrayList(); + + /** + * Notify platform id + */ + private List platformIds = Lists.newArrayList(); + + private ThreadPoolBuilder() { + } + + public static ThreadPoolBuilder newBuilder() { + return new ThreadPoolBuilder(); + } + + public ThreadPoolBuilder threadPoolName(String poolName) { + this.threadPoolName = poolName; + return this; + } + + public ThreadPoolBuilder corePoolSize(int corePoolSize) { + if (corePoolSize >= 0) { + this.corePoolSize = corePoolSize; + } + return this; + } + + public ThreadPoolBuilder maximumPoolSize(int maximumPoolSize) { + if (maximumPoolSize > 0) { + this.maximumPoolSize = maximumPoolSize; + } + return this; + } + + public ThreadPoolBuilder keepAliveTime(long keepAliveTime) { + if (keepAliveTime > 0) { + this.keepAliveTime = keepAliveTime; + } + return this; + } + + public ThreadPoolBuilder timeUnit(TimeUnit timeUnit) { + if (timeUnit != null) { + this.timeUnit = timeUnit; + } + return this; + } + + /** + * Create work queue + * + * @param queueName queue name + * @param capacity queue capacity + * @param fair for SynchronousQueue + * @param maxFreeMemory for MemorySafeLBQ + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder workQueue(String queueName, Integer capacity, Boolean fair, Integer maxFreeMemory) { + if (StringUtils.isNotBlank(queueName)) { + workQueue = QueueTypeEnum.buildLbq(queueName, capacity != null ? capacity : this.queueCapacity, + fair != null && fair, maxFreeMemory != null ? maxFreeMemory : this.maxFreeMemory); + } + return this; + } + + /** + * Create work queue + * + * @param queueName queue name + * @param capacity queue capacity + * @param fair for SynchronousQueue + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder workQueue(String queueName, Integer capacity, Boolean fair) { + if (StringUtils.isNotBlank(queueName)) { + workQueue = QueueTypeEnum.buildLbq(queueName, capacity != null ? capacity : this.queueCapacity, + fair != null && fair, maxFreeMemory); + } + return this; + } + + public ThreadPoolBuilder workQueue(String queueName, Integer capacity) { + if (StringUtils.isNotBlank(queueName)) { + workQueue = QueueTypeEnum.buildLbq(queueName, capacity != null ? capacity : this.queueCapacity, + false, maxFreeMemory); + } + return this; + } + + public ThreadPoolBuilder queueCapacity(int queueCapacity) { + this.queueCapacity = queueCapacity; + return this; + } + + public ThreadPoolBuilder maxFreeMemory(int maxFreeMemory) { + this.maxFreeMemory = maxFreeMemory; + return this; + } + + public ThreadPoolBuilder rejectedExecutionHandler(String rejectedName) { + if (StringUtils.isNotBlank(rejectedName)) { + rejectedExecutionHandler = RejectHandlerGetter.buildRejectedHandler(rejectedName); + } + return this; + } + + public ThreadPoolBuilder rejectedExecutionHandler(RejectedExecutionHandler handler) { + if (Objects.nonNull(handler)) { + rejectedExecutionHandler = handler; + } + return this; + } + + public ThreadPoolBuilder threadFactory(String prefix) { + if (StringUtils.isNotBlank(prefix)) { + threadFactory = new NamedThreadFactory(prefix); + } + return this; + } + + public ThreadPoolBuilder allowCoreThreadTimeOut(boolean allowCoreThreadTimeOut) { + this.allowCoreThreadTimeOut = allowCoreThreadTimeOut; + return this; + } + + public ThreadPoolBuilder dynamic(boolean dynamic) { + this.dynamic = dynamic; + return this; + } + + public ThreadPoolBuilder awaitTerminationSeconds(int awaitTerminationSeconds) { + this.awaitTerminationSeconds = awaitTerminationSeconds; + return this; + } + + public ThreadPoolBuilder waitForTasksToCompleteOnShutdown(boolean waitForTasksToCompleteOnShutdown) { + this.waitForTasksToCompleteOnShutdown = waitForTasksToCompleteOnShutdown; + return this; + } + + /** + * @param eager true or false + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder eager(boolean eager) { + checkExecutorType(); + this.eager = eager; + return this; + } + + /** + * @param ordered true or false + * @return the ThreadPoolBuilder instance + * @deprecated use {@link #ordered()} instead + */ + @Deprecated + public ThreadPoolBuilder ordered(boolean ordered) { + checkExecutorType(); + this.ordered = ordered; + return this; + } + + /** + * @param scheduled true or false + * @return the ThreadPoolBuilder instance + * @deprecated use {@link #scheduled()} instead + */ + @Deprecated + public ThreadPoolBuilder scheduled(boolean scheduled) { + checkExecutorType(); + this.scheduled = scheduled; + return this; + } + + /** + * @param priority true or false + * @return the ThreadPoolBuilder instance + * @deprecated use {@link #priority()} instead + */ + @Deprecated + public ThreadPoolBuilder priority(boolean priority) { + checkExecutorType(); + this.priority = priority; + return this; + } + + /** + * set eager type + * + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder eager() { + checkExecutorType(); + this.eager = true; + return this; + } + + /** + * set ordered type + * + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder ordered() { + checkExecutorType(); + this.ordered = true; + return this; + } + + /** + * set scheduled type + * + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder scheduled() { + checkExecutorType(); + this.scheduled = true; + return this; + } + + /** + * set priority type + * + * @return the ThreadPoolBuilder instance + */ + public ThreadPoolBuilder priority() { + checkExecutorType(); + this.priority = true; + return this; + } + + public ThreadPoolBuilder preStartAllCoreThreads(boolean preStartAllCoreThreads) { + this.preStartAllCoreThreads = preStartAllCoreThreads; + return this; + } + + public ThreadPoolBuilder rejectEnhanced(boolean rejectEnhanced) { + this.rejectEnhanced = rejectEnhanced; + return this; + } + + public ThreadPoolBuilder notifyEnabled(boolean notifyEnabled) { + this.notifyEnabled = notifyEnabled; + return this; + } + + public ThreadPoolBuilder runTimeout(long runTimeout) { + this.runTimeout = runTimeout; + return this; + } + + public ThreadPoolBuilder tryInterrupt(boolean tryInterrupt) { + this.tryInterrupt = tryInterrupt; + return this; + } + + public ThreadPoolBuilder queueTimeout(long queueTimeout) { + this.queueTimeout = queueTimeout; + return this; + } + + public ThreadPoolBuilder taskWrappers(List taskWrappers) { + this.taskWrappers.addAll(taskWrappers); + return this; + } + + public ThreadPoolBuilder taskWrapper(TaskWrapper taskWrapper) { + this.taskWrappers.add(taskWrapper); + return this; + } + + public ThreadPoolBuilder platformIds(List platformIds) { + if (CollectionUtils.isNotEmpty(platformIds)) { + this.platformIds = platformIds; + } + return this; + } + + /** + * Build according to dynamic field. + * + * @return the newly created ThreadPoolExecutor instance + */ + public ThreadPoolExecutor build() { + if (dynamic) { + return buildDtpExecutor(this); + } else { + return buildCommonExecutor(this); + } + } + + /** + * Build a dynamic ThreadPoolExecutor. + * + * @return the newly created DtpExecutor instance + */ + public EngineExecutor buildDynamic() { + return buildDtpExecutor(this); + } + + /** + * Build common ThreadPoolExecutor. + * + * @return the newly created ThreadPoolExecutor instance + */ + public ThreadPoolExecutor buildCommon() { + return buildCommonExecutor(this); + } + + /** + * Build dynamic threadPoolExecutor. + * + * @param builder the targeted builder + * @return the newly created DtpExecutor instance + */ + private EngineExecutor buildDtpExecutor(ThreadPoolBuilder builder) { + Preconditions.checkNotNull(builder.threadPoolName, "The thread pool name must not be null."); + EngineExecutor dtpExecutor = createInternal(builder); + dtpExecutor.setThreadPoolName(builder.threadPoolName); + dtpExecutor.allowCoreThreadTimeOut(builder.allowCoreThreadTimeOut); + dtpExecutor.setWaitForTasksToCompleteOnShutdown(builder.waitForTasksToCompleteOnShutdown); + dtpExecutor.setAwaitTerminationSeconds(builder.awaitTerminationSeconds); + dtpExecutor.setPreStartAllCoreThreads(builder.preStartAllCoreThreads); + dtpExecutor.setRejectEnhanced(builder.rejectEnhanced); + dtpExecutor.setRunTimeout(builder.runTimeout); + dtpExecutor.setTryInterrupt(builder.tryInterrupt); + dtpExecutor.setQueueTimeout(builder.queueTimeout); + dtpExecutor.setTaskWrappers(builder.taskWrappers); + dtpExecutor.setPlatformIds(builder.platformIds); + dtpExecutor.setNotifyEnabled(builder.notifyEnabled); + dtpExecutor.setRejectHandler(builder.rejectedExecutionHandler); + return dtpExecutor; + } + + private EngineExecutor createInternal(ThreadPoolBuilder builder) { + EngineExecutor dtpExecutor; + if (eager) { + TaskQueue taskQueue = new TaskQueue(builder.queueCapacity); + dtpExecutor = new EagerEngineExecutor( + builder.corePoolSize, + builder.maximumPoolSize, + builder.keepAliveTime, + builder.timeUnit, + taskQueue, + builder.threadFactory, + builder.rejectedExecutionHandler); + taskQueue.setExecutor((EagerEngineExecutor) dtpExecutor); + } else { + dtpExecutor = new EngineExecutor( + builder.corePoolSize, + builder.maximumPoolSize, + builder.keepAliveTime, + builder.timeUnit, + builder.workQueue, + builder.threadFactory, + builder.rejectedExecutionHandler); + } + return dtpExecutor; + } + + /** + * Build common threadPoolExecutor, does not manage by DynamicTp framework. + * + * @param builder the targeted builder + * @return the newly created ThreadPoolExecutor instance + */ + private ThreadPoolExecutor buildCommonExecutor(ThreadPoolBuilder builder) { + if (scheduled) { + return new ScheduledThreadPoolExecutor( + builder.corePoolSize, + builder.threadFactory, + builder.rejectedExecutionHandler); + } + ThreadPoolExecutor executor = new ThreadPoolExecutor( + builder.corePoolSize, + builder.maximumPoolSize, + builder.keepAliveTime, + builder.timeUnit, + builder.workQueue, + builder.threadFactory, + builder.rejectedExecutionHandler); + executor.allowCoreThreadTimeOut(builder.allowCoreThreadTimeOut); + return executor; + } + + /** + * Check executor type. + */ + private void checkExecutorType() { + if (eager || ordered || scheduled || priority) { + // 抛异常 + throw new IllegalArgumentException("More than one executor type is defined"); + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/ThreadPoolMetricsHandler.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/ThreadPoolMetricsHandler.java new file mode 100644 index 0000000000..5566d23e9a --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/ThreadPoolMetricsHandler.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect; + +import com.dtstack.taier.metrics.collect.collector.InternalLogCollector; +import com.dtstack.taier.metrics.collect.collector.MetricsCollector; +import com.dtstack.taier.metrics.collect.collector.MicroMeterCollector; +import com.dtstack.taier.metrics.collect.collector.SystemMeterCollector; +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; +import com.dtstack.taier.metrics.collect.util.ExtensionServiceLoader; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +/** + * adapter for ThreadPool Metrics collector + * @author xingyi + * @date 2025/9/16 + */ +public class ThreadPoolMetricsHandler { + + private static final Map COLLECTORS = Maps.newHashMap(); + + private ThreadPoolMetricsHandler() { + List loadedCollectors = ExtensionServiceLoader.get(MetricsCollector.class); + loadedCollectors.forEach(collector -> COLLECTORS.put(collector.type().toLowerCase(), collector)); + + MetricsCollector microMeterCollector = new MicroMeterCollector(); + InternalLogCollector internalLogCollector = new InternalLogCollector(); + SystemMeterCollector systemMeterCollector = new SystemMeterCollector(); + COLLECTORS.put(microMeterCollector.type(), microMeterCollector); + COLLECTORS.put(internalLogCollector.type(), internalLogCollector); + COLLECTORS.put(systemMeterCollector.type(), systemMeterCollector); + } + + public void collect(ThreadPoolStats threadPoolStats, List collectorTypes) { + for (String collectorType : collectorTypes) { + MetricsCollector collector = COLLECTORS.get(collectorType.toLowerCase()); + if (collector != null) { + collector.collect(threadPoolStats); + } + } + } + + public static ThreadPoolMetricsHandler getInstance() { + return CollectorHandlerHolder.INSTANCE; + } + + private static class CollectorHandlerHolder { + + private static final ThreadPoolMetricsHandler INSTANCE = new ThreadPoolMetricsHandler(); + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/AbstractCollector.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/AbstractCollector.java new file mode 100644 index 0000000000..7bd3cd9d47 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/AbstractCollector.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.collector; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public abstract class AbstractCollector implements MetricsCollector { + + @Override + public boolean support(String type) { + return this.type() + + .equalsIgnoreCase(type); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/InternalLogCollector.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/InternalLogCollector.java new file mode 100644 index 0000000000..1596f41767 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/InternalLogCollector.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.collector; + +import com.dtstack.taier.metrics.collect.em.CollectorTypeEnum; +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; +import lombok.extern.slf4j.Slf4j; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +public class InternalLogCollector extends AbstractCollector { + + @Override + public void collect(ThreadPoolStats poolStats) { + log.info("dynamic.tp metrics: {}", poolStats.toString()); + } + + @Override + public String type() { + return CollectorTypeEnum.LOGGING.name().toLowerCase(); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MetricsCollector.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MetricsCollector.java new file mode 100644 index 0000000000..200146669d --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MetricsCollector.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.collector; + +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; + +/** + * metrics collector for ThreadPools + * @author xingyi + * @date 2025/9/16 + */ +public interface MetricsCollector { + + /** + * Collect key metrics. + * @param poolStats ThreadPoolStats instance + */ + void collect(ThreadPoolStats poolStats); + + /** + * Collector type. + * @return collector type + */ + String type(); + + /** + * Judge collector type. + * @param type collector type + * @return true if the collector supports this type, else false + */ + boolean support(String type); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MicroMeteHandler.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MicroMeteHandler.java new file mode 100644 index 0000000000..25b168ddb6 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MicroMeteHandler.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.collector; + +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; + +/** + * support with microMete handler ,@see {@link MicroMeterCollector} + * @author xingyi + * @date 2025/9/18 + */ +public interface MicroMeteHandler { + + void collect(ThreadPoolStats threadPoolStats); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MicroMeterCollector.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MicroMeterCollector.java new file mode 100644 index 0000000000..461b26767c --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/MicroMeterCollector.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.collector; + +import com.dtstack.taier.metrics.collect.em.CollectorTypeEnum; +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; +import com.dtstack.taier.metrics.collect.util.ExtensionServiceLoader; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class MicroMeterCollector extends AbstractCollector { + + /** + * Prefix used for all dtp metric names. + */ + public static final String DTP_METRIC_NAME_PREFIX = "thread.pool"; + + public static final String POOL_NAME_TAG = DTP_METRIC_NAME_PREFIX + ".name"; + + public static final String POOL_ALIAS_TAG = DTP_METRIC_NAME_PREFIX + ".alias"; + + public static final String APP_NAME_TAG = "app.name"; + + private static final Map GAUGE_CACHE = new ConcurrentHashMap<>(); + + @Override + public void collect(ThreadPoolStats threadPoolStats) { + // metrics must be held with a strong reference, even though it is never referenced within this class + ThreadPoolStats oldStats = GAUGE_CACHE.get(threadPoolStats.getPoolName()); + if (Objects.isNull(oldStats)) { + GAUGE_CACHE.put(threadPoolStats.getPoolName(), threadPoolStats); + } else { + // BeanUtil.copyProperties(threadPoolStats, oldStats); + } + gauge(GAUGE_CACHE.get(threadPoolStats.getPoolName())); + } + + @Override + public String type() { + return CollectorTypeEnum.MICROMETER.name().toLowerCase(); + } + + public void gauge(ThreadPoolStats poolStats) { + // use SPI for gauge message + List loadedCollectors = ExtensionServiceLoader.get(MicroMeteHandler.class); + for (MicroMeteHandler microMeteHandler : loadedCollectors) { + microMeteHandler.collect(poolStats); + } + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/SystemMeterCollector.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/SystemMeterCollector.java new file mode 100644 index 0000000000..33ee8b5b5e --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/collector/SystemMeterCollector.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.collector; + +import com.dtstack.taier.metrics.collect.em.CollectorTypeEnum; +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class SystemMeterCollector extends AbstractCollector { + + @Override + public void collect(ThreadPoolStats poolStats) { + System.out.println("system meter collect: " + poolStats); + } + + @Override + public String type() { + return CollectorTypeEnum.OUTPUT.name().toLowerCase(); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/CollectorTypeEnum.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/CollectorTypeEnum.java new file mode 100644 index 0000000000..60fba11a70 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/CollectorTypeEnum.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.em; + +/** + * support metrics collector Type, @see LOGGING,@ see MICROMETER + * @author xingyi + * @date 2025/9/16 + */ +public enum CollectorTypeEnum { + + /** + * Micrometer collect type. + */ + MICROMETER, + + /** + * Logging collect type. + */ + LOGGING, + + /** + * Logging collect by System out + */ + OUTPUT + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/QueueTypeEnum.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/QueueTypeEnum.java new file mode 100644 index 0000000000..70ad312fe0 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/QueueTypeEnum.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.em; + +import com.dtstack.taier.metrics.exception.TpException; +import com.dtstack.taier.metrics.queue.VariableLinkedBlockingQueue; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.SynchronousQueue; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +@Getter +@AllArgsConstructor +public enum QueueTypeEnum { + + /** + * BlockingQueue type. + */ + ARRAY_BLOCKING_QUEUE(1, "ArrayBlockingQueue"), + + LINKED_BLOCKING_QUEUE(2, "LinkedBlockingQueue"), + + PRIORITY_BLOCKING_QUEUE(3, "PriorityBlockingQueue"), + + DELAY_QUEUE(4, "DelayQueue"), + + SYNCHRONOUS_QUEUE(5, "SynchronousQueue"), + + LINKED_TRANSFER_QUEUE(6, "LinkedTransferQueue"), + + LINKED_BLOCKING_DEQUE(7, "LinkedBlockingDeque"), + + VARIABLE_LINKED_BLOCKING_QUEUE(8, "VariableLinkedBlockingQueue"); + + private final int code; + + private final String name; + + public static BlockingQueue buildLbq(String name, int capacity) { + return buildLbq(name, capacity, false, 256); + } + + @SuppressWarnings("all") + public static BlockingQueue buildLbq(String name, int capacity, boolean fair, int maxFreeMemory) { + BlockingQueue blockingQueue = null; + if (Objects.equals(name, ARRAY_BLOCKING_QUEUE.getName())) { + blockingQueue = new ArrayBlockingQueue<>(capacity); + } else if (Objects.equals(name, LINKED_BLOCKING_QUEUE.getName())) { + blockingQueue = new LinkedBlockingQueue<>(capacity); + } else if (Objects.equals(name, PRIORITY_BLOCKING_QUEUE.getName())) { + blockingQueue = new PriorityBlockingQueue<>(capacity); + } else if (Objects.equals(name, DELAY_QUEUE.getName())) { + blockingQueue = new DelayQueue(); + } else if (Objects.equals(name, SYNCHRONOUS_QUEUE.getName())) { + blockingQueue = new SynchronousQueue<>(fair); + } else if (Objects.equals(name, LINKED_TRANSFER_QUEUE.getName())) { + blockingQueue = new LinkedTransferQueue<>(); + } else if (Objects.equals(name, LINKED_BLOCKING_DEQUE.getName())) { + blockingQueue = new LinkedBlockingDeque<>(capacity); + } else if (Objects.equals(name, VARIABLE_LINKED_BLOCKING_QUEUE.getName())) { + blockingQueue = new VariableLinkedBlockingQueue<>(capacity); + } + if (blockingQueue != null) { + return blockingQueue; + } + + log.error("Cannot find specified BlockingQueue {}", name); + throw new TpException("Cannot find specified BlockingQueue " + name); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/RejectedTypeEnum.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/RejectedTypeEnum.java new file mode 100644 index 0000000000..295ed20b95 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/em/RejectedTypeEnum.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.em; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +@Getter +@AllArgsConstructor +public enum RejectedTypeEnum { + + /** + * RejectedExecutionHandler type while triggering reject policy. + */ + ABORT_POLICY("AbortPolicy"), + + CALLER_RUNS_POLICY("CallerRunsPolicy"), + + DISCARD_OLDEST_POLICY("DiscardOldestPolicy"), + + DISCARD_POLICY("DiscardPolicy"); + + private final String name; +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/Metrics.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/Metrics.java new file mode 100644 index 0000000000..d86ef87d10 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/Metrics.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.entity; + +/** + * metrics base class + * @author xingyi + * @date 2025/9/16 + */ +public class Metrics { +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/ThreadPoolStats.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/ThreadPoolStats.java new file mode 100644 index 0000000000..b0c6f926d0 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/ThreadPoolStats.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.entity; + +/** + * collector for ThreadPool ThreadPoolStats + * @author xingyi + * @date 2025/9/16 + */ +public class ThreadPoolStats extends Metrics { + + /** + * 线程池名字 + */ + private String poolName; + + /** + * 线程池别名 + */ + private String poolAliasName; + + /** + * 核心线程数 + */ + private int corePoolSize; + + /** + * 最大线程数 + */ + private int maximumPoolSize; + + /** + * 空闲时间 (ms) + */ + private long keepAliveTime; + + /** + * 队列类型 + */ + private String queueType; + + /** + * 队列容量 + */ + private int queueCapacity; + + /** + * 队列任务数量 + */ + private int queueSize; + + /** + * SynchronousQueue队列模式 + */ + private boolean fair; + + /** + * 队列剩余容量 + */ + private int queueRemainingCapacity; + + /** + * 正在执行任务的活跃线程大致总数 + */ + private int activeCount; + + /** + * 大致任务总数 + */ + private long taskCount; + + /** + * 已执行完成的大致任务总数 + */ + private long completedTaskCount; + + /** + * 池中曾经同时存在的最大线程数量 + */ + private int largestPoolSize; + + /** + * 当前池中存在的线程总数 + */ + private int poolSize; + + /** + * 等待执行的任务数量 + */ + private int waitTaskCount; + + /** + * 拒绝的任务数量 + */ + private long rejectCount; + + /** + * 拒绝策略名称 + */ + private String rejectHandlerName; + + /** + * 是否DtpExecutor线程池 + */ + private boolean dynamic; + + /** + * 执行超时任务数量 + */ + private long runTimeoutCount; + + /** + * 在队列等待超时任务数量 + */ + private long queueTimeoutCount; + + /** + * tps + */ + private double tps; + + /** + * 最大任务耗时 + */ + private long maxRt; + + /** + * 最小任务耗时 + */ + private long minRt; + + /** + * 任务平均耗时(单位:ms) + */ + private double avg; + + public String getPoolName() { + return poolName; + } + + public void setPoolName(String poolName) { + this.poolName = poolName; + } + + public String getPoolAliasName() { + return poolAliasName; + } + + public void setPoolAliasName(String poolAliasName) { + this.poolAliasName = poolAliasName; + } + + public int getCorePoolSize() { + return corePoolSize; + } + + public void setCorePoolSize(int corePoolSize) { + this.corePoolSize = corePoolSize; + } + + public int getMaximumPoolSize() { + return maximumPoolSize; + } + + public void setMaximumPoolSize(int maximumPoolSize) { + this.maximumPoolSize = maximumPoolSize; + } + + public long getKeepAliveTime() { + return keepAliveTime; + } + + public void setKeepAliveTime(long keepAliveTime) { + this.keepAliveTime = keepAliveTime; + } + + public String getQueueType() { + return queueType; + } + + public void setQueueType(String queueType) { + this.queueType = queueType; + } + + public int getQueueCapacity() { + return queueCapacity; + } + + public void setQueueCapacity(int queueCapacity) { + this.queueCapacity = queueCapacity; + } + + public int getQueueSize() { + return queueSize; + } + + public void setQueueSize(int queueSize) { + this.queueSize = queueSize; + } + + public boolean isFair() { + return fair; + } + + public void setFair(boolean fair) { + this.fair = fair; + } + + public int getQueueRemainingCapacity() { + return queueRemainingCapacity; + } + + public void setQueueRemainingCapacity(int queueRemainingCapacity) { + this.queueRemainingCapacity = queueRemainingCapacity; + } + + public int getActiveCount() { + return activeCount; + } + + public void setActiveCount(int activeCount) { + this.activeCount = activeCount; + } + + public long getTaskCount() { + return taskCount; + } + + public void setTaskCount(long taskCount) { + this.taskCount = taskCount; + } + + public long getCompletedTaskCount() { + return completedTaskCount; + } + + public void setCompletedTaskCount(long completedTaskCount) { + this.completedTaskCount = completedTaskCount; + } + + public int getLargestPoolSize() { + return largestPoolSize; + } + + public void setLargestPoolSize(int largestPoolSize) { + this.largestPoolSize = largestPoolSize; + } + + public int getPoolSize() { + return poolSize; + } + + public void setPoolSize(int poolSize) { + this.poolSize = poolSize; + } + + public int getWaitTaskCount() { + return waitTaskCount; + } + + public void setWaitTaskCount(int waitTaskCount) { + this.waitTaskCount = waitTaskCount; + } + + public long getRejectCount() { + return rejectCount; + } + + public void setRejectCount(long rejectCount) { + this.rejectCount = rejectCount; + } + + public String getRejectHandlerName() { + return rejectHandlerName; + } + + public void setRejectHandlerName(String rejectHandlerName) { + this.rejectHandlerName = rejectHandlerName; + } + + public boolean isDynamic() { + return dynamic; + } + + public void setDynamic(boolean dynamic) { + this.dynamic = dynamic; + } + + public long getRunTimeoutCount() { + return runTimeoutCount; + } + + public void setRunTimeoutCount(long runTimeoutCount) { + this.runTimeoutCount = runTimeoutCount; + } + + public long getQueueTimeoutCount() { + return queueTimeoutCount; + } + + public void setQueueTimeoutCount(long queueTimeoutCount) { + this.queueTimeoutCount = queueTimeoutCount; + } + + public double getTps() { + return tps; + } + + public void setTps(double tps) { + this.tps = tps; + } + + public long getMaxRt() { + return maxRt; + } + + public void setMaxRt(long maxRt) { + this.maxRt = maxRt; + } + + public long getMinRt() { + return minRt; + } + + public void setMinRt(long minRt) { + this.minRt = minRt; + } + + public double getAvg() { + return avg; + } + + public void setAvg(double avg) { + this.avg = avg; + } + + @Override + public String toString() { + return "ThreadPoolStats{" + + "poolName='" + poolName + '\'' + + ", poolAliasName='" + poolAliasName + '\'' + + ", corePoolSize=" + corePoolSize + + ", maximumPoolSize=" + maximumPoolSize + + ", keepAliveTime=" + keepAliveTime + + ", queueType='" + queueType + '\'' + + ", queueCapacity=" + queueCapacity + + ", queueSize=" + queueSize + + ", fair=" + fair + + ", queueRemainingCapacity=" + queueRemainingCapacity + + ", activeCount=" + activeCount + + ", taskCount=" + taskCount + + ", completedTaskCount=" + completedTaskCount + + ", largestPoolSize=" + largestPoolSize + + ", poolSize=" + poolSize + + ", waitTaskCount=" + waitTaskCount + + ", rejectCount=" + rejectCount + + ", rejectHandlerName='" + rejectHandlerName + '\'' + + ", dynamic=" + dynamic + + ", runTimeoutCount=" + runTimeoutCount + + ", queueTimeoutCount=" + queueTimeoutCount + + ", tps=" + tps + + ", maxRt=" + maxRt + + ", minRt=" + minRt + + ", avg=" + avg + + '}'; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/TpMainFields.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/TpMainFields.java new file mode 100644 index 0000000000..e3626d5b31 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/entity/TpMainFields.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.entity; + +import lombok.Data; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; + +/** + * TpMainFields related + * + * @author yanhom + * @since 1.0.0 + **/ +@Data +public class TpMainFields { + + private static final List FIELD_NAMES; + + static { + FIELD_NAMES = Arrays.asList(TpMainFields.class.getDeclaredFields()); + } + + private String threadPoolName; + + private int corePoolSize; + + private int maxPoolSize; + + private long keepAliveTime; + + private String queueType; + + private int queueCapacity; + + private String rejectType; + + private boolean allowCoreThreadTimeOut; + + public static List getMainFields() { + return FIELD_NAMES; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/util/ExecutorUtil.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/util/ExecutorUtil.java new file mode 100644 index 0000000000..f9ddfe7b0d --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/util/ExecutorUtil.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.util; + +import lombok.extern.slf4j.Slf4j; +import org.slf4j.MDC; + +import java.util.Objects; +import java.util.concurrent.FutureTask; + +/** + * ExecutorUtil related + * + * @author yanhom + * @since 1.1.9 + */ +@Slf4j +public final class ExecutorUtil { + + private ExecutorUtil() { + } + + public static void tryExecAfterExecute(Runnable r, Throwable t) { + tryPrintError(r, t); + tryClearContext(); + } + + private static void tryPrintError(Runnable r, Throwable t) { + if (Objects.nonNull(t)) { + log.error("DynamicTp execute, thread {} throw exception, traceId {}", + Thread.currentThread(), MDC.get("traceId"), t); + return; + } + if (r instanceof FutureTask) { + try { + FutureTask future = (FutureTask) r; + if (future.isDone() && !future.isCancelled()) { + future.get(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("DynamicTp execute, thread {} throw exception, traceId {}", + Thread.currentThread(), MDC.get("traceId"), e); + } + } + } + + public static void tryClearContext() { + MDC.remove("traceId"); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/util/ExtensionServiceLoader.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/util/ExtensionServiceLoader.java new file mode 100644 index 0000000000..d471b0bb3d --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/collect/util/ExtensionServiceLoader.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.collect.util; + +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Unified ServiceLoader Helper + * @author xingyi + * @date 2025/9/17 + */ +public class ExtensionServiceLoader { + + private static final Map, List> EXTENSION_MAP = new ConcurrentHashMap<>(); + + private ExtensionServiceLoader() { + } + + /** + * load service + * @param clazz SPI interface + * @return services + * @param interface class + */ + @SuppressWarnings("unchecked") + public static List get(Class clazz) { + List services = (List) EXTENSION_MAP.get(clazz); + if (CollectionUtils.isEmpty(services)) { + services = load(clazz); + if (CollectionUtils.isNotEmpty(services)) { + EXTENSION_MAP.put(clazz, services); + } + } + return services; + } + + /** + * load the first service + * @param clazz SPI interface + * @return service + * @param interface class + */ + public static T getFirst(Class clazz) { + List services = get(clazz); + return CollectionUtils.isEmpty(services) ? null : services.get(0); + } + + private static List load(Class clazz) { + ServiceLoader serviceLoader = ServiceLoader.load(clazz); + List services = new ArrayList<>(); + for (T service : serviceLoader) { + services.add(service); + } + return services; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/conveter/ExecutorConverter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/conveter/ExecutorConverter.java new file mode 100644 index 0000000000..188122fbdf --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/conveter/ExecutorConverter.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.conveter; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.PerformanceProvider; +import com.dtstack.taier.metrics.ThreadPoolStatProvider; +import com.dtstack.taier.metrics.adapter.ExecutorAdapter; +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; + +import java.util.concurrent.TimeUnit; + +/** + * ExecutorConverter related + **/ +public class ExecutorConverter { + + private ExecutorConverter() { + } + + public static ThreadPoolStats toMetrics(ExecutorWrapper wrapper) { + ExecutorAdapter executor = wrapper.getExecutor(); + if (executor == null) { + return null; + } + ThreadPoolStatProvider provider = wrapper.getThreadPoolStatProvider(); + PerformanceProvider performanceProvider = provider.getPerformanceProvider(); + PerformanceProvider.PerformanceSnapshot performanceSnapshot = performanceProvider.getSnapshotAndReset(); + ThreadPoolStats poolStats = convertCommon(executor); + poolStats.setPoolName(wrapper.getThreadPoolName()); + poolStats.setPoolAliasName(wrapper.getThreadPoolAliasName()); + poolStats.setRunTimeoutCount(provider.getRunTimeoutCount()); + poolStats.setQueueTimeoutCount(provider.getQueueTimeoutCount()); + poolStats.setRejectCount(provider.getRejectedTaskCount()); + + poolStats.setTps(performanceSnapshot.getTps()); + poolStats.setAvg(performanceSnapshot.getAvg()); + poolStats.setMaxRt(performanceSnapshot.getMaxRt()); + poolStats.setMinRt(performanceSnapshot.getMinRt()); + return poolStats; + } + + private static ThreadPoolStats convertCommon(ExecutorAdapter executor) { + ThreadPoolStats poolStats = new ThreadPoolStats(); + poolStats.setCorePoolSize(executor.getCorePoolSize()); + poolStats.setMaximumPoolSize(executor.getMaximumPoolSize()); + poolStats.setPoolSize(executor.getPoolSize()); + poolStats.setActiveCount(executor.getActiveCount()); + poolStats.setLargestPoolSize(executor.getLargestPoolSize()); + poolStats.setQueueType(executor.getQueueType()); + poolStats.setQueueCapacity(executor.getQueueCapacity()); + poolStats.setQueueSize(executor.getQueueSize()); + poolStats.setQueueRemainingCapacity(executor.getQueueRemainingCapacity()); + poolStats.setTaskCount(executor.getTaskCount()); + poolStats.setCompletedTaskCount(executor.getCompletedTaskCount()); + poolStats.setWaitTaskCount(executor.getQueueSize()); + poolStats.setRejectHandlerName(executor.getRejectHandlerType()); + poolStats.setKeepAliveTime(executor.getKeepAliveTime(TimeUnit.MILLISECONDS)); + return poolStats; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/exception/TpException.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/exception/TpException.java new file mode 100644 index 0000000000..e2f9296193 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/exception/TpException.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.exception; + +/** + * @author xingyi + * @date 2025/9/18 + */ +public class TpException extends RuntimeException { + + public TpException() { + super(); + } + + public TpException(String message) { + super(message); + } + + public TpException(String message, Throwable cause) { + super(message, cause); + } + + public TpException(Throwable cause) { + super(cause); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/executor/EagerEngineExecutor.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/executor/EagerEngineExecutor.java new file mode 100644 index 0000000000..410923fbdf --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/executor/EagerEngineExecutor.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.executor; + +import com.dtstack.taier.metrics.queue.TaskQueue; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * When core threads are all in busy, + * create new thread instead of putting task into blocking queue, + * mainly used in io intensive scenario. + * 优先创建线程池线程,当线程池maximumPoolSize满时,才会将任务放入阻塞队列 + * @author xingyi + * @date 2025/9/17 + */ +public class EagerEngineExecutor extends EngineExecutor { + + /** + * The number of tasks submitted but not yet finished. + */ + private final AtomicInteger submittedTaskCount = new AtomicInteger(0); + + public EagerEngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue) { + this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + Executors.defaultThreadFactory(), new AbortPolicy()); + } + + public EagerEngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory) { + this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + threadFactory, new AbortPolicy()); + } + + public EagerEngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + RejectedExecutionHandler handler) { + this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + Executors.defaultThreadFactory(), handler); + } + + public EagerEngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + } + + public int getSubmittedTaskCount() { + return submittedTaskCount.get(); + } + + @Override + public void execute(Runnable command) { + if (command == null) { + throw new NullPointerException(); + } + submittedTaskCount.incrementAndGet(); + try { + super.execute(command); + } catch (RejectedExecutionException rx) { + if (getQueue() instanceof TaskQueue) { + // If the Executor is close to maximum pool size, concurrent + // calls to execute() may result (due to use of TaskQueue) in + // some tasks being rejected rather than queued. + // If this happens, add them to the queue. + final TaskQueue queue = (TaskQueue) getQueue(); + try { + if (!queue.force(command, 0, TimeUnit.MILLISECONDS)) { + submittedTaskCount.decrementAndGet(); + throw new RejectedExecutionException("Queue capacity is full.", rx); + } + } catch (InterruptedException x) { + submittedTaskCount.decrementAndGet(); + throw new RejectedExecutionException(x); + } + } else { + submittedTaskCount.decrementAndGet(); + throw rx; + } + } + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + submittedTaskCount.decrementAndGet(); + super.afterExecute(r, t); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/executor/EngineExecutor.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/executor/EngineExecutor.java new file mode 100644 index 0000000000..4397196fa5 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/executor/EngineExecutor.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.executor; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.adapter.ExecutorAdapter; +import com.dtstack.taier.metrics.aware.AwareManager; +import com.dtstack.taier.metrics.aware.TaskEnhanceAware; +import com.dtstack.taier.metrics.collect.util.ExecutorUtil; +import com.dtstack.taier.metrics.registry.TpRegistry; +import com.dtstack.taier.metrics.rejects.RejectHandlerGetter; +import com.dtstack.taier.metrics.wrapper.TaskWrapper; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import lombok.val; + +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Dynamic ThreadPoolExecutor, extending ThreadPoolExecutor, implements some new features + * @author xingyi + * @date 2025/9/17 + */ +public class EngineExecutor extends ThreadPoolExecutor + implements + TaskEnhanceAware, + ExecutorAdapter { + + /** + * The name of the thread pool. + */ + protected String threadPoolName; + + /** + * Simple Business alias Name of Dynamic ThreadPool. Use for notify. + */ + private String threadPoolAliasName; + + /** + * If enable notify. + */ + private boolean notifyEnabled = true; + + /** + * Notify platform ids. + */ + private List platformIds; + + /** + * Task wrappers, do sth enhanced. + */ + private List taskWrappers = Lists.newArrayList(); + + /** + * Plugin names. + */ + private Set pluginNames = Sets.newHashSet(); + + /** + * Aware names. + */ + private Set awareNames = Sets.newHashSet(); + + /** + * If pre start all core threads. + */ + private boolean preStartAllCoreThreads; + + /** + * RejectHandler type. + */ + private String rejectHandlerType; + + /** + * If enhance reject. + */ + private boolean rejectEnhanced = true; + + /** + * for manual builder thread pools only + */ + private long runTimeout = 0; + + /** + * for manual builder thread pools only + */ + private boolean tryInterrupt = false; + + /** + * for manual builder thread pools only + */ + private long queueTimeout = 0; + + /** + * Whether to wait for scheduled tasks to complete on shutdown, + * not interrupting running tasks and executing all tasks in the queue. + */ + protected boolean waitForTasksToCompleteOnShutdown = false; + + /** + * The maximum number of seconds that this executor is supposed to block + * on shutdown in order to wait for remaining tasks to complete their execution + * before the rest of the container continues to shut down. + */ + protected int awaitTerminationSeconds = 0; + + public EngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue) { + this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + Executors.defaultThreadFactory(), new AbortPolicy()); + } + + public EngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory) { + this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + threadFactory, new AbortPolicy()); + } + + public EngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + RejectedExecutionHandler handler) { + this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + Executors.defaultThreadFactory(), handler); + } + + public EngineExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + } + + @Override + public ThreadPoolExecutor getOriginal() { + return this; + } + + @Override + public void execute(Runnable command) { + command = getEnhancedTask(command); + AwareManager.execute(this, command); + super.execute(command); + } + + @Override + protected void beforeExecute(Thread t, Runnable r) { + AwareManager.beforeExecute(this, t, r); + super.beforeExecute(t, r); + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + AwareManager.afterExecute(this, r, t); + ExecutorUtil.tryExecAfterExecute(r, t); + } + + @Override + public void shutdown() { + super.shutdown(); + AwareManager.shutdown(this); + } + + @Override + public List shutdownNow() { + List tasks = super.shutdownNow(); + AwareManager.shutdownNow(this, tasks); + return tasks; + } + + @Override + protected void terminated() { + super.terminated(); + AwareManager.terminated(this); + } + + public void initialize() { + if (preStartAllCoreThreads) { + prestartAllCoreThreads(); + } + if (rejectEnhanced) { + // should proxy reject handler if rejectEnhanced is true + // reset reject handler in initialize phase according to rejectEnhanced + RejectedExecutionHandler rejectedExecutionHandler = this.getRejectedExecutionHandler(); + if (rejectedExecutionHandler instanceof Proxy) { + return; + } + setRejectHandler(rejectedExecutionHandler != null ? rejectedExecutionHandler + : RejectHandlerGetter.buildRejectedHandler(getRejectHandlerType())); + } + } + + public void setRejectHandler(RejectedExecutionHandler handler) { + this.rejectHandlerType = handler.getClass().getSimpleName(); + if (!isRejectEnhanced()) { + setRejectedExecutionHandler(handler); + return; + } + setRejectedExecutionHandler(RejectHandlerGetter.getProxy(handler)); + } + + public String getThreadPoolName() { + return threadPoolName; + } + + public void setThreadPoolName(String threadPoolName) { + this.threadPoolName = threadPoolName; + } + + public String getThreadPoolAliasName() { + return threadPoolAliasName; + } + + public void setThreadPoolAliasName(String threadPoolAliasName) { + this.threadPoolAliasName = threadPoolAliasName; + } + + public boolean isNotifyEnabled() { + return notifyEnabled; + } + + public void setNotifyEnabled(boolean notifyEnabled) { + this.notifyEnabled = notifyEnabled; + } + + public List getPlatformIds() { + return platformIds; + } + + public void setPlatformIds(List platformIds) { + this.platformIds = platformIds; + } + + @Override + public List getTaskWrappers() { + return taskWrappers; + } + + @Override + public void setTaskWrappers(List taskWrappers) { + this.taskWrappers = taskWrappers; + } + + public Set getPluginNames() { + return pluginNames; + } + + public void setPluginNames(Set pluginNames) { + this.pluginNames = pluginNames; + } + + public Set getAwareNames() { + return awareNames; + } + + public void setAwareNames(Set awareNames) { + this.awareNames = awareNames; + } + + public boolean isPreStartAllCoreThreads() { + return preStartAllCoreThreads; + } + + public void setPreStartAllCoreThreads(boolean preStartAllCoreThreads) { + this.preStartAllCoreThreads = preStartAllCoreThreads; + } + + public boolean isRejectEnhanced() { + return rejectEnhanced; + } + + public void setRejectEnhanced(boolean rejectEnhanced) { + this.rejectEnhanced = rejectEnhanced; + } + + @Override + public String getRejectHandlerType() { + return rejectHandlerType; + } + + public void setRejectHandlerType(String rejectHandlerType) { + this.rejectHandlerType = rejectHandlerType; + } + + public long getRunTimeout() { + return runTimeout; + } + + public void setRunTimeout(long runTimeout) { + this.runTimeout = runTimeout; + } + + public boolean isTryInterrupt() { + return tryInterrupt; + } + + public void setTryInterrupt(boolean tryInterrupt) { + this.tryInterrupt = tryInterrupt; + } + + public long getQueueTimeout() { + return queueTimeout; + } + + public void setQueueTimeout(long queueTimeout) { + this.queueTimeout = queueTimeout; + } + + public boolean isWaitForTasksToCompleteOnShutdown() { + return waitForTasksToCompleteOnShutdown; + } + + public void setWaitForTasksToCompleteOnShutdown(boolean waitForTasksToCompleteOnShutdown) { + this.waitForTasksToCompleteOnShutdown = waitForTasksToCompleteOnShutdown; + } + + public int getAwaitTerminationSeconds() { + return awaitTerminationSeconds; + } + + public void setAwaitTerminationSeconds(int awaitTerminationSeconds) { + this.awaitTerminationSeconds = awaitTerminationSeconds; + } + + /** + * In order for the field can be assigned by reflection. + * + * @param allowCoreThreadTimeOut allowCoreThreadTimeOut + */ + public void setAllowCoreThreadTimeOut(boolean allowCoreThreadTimeOut) { + allowCoreThreadTimeOut(allowCoreThreadTimeOut); + } + + /** + * register current Executor to TpRegistry + */ + public EngineExecutor registry() { + TpRegistry.registerExecutor(ExecutorWrapper.of(this)); + return this; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/factory/TpThreadFactory.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/factory/TpThreadFactory.java new file mode 100644 index 0000000000..826b287f10 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/factory/TpThreadFactory.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.factory; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author xingyi + * @date 2025/9/18 + */ +public class TpThreadFactory implements ThreadFactory { + + private static final AtomicInteger POOL_NUMBER = new AtomicInteger(1); + private final ThreadGroup group; + private final AtomicInteger threadNumber = new AtomicInteger(1); + private final String namePrefix; + + public TpThreadFactory(String name) { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + namePrefix = "pool-" + name + "-" + + POOL_NUMBER.getAndIncrement() + + "-thread-"; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, + namePrefix + threadNumber.getAndIncrement(), + 0); + if (t.isDaemon()) { + t.setDaemon(false); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/LimitedUniformReservoir.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/LimitedUniformReservoir.java new file mode 100644 index 0000000000..04ae9dc2d4 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/LimitedUniformReservoir.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.metrice; + +import com.codahale.metrics.Reservoir; +import com.codahale.metrics.Snapshot; +import com.codahale.metrics.UniformSnapshot; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; + +/** + * LimitedUniformReservoir related + * + * @author yanhom + * @since 1.1.5 + */ +@SuppressWarnings("all") +public class LimitedUniformReservoir implements Reservoir { + + private static final int DEFAULT_SIZE = 4096; + + private static final int BITS_PER_LONG = 63; + + private final AtomicLong count = new AtomicLong(); + + private volatile AtomicLongArray values = new AtomicLongArray(DEFAULT_SIZE); + + @Override + public int size() { + final long c = count.get(); + if (c > values.length()) { + return values.length(); + } + return (int) c; + } + + @Override + public void update(long value) { + final long c = count.incrementAndGet(); + if (c <= values.length()) { + values.set((int) c - 1, value); + } else { + final long r = nextLong(c); + if (r < values.length()) { + values.set((int) r, value); + } + } + } + + @Override + public Snapshot getSnapshot() { + final int s = size(); + final List copy = new ArrayList<>(s); + for (int i = 0; i < s; i++) { + copy.add(values.get(i)); + } + return new UniformSnapshot(copy); + } + + public void reset() { + count.set(0); + values = new AtomicLongArray(DEFAULT_SIZE); + } + + private static long nextLong(long n) { + long bits; + long val; + do { + bits = ThreadLocalRandom.current().nextLong() & (~(1L << BITS_PER_LONG)); + val = bits % n; + } while (bits - val + (n - 1) < 0L); + return val; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/MMACounter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/MMACounter.java new file mode 100644 index 0000000000..30ab5f8803 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/MMACounter.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.metrice; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.concurrent.atomic.AtomicLong; + +/** + * MMACounter related + * + * @author yanhom + * @since 1.1.5 + */ +@SuppressWarnings("all") +public class MMACounter implements Summary { + + private final AtomicLong total = new AtomicLong(); + + private final AtomicLong count = new AtomicLong(); + + private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); + + private final AtomicLong max = new AtomicLong(Long.MIN_VALUE); + + @Override + public void add(long value) { + total.addAndGet(value); + count.incrementAndGet(); + setMin(value); + setMax(value); + } + + @Override + public void reset() { + total.set(0); + count.set(0); + min.set(Long.MAX_VALUE); + max.set(Long.MIN_VALUE); + } + + public long getTotal() { + return total.get(); + } + + public long getCount() { + return count.get(); + } + + public long getMin() { + long current = min.get(); + return (current == Long.MAX_VALUE) ? 0 : current; + } + + public long getMax() { + long current = max.get(); + return (current == Long.MIN_VALUE) ? 0 : current; + } + + public double getAvg() { + long currentCount = count.get(); + long currentTotal = total.get(); + if (currentCount > 0) { + double avgLatency = currentTotal / (double) currentCount; + BigDecimal bg = new BigDecimal(avgLatency); + return bg.setScale(4, RoundingMode.HALF_UP).doubleValue(); + } + return 0; + } + + private void setMax(long value) { + long current; + while (value > (current = max.get()) && !max.compareAndSet(current, value)) { + // no op + } + } + + private void setMin(long value) { + long current; + while (value < (current = min.get()) && !min.compareAndSet(current, value)) { + // no op + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/MMAPCounter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/MMAPCounter.java new file mode 100644 index 0000000000..80b2268cb3 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/MMAPCounter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.metrice; + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Snapshot; + +/** + * MMAPCounter related + * + * @author yanhom + * @since 1.1.5 + */ +@SuppressWarnings("all") +public class MMAPCounter implements Summary { + + private final MMACounter mmaCounter; + + private final LimitedUniformReservoir reservoir; + + private final Histogram histogram; + + public MMAPCounter() { + this.mmaCounter = new MMACounter(); + reservoir = new LimitedUniformReservoir(); + histogram = new Histogram(reservoir); + } + + @Override + public void add(long value) { + mmaCounter.add(value); + histogram.update(value); + } + + @Override + public void reset() { + mmaCounter.reset(); + reservoir.reset(); + } + + public Snapshot getSnapshot() { + return histogram.getSnapshot(); + } + + public MMACounter getMmaCounter() { + return mmaCounter; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/Meter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/Meter.java new file mode 100644 index 0000000000..9e87b84a6d --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/Meter.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.metrice; + +/** + * Meter related + * + * @author yanhom + * @since 1.1.5 + */ +public interface Meter { + + /** + * Reset meter. + */ + void reset(); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/Summary.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/Summary.java new file mode 100644 index 0000000000..c24aa989bc --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/metrice/Summary.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.metrice; + +/** + * Summary related + * + * @author yanhom + * @since 1.1.5 + */ +public interface Summary extends Meter { + + /** + * Add value. + * + * @param value current value + */ + void add(long value); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/monitor/TpMonitor.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/monitor/TpMonitor.java new file mode 100644 index 0000000000..b1818d622a --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/monitor/TpMonitor.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.monitor; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.collect.ThreadPoolMetricsHandler; +import com.dtstack.taier.metrics.collect.entity.ThreadPoolStats; +import com.dtstack.taier.metrics.conveter.ExecutorConverter; +import com.dtstack.taier.metrics.factory.TpThreadFactory; +import com.dtstack.taier.metrics.registry.TpRegistry; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +public class TpMonitor { + + private static ScheduledExecutorService monitorExecutor; + + private List supportMonitorMetricsType; + + public void interval(long monitorInterval, long monitorDelay, List supportMonitorMetricsType) { + + monitorExecutor = new ScheduledThreadPoolExecutor(1, + new TpThreadFactory("Taier-ThreadPool-Monitor")); + this.supportMonitorMetricsType = supportMonitorMetricsType; + + monitorExecutor.scheduleWithFixedDelay(this::run, monitorDelay, monitorInterval, TimeUnit.SECONDS); + } + + public void run() { + Set executorNames = TpRegistry.getAllExecutorNames(); + try { + collectMetrics(executorNames); + } catch (Exception e) { + log.error("DynamicTp monitor, run error", e); + } + } + + private void collectMetrics(Set executorNames) { + executorNames.forEach(x -> { + ExecutorWrapper wrapper = TpRegistry.getExecutorWrapper(x); + doCollect(ExecutorConverter.toMetrics(wrapper)); + }); + } + + private void doCollect(ThreadPoolStats threadPoolStats) { + try { + ThreadPoolMetricsHandler.getInstance().collect( + threadPoolStats, this.supportMonitorMetricsType); + } catch (Exception e) { + log.error("DynamicTp monitor, metrics collect error.", e); + } + } + + public void destroy() { + monitorExecutor.shutdownNow(); + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/CollectorRegistryHolder.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/CollectorRegistryHolder.java new file mode 100644 index 0000000000..89d0c0eb6a --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/CollectorRegistryHolder.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.prometheus; + +import io.micrometer.core.instrument.Clock; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.prometheus.client.CollectorRegistry; + +/** + * @author xingyi + * @date 2025/9/19 + */ +public class CollectorRegistryHolder { + + private static final CollectorRegistry collectorRegistry = new CollectorRegistry(); + + private static final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusMeterRegistry( + new PrometheusPropertiesConfigAdapter(), CollectorRegistryHolder.getCollectorRegistry(), Clock.SYSTEM); + + /** + * get final CollectorRegistry + */ + public static CollectorRegistry getCollectorRegistry() { + return collectorRegistry; + } + + /** + * get final PrometheusMeterRegistry + */ + public static PrometheusMeterRegistry getPrometheusMeterRegistry() { + return prometheusMeterRegistry; + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/PrometheusPropertiesConfigAdapter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/PrometheusPropertiesConfigAdapter.java new file mode 100644 index 0000000000..4113cadaa1 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/PrometheusPropertiesConfigAdapter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.prometheus; + +import io.micrometer.prometheus.HistogramFlavor; +import io.micrometer.prometheus.PrometheusConfig; + +import java.time.Duration; + +/** + * Adapter to convert to a {@link PrometheusConfig}. + */ +public class PrometheusPropertiesConfigAdapter + implements + PrometheusConfig { + + public PrometheusPropertiesConfigAdapter() { + } + + @Override + public String prefix() { + return "management.metrics.export.prometheus"; + } + + @Override + public String get(String key) { + return null; + } + + @Override + public boolean descriptions() { + return true; + } + + @Override + public HistogramFlavor histogramFlavor() { + return HistogramFlavor.Prometheus; + } + + @Override + public Duration step() { + // TODO 从配置文件中获取 + return Duration + .ofMinutes(Long.parseLong(System.getProperty("management.metrics.export.prometheus.step", "10"))); + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/PrometheusPushGatewayManager.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/PrometheusPushGatewayManager.java new file mode 100644 index 0000000000..5dad7a1d84 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/prometheus/PrometheusPushGatewayManager.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.prometheus; + +import io.micrometer.core.instrument.util.NamedThreadFactory; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.PushGateway; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Class that can be used to manage the pushing of metrics to a {@link PushGateway + * Prometheus PushGateway}. Handles the scheduling of push operations, error handling and + * shutdown operations. + * + * @author David J. M. Karlsen + * @author Phillip Webb + * @since 2.1.0 + */ +public class PrometheusPushGatewayManager { + + private static final Log logger = LogFactory.getLog(PrometheusPushGatewayManager.class); + + private final PushGateway pushGateway; + + private final CollectorRegistry registry; + + private final String job; + + private final Map groupingKey; + + private final ShutdownOperation shutdownOperation; + + private final ScheduledExecutorService scheduler; + + private ScheduledFuture scheduled; + + /** + * Create a new {@link PrometheusPushGatewayManager} instance using a single threaded + * {@link ThreadPoolExecutor}. + * + * @param pushGateway the source push gateway + * @param registry the collector registry to push + * @param pushRate the rate at which push operations occur + * @param job the job ID for the operation + * @param groupingKeys an optional set of grouping keys for the operation + * @param shutdownOperation the shutdown operation that should be performed when + * context is closed. + */ + public PrometheusPushGatewayManager(PushGateway pushGateway, CollectorRegistry registry, long pushRate, + String job, Map groupingKeys, + ShutdownOperation shutdownOperation) { + this(pushGateway, registry, new ScheduledThreadPoolExecutor(1, new NamedThreadFactory(job)), + pushRate, job, groupingKeys, shutdownOperation); + } + + /** + * Create a new {@link PrometheusPushGatewayManager} instance. + * + * @param pushGateway the source push gateway + * @param registry the collector registry to push + * @param scheduler the scheduler used for operations + * @param pushRate the rate at which push operations occur + * @param job the job ID for the operation + * @param groupingKey an optional set of grouping keys for the operation + * @param shutdownOperation the shutdown operation that should be performed when + * context is closed. + */ + public PrometheusPushGatewayManager(PushGateway pushGateway, CollectorRegistry registry, + ScheduledExecutorService scheduler, + long pushRate, String job, Map groupingKey, + ShutdownOperation shutdownOperation) { + this.pushGateway = pushGateway; + this.registry = registry; + this.job = job; + this.groupingKey = groupingKey; + this.shutdownOperation = (shutdownOperation != null) ? shutdownOperation : ShutdownOperation.NONE; + this.scheduler = scheduler; + this.scheduled = this.scheduler.scheduleAtFixedRate(this::push, 20, pushRate, TimeUnit.SECONDS); + } + + private void push() { + try { + this.pushGateway.pushAdd(this.registry, this.job, this.groupingKey); + } catch (Throwable ex) { + logger.warn("Unexpected exception thrown while pushing metrics to Prometheus Pushgateway", ex); + } + } + + private void delete() { + try { + this.pushGateway.delete(this.job, this.groupingKey); + } catch (Throwable ex) { + logger.warn("Unexpected exception thrown while deleting metrics from Prometheus Pushgateway", ex); + } + } + + /** + * Shutdown the manager, running any {@link ShutdownOperation}. + */ + public void shutdown() { + shutdown(this.shutdownOperation); + } + + private void shutdown(ShutdownOperation shutdownOperation) { + this.scheduled.cancel(false); + switch (shutdownOperation) { + case PUSH: + push(); + break; + case DELETE: + delete(); + break; + } + } + + /** + * The operation that should be performed on shutdown. + */ + public enum ShutdownOperation { + + /** + * Don't perform any shutdown operation. + */ + NONE, + + /** + * Perform a 'push' before shutdown. + */ + PUSH, + + /** + * Perform a 'delete' before shutdown. + */ + DELETE + + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/queue/TaskQueue.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/queue/TaskQueue.java new file mode 100644 index 0000000000..6b309aa9e0 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/queue/TaskQueue.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.queue; + +import com.dtstack.taier.metrics.executor.EagerEngineExecutor; +import lombok.NonNull; + +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * @see EagerEngineExecutor io 密集型,优先创建核心线程池,如果pooSize = maximunPoolSzie, 才会将Runnable 加入到对列中 + * @author xingyi + * @date 2025/9/17 + */ +public class TaskQueue extends VariableLinkedBlockingQueue { + + private static final long serialVersionUID = -1L; + + private transient EagerEngineExecutor executor; + + public TaskQueue(int queueCapacity) { + super(queueCapacity); + } + + public void setExecutor(EagerEngineExecutor exec) { + executor = exec; + } + + @Override + public boolean offer(@NonNull Runnable runnable) { + if (executor == null) { + throw new RejectedExecutionException("The task queue does not have executor."); + } + if (executor.getPoolSize() == executor.getMaximumPoolSize()) { + return super.offer(runnable); + } + // have free worker. put task into queue to let the worker deal with task. + if (executor.getSubmittedTaskCount() <= executor.getPoolSize()) { + return super.offer(runnable); + } + // return false to let executor create new worker. + if (executor.getPoolSize() < executor.getMaximumPoolSize()) { + return false; + } + // currentPoolThreadSize >= max + return super.offer(runnable); + } + + /** + * Force offer task + * + * @param o task + * @param timeout timeout + * @param unit unit + * @return offer success or not + * @throws RejectedExecutionException if executor is terminated. + * @throws InterruptedException if interrupted while waiting. + */ + public boolean force(Runnable o, long timeout, TimeUnit unit) throws InterruptedException { + if (executor.isShutdown()) { + throw new RejectedExecutionException("Executor is shutdown."); + } + return super.offer(o, timeout, unit); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/queue/VariableLinkedBlockingQueue.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/queue/VariableLinkedBlockingQueue.java new file mode 100644 index 0000000000..9e4bfe6494 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/queue/VariableLinkedBlockingQueue.java @@ -0,0 +1,1130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.queue; + +/** + * @author xingyi + * @date 2025/9/17 + */ + +import java.util.AbstractQueue; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * A clone of {@linkplain java.util.concurrent.LinkedBlockingQueue} + * with the addition of a {@link #setCapacity(int)} method, allowing us to + * change the capacity of the queue while it is in use.

+ * + * The documentation for LinkedBlockingQueue follows...

+ * + * An optionally-bounded {@linkplain BlockingQueue blocking queue} based on + * linked nodes. + * This queue orders elements FIFO (first-in-first-out). + * The head of the queue is that element that has been on the + * queue the longest time. + * The tail of the queue is that element that has been on the + * queue the shortest time. New elements + * are inserted at the tail of the queue, and the queue retrieval + * operations obtain elements at the head of the queue. + * Linked queues typically have higher throughput than array-based queues but + * less predictable performance in most concurrent applications. + * + *

The optional capacity bound constructor argument serves as a + * way to prevent excessive queue expansion. The capacity, if unspecified, + * is equal to {@link Integer#MAX_VALUE}. Linked nodes are + * dynamically created upon each insertion unless this would bring the + * queue above capacity. + * + *

This class implements all of the optional methods + * of the {@link Collection} and {@link Iterator} interfaces. + * + *

This class is a member of the + * + * Java Collections Framework. + * + * @since 1.5 + * @author Doug Lea + * @param the type of elements held in this collection + * + **/ +public class VariableLinkedBlockingQueue extends AbstractQueue + implements + BlockingQueue, + java.io.Serializable { + + private static final long serialVersionUID = -6903933977591709194L; + + /* + * A variant of the "two lock queue" algorithm. The putLock gates entry to put (and offer), and has an associated + * condition for waiting puts. Similarly for the takeLock. The "count" field that they both rely on is maintained as + * an atomic to avoid needing to get both locks in most cases. Also, to minimize need for puts to get takeLock and + * vice-versa, cascading notifies are used. When a put notices that it has enabled at least one take, it signals + * taker. That taker in turn signals others if more items have been entered since the signal. And symmetrically for + * takes signalling puts. Operations such as remove(Object) and iterators acquire both locks. + * + * Visibility between writers and readers is provided as follows: + * + * Whenever an element is enqueued, the putLock is acquired and count updated. A subsequent reader guarantees + * visibility to the enqueued Node by either acquiring the putLock (via fullyLock) or by acquiring the takeLock, and + * then reading n = count.get(); this gives visibility to the first n items. + * + * To implement weakly consistent iterators, it appears we need to keep all Nodes GC-reachable from a predecessor + * dequeued Node. That would cause two problems: - allow a rogue Iterator to cause unbounded memory retention - + * cause cross-generational linking of old Nodes to new Nodes if a Node was tenured while live, which generational + * GCs have a hard time dealing with, causing repeated major collections. However, only non-deleted Nodes need to be + * reachable from dequeued Nodes, and reachability does not necessarily have to be of the kind understood by the GC. + * We use the trick of linking a Node that has just been dequeued to itself. Such a self-link implicitly means to + * advance to head.next. + */ + + /** + * Linked list node class + */ + static class Node { + + E item; + + /** + * One of: + * - the real successor Node + * - this Node, meaning the successor is head.next + * - null, meaning there is no successor (this is the last node) + */ + Node next; + + Node(E x) { + item = x; + } + } + + /** The capacity bound, or Integer.MAX_VALUE if none */ + private int capacity; + + /** Current number of elements */ + private final AtomicInteger count = new AtomicInteger(); + + /** + * Head of linked list. + * Invariant: head.item == null + */ + transient Node head; + + /** + * Tail of linked list. + * Invariant: last.next == null + */ + private transient Node last; + + /** Lock held by take, poll, etc */ + private final ReentrantLock takeLock = new ReentrantLock(); + + /** Wait queue for waiting takes */ + private final Condition notEmpty = takeLock.newCondition(); + + /** Lock held by put, offer, etc */ + private final ReentrantLock putLock = new ReentrantLock(); + + /** Wait queue for waiting puts */ + private final Condition notFull = putLock.newCondition(); + + /** + * Signals a waiting take. Called only from put/offer (which do not + * otherwise ordinarily lock takeLock.) + */ + private void signalNotEmpty() { + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + notEmpty.signal(); + } finally { + takeLock.unlock(); + } + } + + /** + * Signals a waiting put. Called only from take/poll. + */ + private void signalNotFull() { + final ReentrantLock putLock = this.putLock; + putLock.lock(); + try { + notFull.signal(); + } finally { + putLock.unlock(); + } + } + + /** + * Links node at end of queue. + * + * @param node the node + */ + private void enqueue(Node node) { + // assert putLock.isHeldByCurrentThread(); + // assert last.next == null; + last = last.next = node; + } + + /** + * Removes a node from head of queue. + * + * @return the node + */ + private E dequeue() { + // assert takeLock.isHeldByCurrentThread(); + // assert head.item == null; + Node h = head; + Node first = h.next; + h.next = h; // help GC + head = first; + E x = first.item; + first.item = null; + return x; + } + + /** + * Locks to prevent both puts and takes. + */ + void fullyLock() { + putLock.lock(); + takeLock.lock(); + } + + /** + * Unlocks to allow both puts and takes. + */ + void fullyUnlock() { + takeLock.unlock(); + putLock.unlock(); + } + + // /** + // * Tells whether both locks are held by current thread. + // */ + // boolean isFullyLocked() { + // return (putLock.isHeldByCurrentThread() && + // takeLock.isHeldByCurrentThread()); + // } + + /** + * Creates a {@code VariableLinkedBlockingQueue} with a capacity of + * {@link Integer#MAX_VALUE}. + */ + public VariableLinkedBlockingQueue() { + this(Integer.MAX_VALUE); + } + + /** + * Creates a {@code VariableLinkedBlockingQueue} with the given (fixed) capacity. + * + * @param capacity the capacity of this queue + * @throws IllegalArgumentException if {@code capacity} is not greater + * than zero + */ + public VariableLinkedBlockingQueue(int capacity) { + if (capacity <= 0) { + throw new IllegalArgumentException(); + } + this.capacity = capacity; + last = head = new Node(null); + } + + /** + * Creates a {@code VariableLinkedBlockingQueue} with a capacity of + * {@link Integer#MAX_VALUE}, initially containing the elements of the + * given collection, + * added in traversal order of the collection's iterator. + * + * @param c the collection of elements to initially contain + * @throws NullPointerException if the specified collection or any + * of its elements are null + */ + public VariableLinkedBlockingQueue(Collection c) { + this(Integer.MAX_VALUE); + final ReentrantLock putLock = this.putLock; + putLock.lock(); // Never contended, but necessary for visibility + try { + int n = 0; + for (E e : c) { + if (e == null) { + throw new NullPointerException(); + } + if (n == capacity) { + throw new IllegalStateException("Queue full"); + } + enqueue(new Node(e)); + ++n; + } + count.set(n); + } finally { + putLock.unlock(); + } + } + + // this doc comment is overridden to remove the reference to collections + // greater in size than Integer.MAX_VALUE + /** + * Returns the number of elements in this queue. + * + * @return the number of elements in this queue + */ + @Override + public int size() { + return count.get(); + } + + /** + * Set a new capacity for the queue. Increasing the capacity can + * cause any waiting {@link #put(Object)} invocations to succeed if the new + * capacity is larger than the queue. + * @param capacity the new capacity for the queue + */ + public void setCapacity(int capacity) { + final int oldCapacity = this.capacity; + this.capacity = capacity; + final int size = count.get(); + if (capacity > size && size >= oldCapacity) { + signalNotFull(); + } + } + + // this doc comment is a modified copy of the inherited doc comment, + // without the reference to unlimited queues. + /** + * Returns the number of additional elements that this queue can ideally + * (in the absence of memory or resource constraints) accept without + * blocking. This is always equal to the initial capacity of this queue + * less the current {@code size} of this queue. + * + *

Note that you cannot always tell if an attempt to insert + * an element will succeed by inspecting {@code remainingCapacity} + * because it may be the case that another thread is about to + * insert or remove an element. + */ + @Override + public int remainingCapacity() { + return capacity - count.get(); + } + + /** + * Inserts the specified element at the tail of this queue, waiting if + * necessary for space to become available. + * + * @throws InterruptedException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + */ + @Override + public void put(E e) throws InterruptedException { + if (e == null) { + throw new NullPointerException(); + } + // Note: convention in all put/take/etc is to preset local var + // holding count negative to indicate failure unless set. + int c = -1; + Node node = new Node(e); + final ReentrantLock putLock = this.putLock; + final AtomicInteger count = this.count; + putLock.lockInterruptibly(); + try { + /* + * Note that count is used in wait guard even though it is not protected by lock. This works because count + * can only decrease at this point (all other puts are shut out by lock), and we (or some other waiting put) + * are signalled if it ever changes from capacity. Similarly for all other uses of count in other wait + * guards. + */ + while (count.get() >= capacity) { + notFull.await(); + } + enqueue(node); + c = count.getAndIncrement(); + if (c + 1 < capacity) { + notFull.signal(); + } + } finally { + putLock.unlock(); + } + if (c == 0) { + signalNotEmpty(); + } + } + + /** + * Inserts the specified element at the tail of this queue, waiting if + * necessary up to the specified wait time for space to become available. + * + * @return {@code true} if successful, or {@code false} if + * the specified waiting time elapses before space is available + * @throws InterruptedException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + */ + @Override + public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { + + if (e == null) { + throw new NullPointerException(); + } + long nanos = unit.toNanos(timeout); + int c = -1; + final ReentrantLock putLock = this.putLock; + final AtomicInteger count = this.count; + putLock.lockInterruptibly(); + try { + while (count.get() >= capacity) { + if (nanos <= 0) { + return false; + } + nanos = notFull.awaitNanos(nanos); + } + enqueue(new Node(e)); + c = count.getAndIncrement(); + if (c + 1 < capacity) { + notFull.signal(); + } + } finally { + putLock.unlock(); + } + if (c == 0) { + signalNotEmpty(); + } + return true; + } + + /** + * Inserts the specified element at the tail of this queue if it is + * possible to do so immediately without exceeding the queue's capacity, + * returning {@code true} upon success and {@code false} if this queue + * is full. + * When using a capacity-restricted queue, this method is generally + * preferable to method {@link BlockingQueue#add add}, which can fail to + * insert an element only by throwing an exception. + * + * @throws NullPointerException if the specified element is null + */ + @Override + public boolean offer(E e) { + if (e == null) { + throw new NullPointerException(); + } + final AtomicInteger count = this.count; + if (count.get() >= capacity) { + return false; + } + int c = -1; + Node node = new Node(e); + final ReentrantLock putLock = this.putLock; + putLock.lock(); + try { + if (count.get() < capacity) { + enqueue(node); + c = count.getAndIncrement(); + if (c + 1 < capacity) { + notFull.signal(); + } + } + } finally { + putLock.unlock(); + } + if (c == 0) { + signalNotEmpty(); + } + return c >= 0; + } + + @Override + public E take() throws InterruptedException { + E x; + int c = -1; + final AtomicInteger count = this.count; + final ReentrantLock takeLock = this.takeLock; + takeLock.lockInterruptibly(); + try { + while (count.get() == 0) { + notEmpty.await(); + } + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) { + notEmpty.signal(); + } + } finally { + takeLock.unlock(); + } + if (c >= capacity) { + signalNotFull(); + } + return x; + } + + @Override + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + E x = null; + int c = -1; + long nanos = unit.toNanos(timeout); + final AtomicInteger count = this.count; + final ReentrantLock takeLock = this.takeLock; + takeLock.lockInterruptibly(); + try { + while (count.get() == 0) { + if (nanos <= 0) { + return null; + } + nanos = notEmpty.awaitNanos(nanos); + } + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) { + notEmpty.signal(); + } + } finally { + takeLock.unlock(); + } + if (c >= capacity) { + signalNotFull(); + } + return x; + } + + @Override + public E poll() { + final AtomicInteger count = this.count; + if (count.get() == 0) { + return null; + } + E x = null; + int c = -1; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + if (count.get() > 0) { + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) { + notEmpty.signal(); + } + } + } finally { + takeLock.unlock(); + } + if (c >= capacity) { + signalNotFull(); + } + return x; + } + + @Override + public E peek() { + if (count.get() == 0) { + return null; + } + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + Node first = head.next; + if (first == null) { + return null; + } else { + return first.item; + } + } finally { + takeLock.unlock(); + } + } + + /** + * Unlinks interior Node p with predecessor trail. + */ + void unlink(Node p, Node trail) { + // assert isFullyLocked(); + // p.next is not changed, to allow iterators that are + // traversing p to maintain their weak-consistency guarantee. + p.item = null; + trail.next = p.next; + if (last == p) { + last = trail; + } + if (count.getAndDecrement() >= capacity) { + notFull.signal(); + } + } + + /** + * Removes a single instance of the specified element from this queue, + * if it is present. More formally, removes an element {@code e} such + * that {@code o.equals(e)}, if this queue contains one or more such + * elements. + * Returns {@code true} if this queue contained the specified element + * (or equivalently, if this queue changed as a result of the call). + * + * @param o element to be removed from this queue, if present + * @return {@code true} if this queue changed as a result of the call + */ + @Override + public boolean remove(Object o) { + if (o == null) { + return false; + } + fullyLock(); + try { + for (Node trail = head, p = trail.next; p != null; trail = p, p = p.next) { + if (o.equals(p.item)) { + unlink(p, trail); + return true; + } + } + return false; + } finally { + fullyUnlock(); + } + } + + /** + * Returns {@code true} if this queue contains the specified element. + * More formally, returns {@code true} if and only if this queue contains + * at least one element {@code e} such that {@code o.equals(e)}. + * + * @param o object to be checked for containment in this queue + * @return {@code true} if this queue contains the specified element + */ + @Override + public boolean contains(Object o) { + if (o == null) { + return false; + } + fullyLock(); + try { + for (Node p = head.next; p != null; p = p.next) { + if (o.equals(p.item)) { + return true; + } + } + return false; + } finally { + fullyUnlock(); + } + } + + /** + * Returns an array containing all of the elements in this queue, in + * proper sequence. + * + *

The returned array will be "safe" in that no references to it are + * maintained by this queue. (In other words, this method must allocate + * a new array). The caller is thus free to modify the returned array. + * + *

This method acts as bridge between array-based and collection-based + * APIs. + * + * @return an array containing all of the elements in this queue + */ + @Override + public Object[] toArray() { + fullyLock(); + try { + int size = count.get(); + Object[] a = new Object[size]; + int k = 0; + for (Node p = head.next; p != null; p = p.next) { + a[k++] = p.item; + } + return a; + } finally { + fullyUnlock(); + } + } + + /** + * Returns an array containing all of the elements in this queue, in + * proper sequence; the runtime type of the returned array is that of + * the specified array. If the queue fits in the specified array, it + * is returned therein. Otherwise, a new array is allocated with the + * runtime type of the specified array and the size of this queue. + * + *

If this queue fits in the specified array with room to spare + * (i.e., the array has more elements than this queue), the element in + * the array immediately following the end of the queue is set to + * {@code null}. + * + *

Like the {@link #toArray()} method, this method acts as bridge between + * array-based and collection-based APIs. Further, this method allows + * precise control over the runtime type of the output array, and may, + * under certain circumstances, be used to save allocation costs. + * + *

Suppose {@code x} is a queue known to contain only strings. + * The following code can be used to dump the queue into a newly + * allocated array of {@code String}: + * + *

 {@code String[] y = x.toArray(new String[0]);}
+ * + * Note that {@code toArray(new Object[0])} is identical in function to + * {@code toArray()}. + * + * @param a the array into which the elements of the queue are to + * be stored, if it is big enough; otherwise, a new array of the + * same runtime type is allocated for this purpose + * @return an array containing all of the elements in this queue + * @throws ArrayStoreException if the runtime type of the specified array + * is not a supertype of the runtime type of every element in + * this queue + * @throws NullPointerException if the specified array is null + */ + @Override + @SuppressWarnings("unchecked") + public T[] toArray(T[] a) { + fullyLock(); + try { + int size = count.get(); + if (a.length < size) { + a = (T[]) java.lang.reflect.Array.newInstance(a.getClass().getComponentType(), size); + } + + int k = 0; + for (Node p = head.next; p != null; p = p.next) { + a[k++] = (T) p.item; + } + if (a.length > k) { + a[k] = null; + } + return a; + } finally { + fullyUnlock(); + } + } + + @Override + public String toString() { + fullyLock(); + try { + Node p = head.next; + if (p == null) { + return "[]"; + } + + StringBuilder sb = new StringBuilder(); + sb.append('['); + for (;;) { + E e = p.item; + sb.append(e == this ? "(this Collection)" : e); + p = p.next; + if (p == null) { + return sb.append(']').toString(); + } + sb.append(',').append(' '); + } + } finally { + fullyUnlock(); + } + } + + /** + * Atomically removes all of the elements from this queue. + * The queue will be empty after this call returns. + */ + @Override + public void clear() { + fullyLock(); + try { + for (Node p, h = head; (p = h.next) != null; h = p) { + h.next = h; + p.item = null; + } + head = last; + // assert head.item == null && head.next == null; + if (count.getAndSet(0) >= capacity) { + notFull.signal(); + } + } finally { + fullyUnlock(); + } + } + + /** + * @throws UnsupportedOperationException {@inheritDoc} + * @throws ClassCastException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + * @throws IllegalArgumentException {@inheritDoc} + */ + @Override + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + /** + * @throws UnsupportedOperationException {@inheritDoc} + * @throws ClassCastException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + * @throws IllegalArgumentException {@inheritDoc} + */ + @Override + public int drainTo(Collection c, int maxElements) { + if (c == null) { + throw new NullPointerException(); + } + if (c == this) { + throw new IllegalArgumentException(); + } + if (maxElements <= 0) { + return 0; + } + boolean signalNotFull = false; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + int n = Math.min(maxElements, count.get()); + // count.get provides visibility to first n Nodes + Node h = head; + int i = 0; + try { + while (i < n) { + Node p = h.next; + c.add(p.item); + p.item = null; + h.next = h; + h = p; + ++i; + } + return n; + } finally { + // Restore invariants even if c.add() threw + if (i > 0) { + // assert h.item == null; + head = h; + signalNotFull = (count.getAndAdd(-i) >= capacity); + } + } + } finally { + takeLock.unlock(); + if (signalNotFull) { + signalNotFull(); + } + } + } + + /** + * Returns an iterator over the elements in this queue in proper sequence. + * The elements will be returned in order from first (head) to last (tail). + * + *

The returned iterator is + * weakly consistent. + * + * @return an iterator over the elements in this queue in proper sequence + */ + @Override + public Iterator iterator() { + return new Itr(); + } + + private class Itr implements Iterator { + /* + * Basic weakly-consistent iterator. At all times hold the next item to hand out so that if hasNext() reports + * true, we will still have it to return even if lost race with a take etc. + */ + + private Node current; + private Node lastRet; + private E currentElement; + + Itr() { + fullyLock(); + try { + current = head.next; + if (current != null) { + currentElement = current.item; + } + } finally { + fullyUnlock(); + } + } + + @Override + public boolean hasNext() { + return current != null; + } + + /** + * Returns the next live successor of p, or null if no such. + * + * Unlike other traversal methods, iterators need to handle both: + * - dequeued nodes (p.next == p) + * - (possibly multiple) interior removed nodes (p.item == null) + */ + private Node nextNode(Node p) { + for (;;) { + Node s = p.next; + if (s == p) { + return head.next; + } + if (s == null || s.item != null) { + return s; + } + p = s; + } + } + + @Override + public E next() { + fullyLock(); + try { + if (current == null) { + throw new NoSuchElementException(); + } + E x = currentElement; + lastRet = current; + current = nextNode(current); + currentElement = (current == null) ? null : current.item; + return x; + } finally { + fullyUnlock(); + } + } + + @Override + public void remove() { + if (lastRet == null) { + throw new IllegalStateException(); + } + fullyLock(); + try { + Node node = lastRet; + lastRet = null; + for (Node trail = head, p = trail.next; p != null; trail = p, p = p.next) { + if (p == node) { + unlink(p, trail); + break; + } + } + } finally { + fullyUnlock(); + } + } + } + + /** A customized variant of Spliterators.IteratorSpliterator */ + static final class LBQSpliterator implements Spliterator { + + static final int MAX_BATCH = 1 << 25; // max batch array size; + final VariableLinkedBlockingQueue queue; + Node current; // current node; null until initialized + int batch; // batch size for splits + boolean exhausted; // true when no more nodes + long est; // size estimate + LBQSpliterator(VariableLinkedBlockingQueue queue) { + this.queue = queue; + this.est = queue.size(); + } + + @Override + public long estimateSize() { + return est; + } + + @Override + public Spliterator trySplit() { + Node h; + final VariableLinkedBlockingQueue q = this.queue; + int b = batch; + int n = (b <= 0) ? 1 : (b >= MAX_BATCH) ? MAX_BATCH : b + 1; + if (!exhausted && + ((h = current) != null || (h = q.head.next) != null) && + h.next != null) { + Object[] a = new Object[n]; + int i = 0; + Node p = current; + q.fullyLock(); + try { + if (p != null || (p = q.head.next) != null) { + do { + if ((a[i] = p.item) != null) { + ++i; + } + } while ((p = p.next) != null && i < n); + } + } finally { + q.fullyUnlock(); + } + if ((current = p) == null) { + est = 0L; + exhausted = true; + } else if ((est -= i) < 0L) { + est = 0L; + } + if (i > 0) { + batch = i; + return Spliterators.spliterator(a, 0, i, Spliterator.ORDERED | Spliterator.NONNULL | + Spliterator.CONCURRENT); + } + } + return null; + } + + @Override + public void forEachRemaining(Consumer action) { + if (action == null) { + throw new NullPointerException(); + } + final VariableLinkedBlockingQueue q = this.queue; + if (!exhausted) { + exhausted = true; + Node p = current; + do { + E e = null; + q.fullyLock(); + try { + if (p == null) { + p = q.head.next; + } + while (p != null) { + e = p.item; + p = p.next; + if (e != null) { + break; + } + } + } finally { + q.fullyUnlock(); + } + if (e != null) { + action.accept(e); + } + } while (p != null); + } + } + + @Override + public boolean tryAdvance(Consumer action) { + if (action == null) { + throw new NullPointerException(); + } + final VariableLinkedBlockingQueue q = this.queue; + if (!exhausted) { + E e = null; + q.fullyLock(); + try { + if (current == null) { + current = q.head.next; + } + while (current != null) { + e = current.item; + current = current.next; + if (e != null) { + break; + } + } + } finally { + q.fullyUnlock(); + } + if (current == null) { + exhausted = true; + } + if (e != null) { + action.accept(e); + return true; + } + } + return false; + } + + @Override + public int characteristics() { + return Spliterator.ORDERED | Spliterator.NONNULL | + Spliterator.CONCURRENT; + } + } + + /** + * Returns a {@link Spliterator} over the elements in this queue. + * + *

The returned spliterator is + * weakly consistent. + * + *

The {@code Spliterator} reports {@link Spliterator#CONCURRENT}, + * {@link Spliterator#ORDERED}, and {@link Spliterator#NONNULL}. + * + * The {@code Spliterator} implements {@code trySplit} to permit limited + * parallelism. + * + * @return a {@code Spliterator} over the elements in this queue + * @since 1.8 + */ + @Override + public Spliterator spliterator() { + return new LBQSpliterator(this); + } + + /** + * Saves this queue to a stream (that is, serializes it). + * + * @param s the stream + * @throws java.io.IOException if an I/O error occurs + * @serialData The capacity is emitted (int), followed by all of + * its elements (each an {@code Object}) in the proper order, + * followed by a null + */ + private void writeObject(java.io.ObjectOutputStream s) throws java.io.IOException { + + fullyLock(); + try { + // Write out any hidden stuff, plus capacity + s.defaultWriteObject(); + + // Write out all elements in the proper order. + for (Node p = head.next; p != null; p = p.next) { + s.writeObject(p.item); + } + // Use trailing null as sentinel + s.writeObject(null); + } finally { + fullyUnlock(); + } + } + + /** + * Reconstitutes this queue from a stream (that is, deserializes it). + * @param s the stream + * @throws ClassNotFoundException if the class of a serialized object + * could not be found + * @throws java.io.IOException if an I/O error occurs + */ + private void readObject(java.io.ObjectInputStream s) throws java.io.IOException, ClassNotFoundException { + // Read in capacity, and any hidden stuff + s.defaultReadObject(); + + count.set(0); + last = head = new Node(null); + + // Read in all elements and place in queue + for (;;) { + @SuppressWarnings("unchecked") + E item = (E) s.readObject(); + if (item == null) { + break; + } + add(item); + } + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/registry/TpRegistry.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/registry/TpRegistry.java new file mode 100644 index 0000000000..4dceccb803 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/registry/TpRegistry.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.registry; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.exception.TpException; +import com.dtstack.taier.metrics.executor.EngineExecutor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; + +/** + * 可观测线程池注册中心,内存模式 + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +public class TpRegistry { + + /** + * Maintain all automatically registered and manually registered Executors. + */ + private static final Map EXECUTOR_REGISTRY = new ConcurrentHashMap<>(); + + /** + * Get all Executor names. + * + * @return all executor names + */ + public static Set getAllExecutorNames() { + return Collections.unmodifiableSet(EXECUTOR_REGISTRY.keySet()); + } + + /** + * Get all Executors. + * + * @return all Executors + */ + public static Map getAllExecutors() { + return EXECUTOR_REGISTRY; + } + + /** + * Unregister a executor. + * + * @param name thread pool name + * @return the managed DtpExecutor instance + */ + public static ExecutorWrapper unregisterExecutor(String name) { + ExecutorWrapper executorWrapper = getExecutorWrapper(name); + log.info("DynamicTp unregister executor: {}", executorWrapper); + return EXECUTOR_REGISTRY.remove(name); + } + + /** + * Get DtpExecutor by thread pool name. + * + * @param name thread pool name + * @return the managed DtpExecutor instance + */ + public static EngineExecutor getDtpExecutor(String name) { + val executorWrapper = getExecutorWrapper(name); + if (!executorWrapper.isDtpExecutor()) { + log.error("The specified executor is not a DtpExecutor, name: {}", name); + throw new TpException("The specified executor is not a DtpExecutor, name: " + name); + } + return (EngineExecutor) executorWrapper.getExecutor(); + } + + /** + * Get executor by thread pool name. + * + * @param name thread pool name + * @return the managed executor instance + */ + public static Executor getExecutor(String name) { + val executorWrapper = EXECUTOR_REGISTRY.get(name); + if (Objects.isNull(executorWrapper)) { + log.error("Cannot find a specified executor, name: {}", name); + throw new TpException("Cannot find a specified executor, name: " + name); + } + return executorWrapper.getExecutor(); + } + + /** + * Get ExecutorWrapper by thread pool name. + * + * @param name thread pool name + * @return the managed ExecutorWrapper instance + */ + public static ExecutorWrapper getExecutorWrapper(String name) { + ExecutorWrapper executorWrapper = EXECUTOR_REGISTRY.get(name); + if (Objects.isNull(executorWrapper)) { + log.error("Cannot find a specified executorWrapper, name: {}", name); + throw new TpException("Cannot find a specified executorWrapper, name: " + name); + } + return executorWrapper; + } + + /** + * Register executor. + * + * @param wrapper the newly created ExecutorWrapper instance + */ + public static void registerExecutor(ExecutorWrapper wrapper) { + wrapper.initialize(); + EXECUTOR_REGISTRY.putIfAbsent(wrapper.getThreadPoolName(), wrapper); + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/rejects/RejectHandlerGetter.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/rejects/RejectHandlerGetter.java new file mode 100644 index 0000000000..e271036e5c --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/rejects/RejectHandlerGetter.java @@ -0,0 +1,61 @@ + +package com.dtstack.taier.metrics.rejects; + +import com.dtstack.taier.metrics.collect.util.ExtensionServiceLoader; +import com.dtstack.taier.metrics.exception.TpException; +import lombok.extern.slf4j.Slf4j; + +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; + +import static com.dtstack.taier.metrics.collect.em.RejectedTypeEnum.ABORT_POLICY; +import static com.dtstack.taier.metrics.collect.em.RejectedTypeEnum.CALLER_RUNS_POLICY; +import static com.dtstack.taier.metrics.collect.em.RejectedTypeEnum.DISCARD_OLDEST_POLICY; +import static com.dtstack.taier.metrics.collect.em.RejectedTypeEnum.DISCARD_POLICY; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@Slf4j +public class RejectHandlerGetter { + + private RejectHandlerGetter() { + } + + public static RejectedExecutionHandler buildRejectedHandler(String name) { + if (Objects.equals(name, ABORT_POLICY.getName())) { + return new ThreadPoolExecutor.AbortPolicy(); + } else if (Objects.equals(name, CALLER_RUNS_POLICY.getName())) { + return new ThreadPoolExecutor.CallerRunsPolicy(); + } else if (Objects.equals(name, DISCARD_OLDEST_POLICY.getName())) { + return new ThreadPoolExecutor.DiscardOldestPolicy(); + } else if (Objects.equals(name, DISCARD_POLICY.getName())) { + return new ThreadPoolExecutor.DiscardPolicy(); + } + List loadedHandlers = ExtensionServiceLoader.get(RejectedExecutionHandler.class); + for (RejectedExecutionHandler handler : loadedHandlers) { + String handlerName = handler.getClass().getSimpleName(); + if (name.equalsIgnoreCase(handlerName)) { + return handler; + } + } + + log.error("Cannot find specified rejectedHandler {}", name); + throw new TpException("Cannot find specified rejectedHandler " + name); + } + + public static RejectedExecutionHandler getProxy(String name) { + return getProxy(buildRejectedHandler(name)); + } + + public static RejectedExecutionHandler getProxy(RejectedExecutionHandler handler) { + return (RejectedExecutionHandler) Proxy + .newProxyInstance(handler.getClass().getClassLoader(), + new Class[]{RejectedExecutionHandler.class}, + new RejectedInvocationHandler(handler)); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/rejects/RejectedInvocationHandler.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/rejects/RejectedInvocationHandler.java new file mode 100644 index 0000000000..ce4b9e9b70 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/rejects/RejectedInvocationHandler.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.rejects; + +import com.dtstack.taier.metrics.aware.AwareManager; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.concurrent.Executor; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class RejectedInvocationHandler implements InvocationHandler { + + private final Object target; + + public RejectedInvocationHandler(Object target) { + this.target = target; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + beforeReject((Runnable) args[0], (Executor) args[1]); + try { + return method.invoke(target, args); + } catch (InvocationTargetException ex) { + throw ex.getCause(); + } finally { + afterReject((Runnable) args[0], (Executor) args[1]); + } + } + + /** + * Do sth before reject. + * + * @param runnable the runnable + * @param executor ThreadPoolExecutor instance + */ + private void beforeReject(Runnable runnable, Executor executor) { + AwareManager.beforeReject(runnable, executor); + } + + /** + * Do sth after reject. + * + * @param runnable the runnable + * @param executor ThreadPoolExecutor instance + */ + private void afterReject(Runnable runnable, Executor executor) { + AwareManager.afterReject(runnable, executor); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/runnable/DtpRunnable.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/runnable/DtpRunnable.java new file mode 100644 index 0000000000..2a91c52888 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/runnable/DtpRunnable.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.runnable; + +import lombok.Getter; +import org.slf4j.MDC; + +/** + * DtpRunnable related + * + * @author yanhom + * @since 1.0.4 + */ +@Getter +public class DtpRunnable implements Runnable { + + private final Runnable originRunnable; + + private final Runnable runnable; + + private final String taskName; + + private final String traceId; + + public DtpRunnable(Runnable originRunnable, Runnable runnable, String taskName) { + this.originRunnable = originRunnable; + this.runnable = runnable; + this.taskName = taskName; + this.traceId = MDC.get("traceId"); + } + + @Override + public void run() { + runnable.run(); + } + +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/AbstractTimeoutTimerTask.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/AbstractTimeoutTimerTask.java new file mode 100644 index 0000000000..62ff2ccb6f --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/AbstractTimeoutTimerTask.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.ThreadPoolStatProvider; +import com.dtstack.taier.metrics.runnable.DtpRunnable; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Objects; + +/** + * AbstractTimeoutTimerTask related + * + * @author yanhom + * @since 1.1.4 + **/ +public abstract class AbstractTimeoutTimerTask implements TimerTask { + + protected final ExecutorWrapper executorWrapper; + + protected final Runnable runnable; + + protected AbstractTimeoutTimerTask(ExecutorWrapper executorWrapper, Runnable runnable) { + this.executorWrapper = executorWrapper; + this.runnable = runnable; + } + + @Override + public void run(Timeout timeout) throws Exception { + ThreadPoolStatProvider statProvider = executorWrapper.getThreadPoolStatProvider(); + if (Objects.isNull(statProvider)) { + return; + } + doRun(); + } + + protected Pair getTaskNameAndTraceId() { + String taskName = StringUtils.EMPTY; + String traceId = StringUtils.EMPTY; + if (runnable instanceof DtpRunnable) { + DtpRunnable dtpRunnable = (DtpRunnable) runnable; + taskName = dtpRunnable.getTaskName(); + traceId = dtpRunnable.getTraceId(); + } + return Pair.of(taskName, traceId); + } + + /** + * Do run. + */ + protected abstract void doRun(); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/HashedWheelTimer.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/HashedWheelTimer.java new file mode 100644 index 0000000000..66367d2234 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/HashedWheelTimer.java @@ -0,0 +1,824 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ClassUtils; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A {@link Timer} optimized for approximated I/O timeout scheduling. + * + *

Tick Duration

+ *

+ * As described with 'approximated', this timer does not execute the scheduled + * {@link TimerTask} on time. {@link HashedWheelTimer}, on every tick, will + * check if there are any {@link TimerTask}s behind the schedule and execute + * them. + *

+ * You can increase or decrease the accuracy of the execution timing by + * specifying smaller or larger tick duration in the constructor. In most + * network applications, I/O timeout does not need to be accurate. Therefore, + * the default tick duration is 100 milliseconds, and you will not need to try + * different configurations in most cases. + * + *

Ticks per Wheel (Wheel Size)

+ *

+ * {@link HashedWheelTimer} maintains a data structure called 'wheel'. + * To put simply, a wheel is a hash table of {@link TimerTask}s whose hash + * function is 'deadline of the task'. The default number of ticks per wheel + * (i.e. the size of the wheel) is 512. You could specify a larger value + * if you are going to schedule a lot of timeouts. + * + *

Do not create many instances.

+ *

+ * {@link HashedWheelTimer} creates a new thread whenever it is instantiated and + * started. Therefore, you should make sure to create only one instance and + * share it across your application. One of the common mistakes, that makes + * your application unresponsive, is to create a new instance for every connection. + * + *

Implementation Details

+ *

+ * {@link HashedWheelTimer} is based on + * George Varghese and + * Tony Lauck's paper, + * 'Hashed + * and Hierarchical Timing Wheels: data structures to efficiently implement a + * timer facility'. More comprehensive slides are located + * here. + *

+ * Copy from dubbo, see here for more details. + *

+ */ +@Slf4j +public class HashedWheelTimer implements Timer { + + /** + * may be in spi? + */ + public static final String NAME = "hased"; + + private static final AtomicInteger INSTANCE_COUNTER = new AtomicInteger(); + private static final AtomicBoolean WARNED_TOO_MANY_INSTANCES = new AtomicBoolean(); + private static final int INSTANCE_COUNT_LIMIT = 64; + private static final AtomicIntegerFieldUpdater WORKER_STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimer.class, "workerState"); + + private final Worker worker = new Worker(); + private final Thread workerThread; + + private static final int WORKER_STATE_INIT = 0; + private static final int WORKER_STATE_STARTED = 1; + private static final int WORKER_STATE_SHUTDOWN = 2; + + /** + * 0 - init, 1 - started, 2 - shut down + */ + @SuppressWarnings({"unused", "FieldMayBeFinal"}) + private volatile int workerState; + + private final long tickDuration; + private final HashedWheelBucket[] wheel; + private final int mask; + private final CountDownLatch startTimeInitialized = new CountDownLatch(1); + private final Queue timeouts = new LinkedBlockingQueue<>(); + private final Queue cancelledTimeouts = new LinkedBlockingQueue<>(); + private final AtomicLong pendingTimeouts = new AtomicLong(0); + private final long maxPendingTimeouts; + + private volatile long startTime; + + /** + * Creates a new timer with the default thread factory + * ({@link Executors#defaultThreadFactory()}), default tick duration, and + * default number of ticks per wheel. + */ + public HashedWheelTimer() { + this(Executors.defaultThreadFactory()); + } + + /** + * Creates a new timer with the default thread factory + * ({@link Executors#defaultThreadFactory()}) and default number of ticks + * per wheel. + * + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @throws NullPointerException if {@code unit} is {@code null} + * @throws IllegalArgumentException if {@code tickDuration} is <= 0 + */ + public HashedWheelTimer(long tickDuration, TimeUnit unit) { + this(Executors.defaultThreadFactory(), tickDuration, unit); + } + + /** + * Creates a new timer with the default thread factory + * ({@link Executors#defaultThreadFactory()}). + * + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @param ticksPerWheel the size of the wheel + * @throws NullPointerException if {@code unit} is {@code null} + * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 + */ + public HashedWheelTimer(long tickDuration, TimeUnit unit, int ticksPerWheel) { + this(Executors.defaultThreadFactory(), tickDuration, unit, ticksPerWheel); + } + + /** + * Creates a new timer with the default tick duration and default number of + * ticks per wheel. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @throws NullPointerException if {@code threadFactory} is {@code null} + */ + public HashedWheelTimer(ThreadFactory threadFactory) { + this(threadFactory, 100, TimeUnit.MILLISECONDS); + } + + /** + * Creates a new timer with the default number of ticks per wheel. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} + * @throws IllegalArgumentException if {@code tickDuration} is <= 0 + */ + public HashedWheelTimer( + ThreadFactory threadFactory, long tickDuration, TimeUnit unit) { + this(threadFactory, tickDuration, unit, 512); + } + + /** + * Creates a new timer. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @param ticksPerWheel the size of the wheel + * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} + * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 + */ + public HashedWheelTimer( + ThreadFactory threadFactory, + long tickDuration, TimeUnit unit, int ticksPerWheel) { + this(threadFactory, tickDuration, unit, ticksPerWheel, -1); + } + + /** + * Creates a new timer. + * + * @param threadFactory a {@link ThreadFactory} that creates a + * background {@link Thread} which is dedicated to + * {@link TimerTask} execution. + * @param tickDuration the duration between tick + * @param unit the time unit of the {@code tickDuration} + * @param ticksPerWheel the size of the wheel + * @param maxPendingTimeouts The maximum number of pending timeouts after which call to + * {@code newTimeout} will result in + * {@link RejectedExecutionException} + * being thrown. No maximum pending timeouts limit is assumed if + * this value is 0 or negative. + * @throws NullPointerException if either of {@code threadFactory} and {@code unit} is {@code null} + * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0 + */ + public HashedWheelTimer( + ThreadFactory threadFactory, + long tickDuration, + TimeUnit unit, + int ticksPerWheel, + long maxPendingTimeouts) { + + if (threadFactory == null) { + throw new NullPointerException("threadFactory"); + } + if (unit == null) { + throw new NullPointerException("unit"); + } + if (tickDuration <= 0) { + throw new IllegalArgumentException("tickDuration must be greater than 0: " + tickDuration); + } + if (ticksPerWheel <= 0) { + throw new IllegalArgumentException("ticksPerWheel must be greater than 0: " + ticksPerWheel); + } + + // Normalize ticksPerWheel to power of two and initialize the wheel. + wheel = createWheel(ticksPerWheel); + mask = wheel.length - 1; + + // Convert tickDuration to nanos. + this.tickDuration = unit.toNanos(tickDuration); + + // Prevent overflow. + if (this.tickDuration >= Long.MAX_VALUE / wheel.length) { + throw new IllegalArgumentException(String.format( + "tickDuration: %d (expected: 0 < tickDuration in nanos < %d", + tickDuration, Long.MAX_VALUE / wheel.length)); + } + workerThread = threadFactory.newThread(worker); + + this.maxPendingTimeouts = maxPendingTimeouts; + + if (INSTANCE_COUNTER.incrementAndGet() > INSTANCE_COUNT_LIMIT && + WARNED_TOO_MANY_INSTANCES.compareAndSet(false, true)) { + reportTooManyInstances(); + } + } + + @Override + protected void finalize() throws Throwable { + try { + super.finalize(); + } finally { + // This object is going to be GCed and it is assumed the ship has sailed to do a proper shutdown. If + // we have not yet shutdown then we want to make sure we decrement the active instance count. + if (WORKER_STATE_UPDATER.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) { + INSTANCE_COUNTER.decrementAndGet(); + } + } + } + + private static HashedWheelBucket[] createWheel(int ticksPerWheel) { + if (ticksPerWheel <= 0) { + throw new IllegalArgumentException( + "ticksPerWheel must be greater than 0: " + ticksPerWheel); + } + if (ticksPerWheel > 1073741824) { + throw new IllegalArgumentException( + "ticksPerWheel may not be greater than 2^30: " + ticksPerWheel); + } + + ticksPerWheel = normalizeTicksPerWheel(ticksPerWheel); + HashedWheelBucket[] wheel = new HashedWheelBucket[ticksPerWheel]; + for (int i = 0; i < wheel.length; i++) { + wheel[i] = new HashedWheelBucket(); + } + return wheel; + } + + private static int normalizeTicksPerWheel(int ticksPerWheel) { + int normalizedTicksPerWheel = ticksPerWheel - 1; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 1; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 2; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 4; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 8; + normalizedTicksPerWheel |= normalizedTicksPerWheel >>> 16; + return normalizedTicksPerWheel + 1; + } + + /** + * Starts the background thread explicitly. The background thread will + * start automatically on demand even if you did not call this method. + * + * @throws IllegalStateException if this timer has been + * {@linkplain #stop() stopped} already + */ + public void start() { + switch (WORKER_STATE_UPDATER.get(this)) { + case WORKER_STATE_INIT: + if (WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_INIT, WORKER_STATE_STARTED)) { + workerThread.start(); + } + break; + case WORKER_STATE_STARTED: + break; + case WORKER_STATE_SHUTDOWN: + throw new IllegalStateException("cannot be started once stopped"); + default: + throw new Error("Invalid WorkerState"); + } + + // Wait until the startTime is initialized by the worker. + while (startTime == 0) { + try { + startTimeInitialized.await(); + } catch (InterruptedException ignore) { + // Ignore - it will be ready very soon. + } + } + } + + @Override + public Set stop() { + if (Thread.currentThread() == workerThread) { + throw new IllegalStateException( + HashedWheelTimer.class.getSimpleName() + + ".stop() cannot be called from " + + TimerTask.class.getSimpleName()); + } + + if (!WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_STARTED, WORKER_STATE_SHUTDOWN)) { + // workerState can be 0 or 2 at this moment - let it always be 2. + if (WORKER_STATE_UPDATER.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) { + INSTANCE_COUNTER.decrementAndGet(); + } + + return Collections.emptySet(); + } + + try { + boolean interrupted = false; + while (workerThread.isAlive()) { + workerThread.interrupt(); + try { + workerThread.join(100); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } finally { + INSTANCE_COUNTER.decrementAndGet(); + } + return worker.unprocessedTimeouts(); + } + + @Override + public boolean isStop() { + return WORKER_STATE_SHUTDOWN == WORKER_STATE_UPDATER.get(this); + } + + @Override + public Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) { + if (task == null) { + throw new NullPointerException("task"); + } + if (unit == null) { + throw new NullPointerException("unit"); + } + + long pendingTimeoutsCount = pendingTimeouts.incrementAndGet(); + + if (maxPendingTimeouts > 0 && pendingTimeoutsCount > maxPendingTimeouts) { + pendingTimeouts.decrementAndGet(); + throw new RejectedExecutionException("Number of pending timeouts (" + + pendingTimeoutsCount + ") is greater than or equal to maximum allowed pending " + + "timeouts (" + maxPendingTimeouts + ")"); + } + + start(); + + // Add the timeout to the timeout queue which will be processed on the next tick. + // During processing all the queued HashedWheelTimeouts will be added to the correct HashedWheelBucket. + long deadline = System.nanoTime() + unit.toNanos(delay) - startTime; + + // Guard against overflow. + if (delay > 0 && deadline < 0) { + deadline = Long.MAX_VALUE; + } + HashedWheelTimeout timeout = new HashedWheelTimeout(this, task, deadline); + timeouts.add(timeout); + return timeout; + } + + /** + * @return the number of pending timeouts of this {@link Timer}. + */ + public long pendingTimeouts() { + return pendingTimeouts.get(); + } + + private static void reportTooManyInstances() { + String resourceType = ClassUtils.getSimpleName((HashedWheelTimer.class)); + log.error("You are creating too many " + resourceType + " instances. " + + resourceType + " is a shared resource that must be reused across the JVM," + + "so that only a few instances are created."); + } + + private final class Worker implements Runnable { + + private final Set unprocessedTimeouts = new HashSet<>(); + + private long tick; + + @Override + public void run() { + // Initialize the startTime. + startTime = System.nanoTime(); + if (startTime == 0) { + // We use 0 as an indicator for the uninitialized value here, so make sure it's not 0 when initialized. + startTime = 1; + } + + // Notify the other threads waiting for the initialization at start(). + startTimeInitialized.countDown(); + + do { + final long deadline = waitForNextTick(); + if (deadline > 0) { + int idx = (int) (tick & mask); + processCancelledTasks(); + HashedWheelBucket bucket = + wheel[idx]; + transferTimeoutsToBuckets(); + bucket.expireTimeouts(deadline); + tick++; + } + } while (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_STARTED); + + // Fill the unprocessedTimeouts so we can return them from stop() method. + for (HashedWheelBucket bucket : wheel) { + bucket.clearTimeouts(unprocessedTimeouts); + } + for (;;) { + HashedWheelTimeout timeout = timeouts.poll(); + if (timeout == null) { + break; + } + if (!timeout.isCancelled()) { + unprocessedTimeouts.add(timeout); + } + } + processCancelledTasks(); + } + + private void transferTimeoutsToBuckets() { + // transfer only max. 100000 timeouts per tick to prevent a thread to stale the workerThread when it just + // adds new timeouts in a loop. + for (int i = 0; i < 100000; i++) { + HashedWheelTimeout timeout = timeouts.poll(); + if (timeout == null) { + // all processed + break; + } + if (timeout.state() == HashedWheelTimeout.ST_CANCELLED) { + // Was cancelled in the meantime. + continue; + } + + long calculated = timeout.deadline / tickDuration; + timeout.remainingRounds = (calculated - tick) / wheel.length; + + // Ensure we don't schedule for past. + final long ticks = Math.max(calculated, tick); + int stopIndex = (int) (ticks & mask); + + HashedWheelBucket bucket = wheel[stopIndex]; + bucket.addTimeout(timeout); + } + } + + private void processCancelledTasks() { + for (;;) { + HashedWheelTimeout timeout = cancelledTimeouts.poll(); + if (timeout == null) { + // all processed + break; + } + try { + timeout.remove(); + } catch (Throwable t) { + if (log.isWarnEnabled()) { + log.warn("An exception was thrown while process a cancellation task", t); + } + } + } + } + + /** + * calculate goal nanoTime from startTime and current tick number, + * then wait until that goal has been reached. + * + * @return Long.MIN_VALUE if received a shutdown request, + * current time otherwise (with Long.MIN_VALUE changed by +1) + */ + private long waitForNextTick() { + // 34007 753416 + // 100 000000 + // 2876 101333 + // 23708 + long deadline = tickDuration * (tick + 1); // 100 000000 + + for (;;) { + final long currentTime = System.nanoTime() - startTime; + long sleepTimeMs = (deadline - currentTime + 999999) / 1000000; + + if (sleepTimeMs <= 0) { + if (currentTime == Long.MIN_VALUE) { + return -Long.MAX_VALUE; + } else { + return currentTime; + } + } + if (isWindows()) { + sleepTimeMs = sleepTimeMs / 10 * 10; + } + + try { + Thread.sleep(sleepTimeMs); + } catch (InterruptedException ignored) { + if (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_SHUTDOWN) { + return Long.MIN_VALUE; + } + } + } + } + + Set unprocessedTimeouts() { + return Collections.unmodifiableSet(unprocessedTimeouts); + } + } + + private static final class HashedWheelTimeout implements Timeout { + + private static final int ST_INIT = 0; + private static final int ST_CANCELLED = 1; + private static final int ST_EXPIRED = 2; + private static final AtomicIntegerFieldUpdater STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimeout.class, "state"); + + private final HashedWheelTimer timer; + private final TimerTask task; + private final long deadline; + + @SuppressWarnings({"unused", "FieldMayBeFinal", "RedundantFieldInitialization"}) + private volatile int state = ST_INIT; + + /** + * RemainingRounds will be calculated and set by Worker.transferTimeoutsToBuckets() before the + * HashedWheelTimeout will be added to the correct HashedWheelBucket. + */ + long remainingRounds; + + /** + * This will be used to chain timeouts in HashedWheelTimerBucket via a double-linked-list. + * As only the workerThread will act on it there is no need for synchronization / volatile. + */ + HashedWheelTimeout next; + HashedWheelTimeout prev; + + /** + * The bucket to which the timeout was added + */ + HashedWheelBucket bucket; + + HashedWheelTimeout(HashedWheelTimer timer, TimerTask task, long deadline) { + this.timer = timer; + this.task = task; + this.deadline = deadline; + } + + @Override + public Timer timer() { + return timer; + } + + @Override + public TimerTask task() { + return task; + } + + @Override + public boolean cancel() { + // only update the state it will be removed from HashedWheelBucket on next tick. + if (!compareAndSetState(ST_INIT, ST_CANCELLED)) { + return false; + } + // If a task should be canceled we put this to another queue which will be processed on each tick. + // So this means that we will have a GC latency of max. 1 tick duration which is good enough. This way we + // can make again use of our LinkedBlockingQueue and so minimize the locking / overhead as much as possible. + timer.cancelledTimeouts.add(this); + return true; + } + + void remove() { + HashedWheelBucket bucket = this.bucket; + if (bucket != null) { + bucket.remove(this); + } else { + timer.pendingTimeouts.decrementAndGet(); + } + } + + public boolean compareAndSetState(int expected, int state) { + return STATE_UPDATER.compareAndSet(this, expected, state); + } + + public int state() { + return state; + } + + @Override + public boolean isCancelled() { + return state() == ST_CANCELLED; + } + + @Override + public boolean isExpired() { + return state() == ST_EXPIRED; + } + + public void expire() { + if (!compareAndSetState(ST_INIT, ST_EXPIRED)) { + return; + } + + try { + task.run(this); + } catch (Throwable t) { + if (log.isWarnEnabled()) { + log.warn("An exception was thrown by " + TimerTask.class.getSimpleName() + '.', t); + } + } + } + + @Override + public String toString() { + final long currentTime = System.nanoTime(); + long remaining = deadline - currentTime + timer.startTime; + String simpleClassName = ClassUtils.getSimpleName(this.getClass()); + + StringBuilder buf = new StringBuilder(192) + .append(simpleClassName) + .append('(') + .append("deadline: "); + if (remaining > 0) { + buf.append(remaining) + .append(" ns later"); + } else if (remaining < 0) { + buf.append(-remaining) + .append(" ns ago"); + } else { + buf.append("now"); + } + + if (isCancelled()) { + buf.append(", cancelled"); + } + + return buf.append(", task: ") + .append(task()) + .append(')') + .toString(); + } + } + + /** + * Bucket that stores HashedWheelTimeouts. These are stored in a linked-list like datastructure to allow easy + * removal of HashedWheelTimeouts in the middle. Also the HashedWheelTimeout act as nodes themself and so no + * extra object creation is needed. + */ + private static final class HashedWheelBucket { + + /** + * Used for the linked-list datastructure + */ + private HashedWheelTimeout head; + private HashedWheelTimeout tail; + + /** + * Add {@link HashedWheelTimeout} to this bucket. + */ + void addTimeout(HashedWheelTimeout timeout) { + assert timeout.bucket == null; + timeout.bucket = this; + if (head == null) { + head = tail = timeout; + } else { + tail.next = timeout; + timeout.prev = tail; + tail = timeout; + } + } + + /** + * Expire all {@link HashedWheelTimeout}s for the given {@code deadline}. + */ + void expireTimeouts(long deadline) { + HashedWheelTimeout timeout = head; + + // process all timeouts + while (timeout != null) { + HashedWheelTimeout next = timeout.next; + if (timeout.remainingRounds <= 0) { + next = remove(timeout); + if (timeout.deadline <= deadline) { + timeout.expire(); + } else { + // The timeout was placed into a wrong slot. This should never happen. + throw new IllegalStateException(String.format( + "timeout.deadline (%d) > deadline (%d)", timeout.deadline, deadline)); + } + } else if (timeout.isCancelled()) { + next = remove(timeout); + } else { + timeout.remainingRounds--; + } + timeout = next; + } + } + + public HashedWheelTimeout remove(HashedWheelTimeout timeout) { + HashedWheelTimeout next = timeout.next; + // remove timeout that was either processed or cancelled by updating the linked-list + if (timeout.prev != null) { + timeout.prev.next = next; + } + if (timeout.next != null) { + timeout.next.prev = timeout.prev; + } + + if (timeout == head) { + // if timeout is also the tail we need to adjust the entry too + if (timeout == tail) { + tail = null; + head = null; + } else { + head = next; + } + } else if (timeout == tail) { + // if the timeout is the tail modify the tail to be the prev node. + tail = timeout.prev; + } + // null out prev, next and bucket to allow for GC. + timeout.prev = null; + timeout.next = null; + timeout.bucket = null; + timeout.timer.pendingTimeouts.decrementAndGet(); + return next; + } + + /** + * Clear this bucket and return all not expired / cancelled {@link Timeout}s. + */ + void clearTimeouts(Set set) { + for (;;) { + HashedWheelTimeout timeout = pollTimeout(); + if (timeout == null) { + return; + } + if (timeout.isExpired() || timeout.isCancelled()) { + continue; + } + set.add(timeout); + } + } + + private HashedWheelTimeout pollTimeout() { + HashedWheelTimeout head = this.head; + if (head == null) { + return null; + } + HashedWheelTimeout next = head.next; + if (next == null) { + tail = this.head = null; + } else { + this.head = next; + next.prev = null; + } + + // null out prev and next to allow for GC. + head.next = null; + head.prev = null; + head.bucket = null; + return head; + } + } + + private static final boolean IS_OS_WINDOWS = + System.getProperty("os.name", "").toLowerCase(Locale.US).contains("win"); + + private boolean isWindows() { + return IS_OS_WINDOWS; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/HashedWheelTimerFactory.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/HashedWheelTimerFactory.java new file mode 100644 index 0000000000..e4d6293d96 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/HashedWheelTimerFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import com.dtstack.taier.metrics.factory.TpThreadFactory; + +/** + * @author xingyi + * @date 2025/11/26 + */ +public class HashedWheelTimerFactory { + + /** + * hold a single instance of HashedWheelTimer + */ + private static final HashedWheelTimer timer = new HashedWheelTimer(new TpThreadFactory("HashedWheelTimer-Monitor")); + + public static HashedWheelTimer holderHashedWheelTimer() { + return timer; + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/QueueTimeoutTimerTask.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/QueueTimeoutTimerTask.java new file mode 100644 index 0000000000..5f8ed6a80f --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/QueueTimeoutTimerTask.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.ThreadPoolStatProvider; +import com.dtstack.taier.metrics.adapter.ExecutorAdapter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; + +/** + * A timer task used to handle queued timeout. + **/ +@Slf4j +public class QueueTimeoutTimerTask extends AbstractTimeoutTimerTask { + + public QueueTimeoutTimerTask(ExecutorWrapper executorWrapper, Runnable runnable) { + super(executorWrapper, runnable); + } + + @Override + protected void doRun() { + ThreadPoolStatProvider statProvider = executorWrapper.getThreadPoolStatProvider(); + ExecutorAdapter executor = statProvider.getExecutorWrapper().getExecutor(); + Pair pair = getTaskNameAndTraceId(); + statProvider.incQueueTimeoutCount(1); + String logMsg = String.format("DynamicTp execute, queue timeout, " + + "tpName: %s, taskName: %s, traceId: %s, queueTimeout: %sms, " + + "poolSize: %s (active: %s, core: %s, max: %s, largest: %s), " + + "queueCapacity: %s (currSize: %s, remaining: %s)", + statProvider.getExecutorWrapper().getThreadPoolName(), pair.getLeft(), pair.getRight(), + statProvider.getQueueTimeout(), executor.getPoolSize(), executor.getActiveCount(), + executor.getCorePoolSize(), executor.getMaximumPoolSize(), + executor.getLargestPoolSize(), statProvider.getExecutorWrapper().getExecutor().getQueueCapacity(), + executor.getQueue().size(), executor.getQueue().remainingCapacity()); + // log.warn(logMsg); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/RunTimeoutTimerTask.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/RunTimeoutTimerTask.java new file mode 100644 index 0000000000..912c28d6c5 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/RunTimeoutTimerTask.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import com.dtstack.taier.metrics.ExecutorWrapper; +import com.dtstack.taier.metrics.ThreadPoolStatProvider; +import com.dtstack.taier.metrics.adapter.ExecutorAdapter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; + +/** + * A timer task used to handle run timeout. + * + * @author kamtohung + **/ +@Slf4j +public class RunTimeoutTimerTask extends AbstractTimeoutTimerTask { + + private final Thread thread; + + public RunTimeoutTimerTask(ExecutorWrapper executorWrapper, Runnable runnable, Thread thread) { + super(executorWrapper, runnable); + this.thread = thread; + } + + @Override + protected void doRun() { + ThreadPoolStatProvider statProvider = executorWrapper.getThreadPoolStatProvider(); + ExecutorAdapter executor = statProvider.getExecutorWrapper().getExecutor(); + Pair pair = getTaskNameAndTraceId(); + statProvider.incRunTimeoutCount(1); + String logMsg = String.format("DynamicTp execute, run timeout, " + + "tpName: %s, taskName: %s, traceId: %s, runTimeout: %sms, " + + "poolSize: %s (active: %s, core: %s, max: %s, largest: %s), " + + "queueCapacity: %s (currSize: %s, remaining: %s), stackTrace: %s", + statProvider.getExecutorWrapper().getThreadPoolName(), pair.getLeft(), pair.getRight(), + statProvider.getRunTimeout(), executor.getPoolSize(), executor.getActiveCount(), + executor.getCorePoolSize(), executor.getMaximumPoolSize(), executor.getLargestPoolSize(), + statProvider.getExecutorWrapper().getExecutor().getQueueCapacity(), executor.getQueue().size(), + executor.getQueue().remainingCapacity(), traceToString(thread.getStackTrace())); + // log.warn(logMsg); + if (statProvider.isTryInterrupt()) { + thread.interrupt(); + } + } + + public String traceToString(StackTraceElement[] trace) { + StringBuilder builder = new StringBuilder(512); + builder.append("\n"); + for (StackTraceElement traceElement : trace) { + builder.append("\tat ").append(traceElement).append("\n"); + } + return builder.toString(); + } +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/Timeout.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/Timeout.java new file mode 100644 index 0000000000..1aef4d6843 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/Timeout.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +/** + * A handle associated with a {@link TimerTask} that is returned by a{@link Timer}. + *

+ * Copy from dubbo, see here for more details. + *

+ */ +public interface Timeout { + + /** + * Returns the {@link Timer} that created this handle. + * + * @return the {@link Timer} that created this handle + */ + Timer timer(); + + /** + * Returns the {@link TimerTask} which is associated with this handle. + * + * @return the {@link TimerTask} which is associated with this handle + */ + TimerTask task(); + + /** + * Returns {@code true} if and only if the {@link TimerTask} associated + * with this handle has been expired. + * + * @return {@code true} if and only if the {@link TimerTask} associated + */ + boolean isExpired(); + + /** + * Returns {@code true} if and only if the {@link TimerTask} associated + * with this handle has been cancelled. + * + * @return {@code true} if and only if the {@link TimerTask} associated + */ + boolean isCancelled(); + + /** + * Attempts to cancel the {@link TimerTask} associated with this handle. + * If the task has been executed or cancelled already, it will return with + * no side effect. + * + * @return True if the cancellation completed successfully, otherwise false + */ + boolean cancel(); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/Timer.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/Timer.java new file mode 100644 index 0000000000..143b6d0095 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/Timer.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import java.util.Set; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * Schedules {@link TimerTask}s for one-time future execution in a background thread. + *

+ * Copy from dubbo, see here for more details. + *

+ */ +public interface Timer { + + /** + * Schedules the specified {@link TimerTask} for one-time execution after + * the specified delay. + * @param task the task to execute + * @param delay the time from now to delay execution + * @param unit the time unit of the delay parameter + * + * @return a handle which is associated with the specified task + * @throws IllegalStateException if this timer has been {@linkplain #stop() stopped} already + * @throws RejectedExecutionException if the pending timeouts are too many and creating new timeout + * can cause instability in the system. + */ + Timeout newTimeout(TimerTask task, long delay, TimeUnit unit); + + /** + * Releases all resources acquired by this {@link Timer} and cancels all + * tasks which were scheduled but not executed yet. + * + * @return the handles associated with the tasks which were canceled by + * this method + */ + Set stop(); + + /** + * the timer is stop + * + * @return true for stop + */ + boolean isStop(); +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/TimerTask.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/TimerTask.java new file mode 100644 index 0000000000..bc2188b7cf --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/timer/TimerTask.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.timer; + +import java.util.concurrent.TimeUnit; + +/** + * A task which is executed after the delay specified with + * {@link Timer#newTimeout(TimerTask, long, TimeUnit)} (TimerTask, long, TimeUnit)}. + *

+ * Copy from dubbo, see here for more details. + *

+ */ +public interface TimerTask { + + /** + * Executed after the delay specified with + * {@link Timer#newTimeout(TimerTask, long, TimeUnit)}. + * + * @param timeout a handle which is associated with this task + * @throws Exception if an error occurs + */ + void run(Timeout timeout) throws Exception; +} diff --git a/taier-metrics/src/main/java/com/dtstack/taier/metrics/wrapper/TaskWrapper.java b/taier-metrics/src/main/java/com/dtstack/taier/metrics/wrapper/TaskWrapper.java new file mode 100644 index 0000000000..3c1d915162 --- /dev/null +++ b/taier-metrics/src/main/java/com/dtstack/taier/metrics/wrapper/TaskWrapper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.metrics.wrapper; + +/** + * @author xingyi + * @date 2025/9/17 + */ +@FunctionalInterface +public interface TaskWrapper { + + /** + * Task wrapper name, for config. + * + * @return name + */ + default String name() { + return null; + } + + /** + * Enhance the given runnable. + * + * @param runnable source runnable + * @return target runnable + */ + Runnable wrap(Runnable runnable); +} diff --git a/taier-metrics/src/test/java/TpMetricsTests.java b/taier-metrics/src/test/java/TpMetricsTests.java new file mode 100644 index 0000000000..23c3147e75 --- /dev/null +++ b/taier-metrics/src/test/java/TpMetricsTests.java @@ -0,0 +1,45 @@ +import com.dtstack.taier.metrics.builder.ThreadPoolBuilder; +import com.dtstack.taier.metrics.executor.EngineExecutor; +import com.dtstack.taier.metrics.monitor.TpMonitor; + +import java.util.Collections; + +/** + * @author xingyi + * @date 2025/9/17 + */ +public class TpMetricsTests { + + public static void main(String[] args) throws InterruptedException { + // 1. build a thread pool + EngineExecutor eager = ThreadPoolBuilder.newBuilder() + .corePoolSize(1) + .maximumPoolSize(10) + .eager() + .queueTimeout(400) + .runTimeout(400) + .buildDynamic() + .registry(); // register by build + + // eager.execute(() -> System.out.println("")); + + // 执行任务 + for (int i = 0; i < 100; i++) { + eager.execute(() -> { + try { + System.out.println(Thread.currentThread().getName() + " 执行任务"); + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + } + + TpMonitor sm = new TpMonitor(); + sm.interval(2, 0, Collections.singletonList("output")); + + Thread.sleep(2000l); + + } + +} diff --git a/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/jobdealer/JobSubmitDealer.java b/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/jobdealer/JobSubmitDealer.java index ebacd5f6be..2624587429 100644 --- a/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/jobdealer/JobSubmitDealer.java +++ b/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/jobdealer/JobSubmitDealer.java @@ -31,12 +31,14 @@ import com.dtstack.taier.common.queue.DelayBlockingQueue; import com.dtstack.taier.common.util.SleepUtil; import com.dtstack.taier.dao.domain.ScheduleJobCache; +import com.dtstack.taier.metrics.collect.em.QueueTypeEnum; import com.dtstack.taier.pluginapi.CustomThreadFactory; import com.dtstack.taier.pluginapi.JobClient; import com.dtstack.taier.pluginapi.constrant.JobResultConstant; import com.dtstack.taier.pluginapi.enums.EQueueSourceType; import com.dtstack.taier.pluginapi.enums.TaskStatus; import com.dtstack.taier.pluginapi.exception.ClientArgumentException; +import com.dtstack.taier.pluginapi.metrics.DynamicMetricsThreadPoolUtil; import com.dtstack.taier.pluginapi.pojo.JobResult; import com.dtstack.taier.pluginapi.pojo.JudgeResult; import com.dtstack.taier.scheduler.WorkerOperator; @@ -132,8 +134,19 @@ public JobSubmitDealer(String localAddress, GroupPriorityQueue priorityQueue, Ap new LinkedBlockingQueue<>(), new CustomThreadFactory(this.getClass().getSimpleName() + "_" + jobResource + "_DelayJobProcessor")); executorService.submit(new RestartJobProcessor()); - this.jobSubmitConcurrentService = new ThreadPoolExecutor(1, jobSubmitConcurrent, 60L, TimeUnit.SECONDS, - new SynchronousQueue<>(true), new CustomThreadFactory(this.getClass().getSimpleName() + "_" + jobResource + "_JobSubmitConcurrent"), new BlockCallerPolicy()); + this.jobSubmitConcurrentService = (ThreadPoolExecutor) DynamicMetricsThreadPoolUtil.buildDynamicThreadPool( + jobSubmitConcurrent, + jobSubmitConcurrent, + 60L, + TimeUnit.SECONDS, + this.getClass().getSimpleName() + "_" + jobResource + "_JobSubmitConcurrent", + QueueTypeEnum.SYNCHRONOUS_QUEUE.getName(), + 0, + false, + 0, + this.getClass().getSimpleName() + "_" + jobResource + "_JobSubmitConcurrent", + new BlockCallerPolicy(), false); + rdbJobExecutor = new RdbJobExecutor(applicationContext, jobResource); } diff --git a/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/monitor/ScheduleTpMonitor.java b/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/monitor/ScheduleTpMonitor.java new file mode 100644 index 0000000000..0cfd748b06 --- /dev/null +++ b/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/monitor/ScheduleTpMonitor.java @@ -0,0 +1,90 @@ +package com.dtstack.taier.scheduler.monitor; + + +import com.dtstack.taier.common.env.EnvironmentContext; +import com.dtstack.taier.metrics.monitor.TpMonitor; +import com.dtstack.taier.metrics.prometheus.CollectorRegistryHolder; +import com.dtstack.taier.metrics.prometheus.PrometheusPushGatewayManager; +import io.prometheus.client.exporter.PushGateway; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + + +/** + * Service with monitor with {@link java.util.concurrent.ScheduledThreadPoolExecutor} + * @author xingyi + * @date 2025/9/17 + */ +@Service +public class ScheduleTpMonitor + implements + InitializingBean, + DisposableBean, + ApplicationListener { + + private final Logger LOGGER = LoggerFactory.getLogger(ScheduleTpMonitor.class); + + @Resource + private EnvironmentContext environmentContext; + + private TpMonitor tpMonitor; + + public void interval() { + + long monitorInterval = environmentContext.getMonitorMetricsInterval(); + long monitorDelay = environmentContext.getMonitorMetricsDelay(); + tpMonitor.interval(monitorInterval, monitorDelay, + environmentContext.getSupportMonitorMetricsType()); + } + + @Override + public void destroy() { + tpMonitor.destroy(); + } + + @Override + public void onApplicationEvent(ApplicationStartedEvent event) { + // @see CollectorTypeEnum配置为Micrometer ,需要实现对应的MicroMeterCollector + if (environmentContext.getIsMonitorMetrics()) { + interval(); + } + if (StringUtils.isNotEmpty(environmentContext.getPrometheusPushGatewayUrl())) { + + String prometheusPushGatewayUrl = environmentContext.getPrometheusPushGatewayUrl(); + PushGateway pushGateway = initializePushGateway(prometheusPushGatewayUrl); + + // register metrics to prometheus push gateway + new PrometheusPushGatewayManager(pushGateway, + CollectorRegistryHolder.getCollectorRegistry(), + environmentContext.getPrometheusPushGatewayInterval(), "schedule_tp", null, + PrometheusPushGatewayManager.ShutdownOperation.NONE); + + } + } + + private PushGateway initializePushGateway(String url) { + try { + return new PushGateway(new URL(url)); + } catch (MalformedURLException ex) { + LOGGER.error("Invalid PushGateway base url '{}': update your configuration to a valid URL", url); + return new PushGateway(url); + } + } + + @Override + public void afterPropertiesSet() throws Exception { + tpMonitor = new TpMonitor(); + } +} diff --git a/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/server/scheduler/AbstractJobSummitScheduler.java b/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/server/scheduler/AbstractJobSummitScheduler.java index 6662dbb91a..7bb412b4bc 100644 --- a/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/server/scheduler/AbstractJobSummitScheduler.java +++ b/taier-scheduler/src/main/java/com/dtstack/taier/scheduler/server/scheduler/AbstractJobSummitScheduler.java @@ -21,8 +21,11 @@ import com.dtstack.taier.common.CustomThreadRunsPolicy; import com.dtstack.taier.common.enums.EScheduleJobType; import com.dtstack.taier.dao.domain.ScheduleJob; +import com.dtstack.taier.metrics.collect.em.QueueTypeEnum; +import com.dtstack.taier.metrics.executor.EngineExecutor; import com.dtstack.taier.pluginapi.enums.TaskStatus; import com.dtstack.taier.pluginapi.exception.ExceptionUtil; +import com.dtstack.taier.pluginapi.metrics.DynamicMetricsThreadPoolUtil; import com.dtstack.taier.scheduler.enums.JobPhaseStatus; import com.dtstack.taier.scheduler.server.ScheduleJobDetails; import com.dtstack.taier.scheduler.service.ScheduleJobService; @@ -102,12 +105,18 @@ public void onApplicationEvent(ApplicationStartedEvent applicationStartedEvent) super.onApplicationEvent(applicationStartedEvent); String threadName = this.getClass().getSimpleName() + "_" + getSchedulerName() + "_startJobProcessor"; - executorService = new ThreadPoolExecutor(env.getJobExecutorPoolCorePoolSize(), env.getJobExecutorPoolMaximumPoolSize(), env.getJobExecutorPoolKeepAliveTime(), TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(env.getJobExecutorPoolQueueSize()), + executorService = (EngineExecutor) DynamicMetricsThreadPoolUtil.buildDynamicThreadPool( + env.getJobExecutorPoolCorePoolSize(), + env.getJobExecutorPoolMaximumPoolSize(), + env.getJobExecutorPoolKeepAliveTime(), TimeUnit.MILLISECONDS, + threadName, QueueTypeEnum.LINKED_BLOCKING_QUEUE.getName(), + env.getJobExecutorPoolQueueSize(), false, 0, + threadName, new CustomThreadRunsPolicy(threadName, getSchedulerName(), (job -> { scheduleJobService.updatePhaseStatusById(job.getId(), JobPhaseStatus.JOIN_THE_TEAM, JobPhaseStatus.CREATE); LOGGER.warn("start job processor reject job {},return job to db", job.getJobId()); - }))); + })), true); + } /** diff --git a/taier-worker/taier-worker-api/pom.xml b/taier-worker/taier-worker-api/pom.xml index 38b662e030..b4cd68d2fb 100644 --- a/taier-worker/taier-worker-api/pom.xml +++ b/taier-worker/taier-worker-api/pom.xml @@ -93,6 +93,13 @@ 1.0.31 + + com.dtstack.taier + taier-metrics + 1.0.0 + compile + + diff --git a/taier-worker/taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/metrics/DynamicMetricsThreadPoolUtil.java b/taier-worker/taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/metrics/DynamicMetricsThreadPoolUtil.java new file mode 100644 index 0000000000..f53c22579f --- /dev/null +++ b/taier-worker/taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/metrics/DynamicMetricsThreadPoolUtil.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.taier.pluginapi.metrics; + + +import com.dtstack.taier.metrics.builder.ThreadPoolBuilder; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.TimeUnit; + +/** + * 可观测线程池创建工具类 + * @author xingyi + * @date 2025/10/15 + */ +public class DynamicMetricsThreadPoolUtil { + + /** + * 创建可观测线程池 + * @param corePoolSize 核心线程数 + * @param maximumPoolSize 最大线程数 + * @param keepAliveTime 当线程数超过核心数时,这是多余空闲线程在等待新任务前的最大等待时间。 + * @param unit 时间单位 + * @param threadPoolName 线程池名称 + * @param queueName 队列名称 + * @param capacity 队列容量 + * @param fair 是否公平队列 + * @param timeout 线程执行超时时间, 单位毫秒,默认为0时,不开启超时检测, 注意runTimeout 和queueTimeout的区别 + * runTimeout 表示线程执行该Runnable的超时检测时间 + * queueTimeout 表示线程从队列取任务的超时检测时间, + * @param threadNamePrefix 线程名称前缀 + * @param handler 拒绝策略 + * @param eager 是否饥饿模式,true: 适用于 IO 密集型场景,在线程池没达到设置的最大值之前优先创建新线程执行任务而不是放入队列等待,比如 tomcat 线程池、dubbo 线程池都是采用这种模式, 默认false: jvm 官方线程池模式 + * @return ExecutorService + */ + public static ExecutorService buildDynamicThreadPool( + int corePoolSize, int maximumPoolSize, long keepAliveTime, + TimeUnit unit, + String threadPoolName, String queueName, Integer capacity, + Boolean fair, long timeout, String threadNamePrefix, + RejectedExecutionHandler handler, boolean eager) { + return ThreadPoolBuilder.newBuilder() + .threadPoolName(threadPoolName) + .corePoolSize(corePoolSize) + .maximumPoolSize(maximumPoolSize) + .workQueue(queueName, capacity, fair) + .queueTimeout(timeout) + .threadFactory(threadNamePrefix) + .keepAliveTime(keepAliveTime) + .timeUnit(unit) + .rejectEnhanced(true) + .rejectedExecutionHandler(handler) + .runTimeout(timeout) + .eager(eager) + .buildDynamic() + .registry(); // register by build + } +} diff --git a/website/docs/functions/metrics-monitor.md b/website/docs/functions/metrics-monitor.md new file mode 100644 index 0000000000..a1d70a206b --- /dev/null +++ b/website/docs/functions/metrics-monitor.md @@ -0,0 +1,163 @@ +--- +title: 指标监控 +sidebar_label: 指标监控 +--- + +# 指标监控 + + +可观测的性能指标监控是数栈平台指标监控的指标维度数据之一,本文主要从实现框架介绍接入逻辑。 + +## 一 构建可观测的线程池框架 + +### 1 线程池监控 + +核心原理主要依赖动态 Proxy 和切面处理,从而监控对应线程池的核心数据,包括线程池的核心参数信息,以及计算线程池对应 Runnable 执行耗时的 RT 计算性能指标,并通过配置指标输出策略进行指标推送和指标监控。 + +框架封装:`taier-metrics` + +#### 1.1 指标输出 + +目前主要支持三种模式的指标推送: + +- `MICROMETER`:该模式下会配合 micrometer 框架和 prometheus,将对应的监控指标推送到 prometheus。 +- `LOGGING`:该模式下将对应的监控指标以 log 日志的形式进行打印。 +- `OUTPUT`:该模式下将对应的监控指标以 `System.out` 的形式打印到当前 PID 进程下。 + +#### 1.2 线程池模式 + +可观测线程监控主要通过代理原生的 `ThreadPoolExecutor` 进行指标监控,目前主要内置了两种线程池模式: + +- `common` 模式:对应线程池 `EngineExecutor`,`EngineExecutor` 是该框架的核心顶层设计类,其他类都继承自该类。`common` 模式是默认的线程池模式,适用于 CPU 密集型场景,当核心线程数满了优先放入队列等待。 +- `eager` 模式:对应线程池 `EagerEngineExecutor`,`eager` 模式适用于 IO 密集型场景,在线程池没达到设置的最大值之前优先创建新线程执行任务,而不是放入队列等待,比如 tomcat 线程池、dubbo 线程池都是采用这种模式。 + +#### 1.3 线程池指标数据 + +- `poolName`:当前线程池的名称,唯一标识。 +- `poolAliasName`:当前线程池的别名。 +- `corePoolSize`:核心线程数。 +- `maximumPoolSize`:最大线程数。 +- `keepAliveTime`:空闲时间。 +- `queueType`:队列类型。 +- `queueCapacity`:队列容量。 +- `queueSize`:队列任务数。 +- `fair`:`SynchronousQueue` 队列模式。 +- `queueRemainingCapacity`:队列剩余容量。 +- `activeCount`:正在执行任务的活跃线程大致总数。 +- `taskCount`:大致任务总数。 +- `completedTaskCount`:已执行完成的大致任务总数。 +- `largestPoolSize`:池中曾经同时存在的最大线程数量。 +- `poolSize`:当前池中存在的线程总数。 +- `waitTaskCount`:等待执行的任务数量。 +- `rejectCount`:拒绝的任务数量。 +- `rejectHandlerName`:拒绝策略名称。 +- `runTimeoutCount`:执行超时任务数量。 +- `queueTimeoutCount`:在队列中等待超时的数量。 +- `tps`:TPS。 +- `maxRt`:最大执行耗时。 +- `minRt`:最小执行耗时。 +- `avg`:平均执行耗时。 + +#### 1.4 线程池指标示例 + +```java +// 1. start a monitor +TpMonitor sm = new TpMonitor(); +sm.interval(20, 0, Collections.singletonList("output")); + +// set timeout +int timeout = 1000; + +// 2. build a metricsThreadPoolExecutor +EngineExecutor eager = ThreadPoolBuilder.newBuilder() + .threadPoolName("ClientProxy") // 当前线程池名称,唯一标识 + .corePoolSize(1) // 核心线程数 + .maximumPoolSize(1) // 最大线程数 + .workQueue(QueueTypeEnum.SYNCHRONOUS_QUEUE.getName(), null, false) // 线程队列数据 + .queueTimeout(timeout) // 队列超时时间 + .keepAliveTime(7200000) + .rejectEnhanced(true) // 拒绝策略增强,内部会通过 Proxy 动态代理构建并进行指标统计 + .rejectedExecutionHandler(new BlockCallerTimeoutPolicy(timeout)) + .runTimeout(timeout) // 查询超时 + // .eager(false) // eager 模式,IO 密集型,默认 false + .buildDynamic() + .registry(); // register by build + +ThreadPoolExecutor executorService = new ThreadPoolExecutor( + 1, + 1, + 7200000, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + new CustomThreadFactory("xingyi"), + new BlockCallerTimeoutPolicy(timeout) +); + +// run 10s, put 5s Future timeout 5s +for (int i = 0; i < 20; i++) { + try { + JobResult jobResult = CompletableFuture.supplyAsync(() -> { + try { + return ClassLoaderCallBackMethod.callbackAndReset(new CallBack() { + @Override + public JobResult execute() throws Exception { + System.out.println(Thread.currentThread().getName() + " : execute"); + Thread.sleep(30000); + System.out.println(Thread.currentThread().getName() + " : FINISH"); + return new JobResult(); + } + }, Thread.currentThread().getContextClassLoader(), true); + } catch (Exception e) { + throw new RdosDefineException(e); + } + }, eager).get(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | RejectedExecutionException e) { + System.out.println(e.getMessage()); + } catch (TimeoutException e) { + System.out.println("timeout"); + } +} + +for (int i = 0; i < 20; i++) { + try { + JobResult jobResult = CompletableFuture.supplyAsync(() -> { + try { + return ClassLoaderCallBackMethod.callbackAndReset(new CallBack() { + @Override + public JobResult execute() throws Exception { + System.out.println(Thread.currentThread().getName() + " : execute"); + Thread.sleep(200); + System.out.println(Thread.currentThread().getName() + " : FINISH"); + return new JobResult(); + } + }, Thread.currentThread().getContextClassLoader(), true); + } catch (Exception e) { + throw new RdosDefineException(e); + } + }, eager).get(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | RejectedExecutionException e) { + System.out.println(e.getMessage()); + } catch (TimeoutException e) { + System.out.println("timeout"); + } +} + +Thread.sleep(2000000L); +``` + +指标打印: + +```java +system meter collect: ThreadPoolStats{poolName='ThreadPool', poolAliasName='null', corePoolSize=1, maximumPoolSize=1, keepAliveTime=7200000000, queueType='SynchronousQueue', queueCapacity=0, queueSize=0, fair=false, queueRemainingCapacity=0, activeCount=0, taskCount=12, completedTaskCount=12, largestPoolSize=1, poolSize=1, waitTaskCount=0, rejectCount=29, rejectHandlerName='BlockCallerTimeoutPolicy', dynamic=false, runTimeoutCount=1, queueTimeoutCount=0, tps=1.1, maxRt=30024, minRt=203, avg=1953.5455} +system meter collect: ThreadPoolStats{poolName='ThreadPool', poolAliasName='null', corePoolSize=1, maximumPoolSize=1, keepAliveTime=7200000000, queueType='SynchronousQueue', queueCapacity=0, queueSize=0, fair=false, queueRemainingCapacity=0, activeCount=0, taskCount=12, completedTaskCount=12, largestPoolSize=1, poolSize=1, waitTaskCount=0, rejectCount=29, rejectHandlerName='BlockCallerTimeoutPolicy', dynamic=false, runTimeoutCount=1, queueTimeoutCount=0, tps=0.0, maxRt=0, minRt=0, avg=0.0} +``` + +#### 1.5 MICROMETER 指标示例 + +当配置MICROMETER指标时,会自动收集线程池指标。 +配置参数: +- taier.monitor.metrics.support.type=micrometer +- taier.monitor.metrics.enabled=true +- taier.monitor.metrics.prometheus.pushgateway.url=http://localhost:9091/metrics # pushgateway地址 +- taier.monitor.metrics.prometheus.pushgateway.timeout=5 # pushgateway超时时间, 单位s 默认5s +- taier.monitor.metrics.prometheus.pushgateway.interval=60 # pushgateway 默认间隔时间,单位s, 默认60s diff --git a/website/sidebars.js b/website/sidebars.js index deda38052f..0c9f039524 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -104,6 +104,7 @@ const sidebars = { 'functions/depend', 'functions/task-param', 'functions/environmental-parameters', + 'functions/metrics-monitor', ], }, {