Skip to content

Commit

Permalink
PAYARA-1056 global thread pool statistics incorrect (payara#1123)
Browse files Browse the repository at this point in the history 
* PAYARA-1056 Prevent < 0 values, and calculate global core and max thread count.

* PAYARA-1056 Change the get methods to query the JVM rather than their stored values. Also adds in copyright.

* PAYARA-1056 Remove whitespace edit

* PAYARA-1056 Make the Global thread pool busy and current thread counts more responsive and accurate.

* PAYARA-1056 Remove now redundant event monitoring methods, and other cleanup.
  • Loading branch information
@Pandrex247 @lprimak
Pandrex247 authored and lprimak committed Jul 5, 2017
1 parent 57a0468 commit 6a6ab11
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 67 deletions.
67 changes: 66 additions & 1 deletion ...e/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/GrizzlyMonitoring.java
View file
Expand Up @@ -38,6 +38,8 @@
* holder.
*/

// Portions Copyright [2016] [Payara Foundation and/or its affiliates]

package com.sun.enterprise.v3.services.impl.monitor;

import com.sun.enterprise.v3.services.impl.monitor.probes.ConnectionQueueProbeProvider;
Expand All @@ -52,10 +54,13 @@
import com.sun.enterprise.v3.services.impl.monitor.stats.KeepAliveStatsProviderGlobal;
import com.sun.enterprise.v3.services.impl.monitor.stats.ThreadPoolStatsProvider;
import com.sun.enterprise.v3.services.impl.monitor.stats.ThreadPoolStatsProviderGlobal;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.glassfish.external.probe.provider.PluginPoint;
import org.glassfish.external.probe.provider.StatsProviderManager;
import org.glassfish.grizzly.threadpool.ThreadPoolConfig;

/**
* Grizzly monitoring manager, which is responsible for registering, unregistering
Expand Down Expand Up @@ -147,6 +152,7 @@ public void registerThreadPoolStatsProvider(String name) {

StatsProviderManager.register(CONFIG_ELEMENT, PluginPoint.SERVER,
subtreePrefix(name) + "/thread-pool", threadPoolStatsProvider);
updateGlobalThreadPoolStatsProvider();
}

/**
Expand All @@ -159,7 +165,64 @@ public void unregisterThreadPoolStatsProvider(String name) {
threadPoolStatsProvidersMap.remove(name);
if (threadPoolStatsProvider != null) {
StatsProviderManager.unregister(threadPoolStatsProvider);
updateGlobalThreadPoolStatsProvider();
}
}

/**
* Updates the global thread pool stats provider with the most current
* total values.
*/
private void updateGlobalThreadPoolStatsProvider() {
final ThreadPoolStatsProvider globalThreadPoolStatsProvider =
threadPoolStatsProvidersMap.get("");

// Exit method if global stats provider not registered
if (globalThreadPoolStatsProvider == null) {
return;
}

int coreThreadTotal = 0;
int maxThreadTotal = 0;

// If multiple listeners use the same thread pool, we don't want to
// count the threads twice, so we'll store the names of those we've
// already counted in here
List<String> countedThreadPoolNames = new ArrayList<>();

// Calculate the totals for each of the metrics
for (Map.Entry<String, ThreadPoolStatsProvider> threadPoolStatsProvider
: threadPoolStatsProvidersMap.entrySet()) {
if (!threadPoolStatsProvider.getKey().equals("")) {
// Get the pool config so we can check the thread pool name
ThreadPoolConfig threadPoolConfig = (ThreadPoolConfig)
threadPoolStatsProvider.getValue().getStatsObject();
if (threadPoolConfig != null) {
String threadPoolName = threadPoolConfig.getPoolName();

// Check we haven't already counted the threads in this
// thread pool
if (!countedThreadPoolNames.contains(threadPoolName)) {
// Safe to cast from long to int, as the values cannot
// actually be higher than max int.
coreThreadTotal += (int) (long) threadPoolStatsProvider
.getValue().getCoreThreadsCount().getCount();
maxThreadTotal += (int) (long) threadPoolStatsProvider
.getValue().getMaxThreadsCount().getCount();

// Add to the list of counted thread pools so we don't
// count the threads twice
countedThreadPoolNames.add(threadPoolName);
}
}
}
}

// Now that we've calculated the global core and max values, set them
globalThreadPoolStatsProvider.setCoreThreadsEvent("", "",
coreThreadTotal);
globalThreadPoolStatsProvider.setMaxThreadsEvent("", "",
maxThreadTotal);
}

/**
Expand Down Expand Up @@ -268,7 +331,9 @@ public void registerThreadPoolStatsProviderGlobal(String name) {
}

StatsProviderManager.register(CONFIG_ELEMENT, PluginPoint.SERVER,
subtreePrefix(name) + "/thread-pool", threadPoolStatsProvider);
subtreePrefix(name) + "/global-thread-pool-stats", threadPoolStatsProvider);

updateGlobalThreadPoolStatsProvider();
}

/**
Expand Down
87 changes: 50 additions & 37 deletions .../main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java
View file
Expand Up @@ -38,8 +38,13 @@
* holder.
*/

// Portions Copyright [2016] [Payara Foundation and/or its affiliates]

package com.sun.enterprise.v3.services.impl.monitor.stats;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.glassfish.external.probe.provider.annotations.ProbeListener;
import org.glassfish.external.probe.provider.annotations.ProbeParam;
import org.glassfish.external.statistics.CountStatistic;
Expand Down Expand Up @@ -70,7 +75,11 @@ public class ThreadPoolStatsProvider implements StatsProvider {
protected final CountStatisticImpl currentThreadsBusy = new CountStatisticImpl("CurrentThreadsBusy", "count", "Provides the number of request processing threads currently in use in the listener thread pool serving requests");

protected volatile ThreadPoolConfig threadPoolConfig;


// We need to keep track of the thread pool names for calculating the
// global values
protected volatile static List<String> threadPoolNames = new ArrayList<>();

public ThreadPoolStatsProvider(String name) {
this.name = name;
}
Expand All @@ -84,6 +93,11 @@ public Object getStatsObject() {
public void setStatsObject(Object object) {
if (object instanceof ThreadPoolConfig) {
threadPoolConfig = (ThreadPoolConfig) object;

// We don't want to add the global thread pool
if (!threadPoolConfig.getPoolName().equals("")) {
threadPoolNames.add(threadPoolConfig.getPoolName());
}
} else {
threadPoolConfig = null;
}
Expand All @@ -110,12 +124,18 @@ public CountStatistic getTotalExecutedTasksCount() {
@ManagedAttribute(id = "currentthreadcount")
@Description("Provides the number of request processing threads currently in the listener thread pool")
public CountStatistic getCurrentThreadCount() {
if (threadPoolConfig != null) {
countThreadsInThreadPool(threadPoolConfig.getPoolName());
}
return currentThreadCount;
}

@ManagedAttribute(id = "currentthreadsbusy")
@Description("Provides the number of request processing threads currently in use in the listener thread pool serving requests.")
public CountStatistic getCurrentThreadsBusy() {
if (threadPoolConfig != null) {
countThreadsInThreadPool(threadPoolConfig.getPoolName());
}
return currentThreadsBusy;
}

Expand All @@ -141,39 +161,6 @@ public void setCoreThreadsEvent(
}
}

@ProbeListener("glassfish:kernel:thread-pool:threadAllocatedEvent")
public void threadAllocatedEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

if (name.equals(monitoringId)) {
currentThreadCount.increment();
}
}

@ProbeListener("glassfish:kernel:thread-pool:threadReleasedEvent")
public void threadReleasedEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

if (name.equals(monitoringId)) {
currentThreadCount.decrement();
}
}

@ProbeListener("glassfish:kernel:thread-pool:threadDispatchedFromPoolEvent")
public void threadDispatchedFromPoolEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

if (name.equals(monitoringId)) {
currentThreadsBusy.increment();
}
}

@ProbeListener("glassfish:kernel:thread-pool:threadReturnedToPoolEvent")
public void threadReturnedToPoolEvent(
@ProbeParam("monitoringId") String monitoringId,
Expand All @@ -182,7 +169,6 @@ public void threadReturnedToPoolEvent(

if (name.equals(monitoringId)) {
totalExecutedTasksCount.increment();
currentThreadsBusy.decrement();
}
}

Expand All @@ -191,10 +177,37 @@ public void reset() {
if (threadPoolConfig != null) {
maxThreadsCount.setCount(threadPoolConfig.getMaxPoolSize());
coreThreadsCount.setCount(threadPoolConfig.getCorePoolSize());
currentThreadCount.setCount(0);
currentThreadsBusy.setCount(0);
}

totalExecutedTasksCount.setCount(0);
}

/**
* Counts the threads in the given thread pool by querying the JVM. Also
* counts the number of threads that are running.
* @param threadPoolName The name of the thread pool to count the threads of
*/
private void countThreadsInThreadPool(String threadPoolName) {
// Set to 0 as we want to reset them
currentThreadCount.setCount(0);
currentThreadsBusy.setCount(0);

// Get all the threads currently in the JVM
Set<Thread> threads = Thread.getAllStackTraces().keySet();

// If multiple listeners use the same thread pool, you will get
// duplicate named threads, so we want to filter these out
List<String> alreadyCounted = new ArrayList<>();
for (Thread thread : threads) {
String threadName = thread.getName();
if (thread.isAlive() && threadName.contains(threadPoolName
+ "(") && !alreadyCounted.contains(threadName)) {
alreadyCounted.add(threadName);
currentThreadCount.increment();
if (thread.getState() == Thread.State.RUNNABLE) {
currentThreadsBusy.increment();
}
}
}
}
}
87 changes: 58 additions & 29 deletions ...java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProviderGlobal.java
View file
Expand Up @@ -38,12 +38,19 @@
* holder.
*/

// Portions Copyright [2016] [Payara Foundation and/or its affiliates]

package com.sun.enterprise.v3.services.impl.monitor.stats;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.glassfish.external.probe.provider.annotations.ProbeListener;
import org.glassfish.external.probe.provider.annotations.ProbeParam;
import org.glassfish.external.statistics.CountStatistic;
import org.glassfish.gmbal.AMXMetadata;
import org.glassfish.gmbal.Description;
import org.glassfish.gmbal.ManagedAttribute;
import org.glassfish.gmbal.ManagedObject;

/**
Expand All @@ -55,12 +62,29 @@
@ManagedObject
@Description("Thread Pool Statistics")
public class ThreadPoolStatsProviderGlobal extends ThreadPoolStatsProvider {

public ThreadPoolStatsProviderGlobal(String name) {
super(name);
}

@ManagedAttribute(id = "currentthreadcount")
@Description("Provides the number of request processing threads currently in the listener thread pool")
@Override
public CountStatistic getCurrentThreadCount() {
countThreadsInThreadPools();
return currentThreadCount;
}

@ManagedAttribute(id = "currentthreadsbusy")
@Description("Provides the number of request processing threads currently in use in the listener thread pool serving requests.")
@Override
public CountStatistic getCurrentThreadsBusy() {
countThreadsInThreadPools();
return currentThreadsBusy;
}

@ProbeListener("glassfish:kernel:thread-pool:setMaxThreadsEvent")
@Override
public void setMaxThreadsEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
Expand All @@ -70,6 +94,7 @@ public void setMaxThreadsEvent(
}

@ProbeListener("glassfish:kernel:thread-pool:setCoreThreadsEvent")
@Override
public void setCoreThreadsEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
Expand All @@ -78,41 +103,45 @@ public void setCoreThreadsEvent(
coreThreadsCount.setCount(coreNumberOfThreads);
}

@ProbeListener("glassfish:kernel:thread-pool:threadAllocatedEvent")
public void threadAllocatedEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

currentThreadCount.increment();
}

@ProbeListener("glassfish:kernel:thread-pool:threadReleasedEvent")
public void threadReleasedEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

currentThreadCount.decrement();
}

@ProbeListener("glassfish:kernel:thread-pool:threadDispatchedFromPoolEvent")
public void threadDispatchedFromPoolEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

currentThreadsBusy.increment();
}

@ProbeListener("glassfish:kernel:thread-pool:threadReturnedToPoolEvent")
@Override
public void threadReturnedToPoolEvent(
@ProbeParam("monitoringId") String monitoringId,
@ProbeParam("threadPoolName") String threadPoolName,
@ProbeParam("threadId") long threadId) {

totalExecutedTasksCount.increment();
currentThreadsBusy.decrement();
}

/**
* Counts the threads in all thread pools by querying the JVM. Also
* counts the number of threads that are running.
*/
private void countThreadsInThreadPools() {
// Set to 0 as we want to reset them
currentThreadCount.setCount(0);
currentThreadsBusy.setCount(0);

// Get all the threads currently in the JVM
Set<Thread> threads = Thread.getAllStackTraces().keySet();

// If multiple listeners use the same thread pool, you will get
// duplicate named threads, so we want to filter these out
List<String> alreadyCounted = new ArrayList<>();
for (Thread thread : threads) {
String threadName = thread.getName();
for (String threadPoolName : threadPoolNames) {
if (thread.isAlive() && threadName.contains(threadPoolName
+ "(") && !alreadyCounted.contains(threadName)) {
alreadyCounted.add(threadName);
currentThreadCount.increment();
if (thread.getState() == Thread.State.RUNNABLE) {
currentThreadsBusy.increment();
}

break;
}
}
}
}
}

0 comments on commit 6a6ab11

Please sign in to comment.