TECH-1808 Refactor Load Average

src/main/java/org/jahia/modules/sam/graphql/Load.java

@@ -2,28 +2,43 @@
 
 import graphql.annotations.annotationTypes.GraphQLDescription;
 import graphql.annotations.annotationTypes.GraphQLField;
-import org.jahia.utils.JCRNodeCacheLoadAverage;
-import org.jahia.utils.JCRSessionLoadAverage;
-import org.jahia.utils.RequestLoadAverage;
+import org.jahia.modules.graphql.provider.dxm.osgi.annotations.GraphQLOsgiService;
+import org.jahia.modules.sam.load.LoadAverageService;
+import org.jahia.modules.sam.load.provider.JCRNodeCacheLoadAverage;
+import org.jahia.modules.sam.load.provider.JCRSessionLoadAverage;
+import org.jahia.modules.sam.load.provider.RequestLoadAverage;
+import org.jahia.modules.sam.load.provider.ThreadLoadAverage;
+
+import javax.inject.Inject;
 
 @GraphQLDescription("Server load")
 public class Load {
 
+    @Inject
+    @GraphQLOsgiService(service = LoadAverageService.class)
+    private LoadAverageService LoadAverageService;
+
+    @GraphQLField
+    @GraphQLDescription("Get Thread load")
+    public LoadProvider getThread() {
+        return new LoadProvider(LoadAverageService.findProvider(ThreadLoadAverage.class.getName()).orElse(null));
+    }
+
     @GraphQLField
     @GraphQLDescription("Get requests load")
-    public LoadValue getRequests() {
-        return new LoadValue(RequestLoadAverage.getInstance());
+    public LoadProvider getRequests() {
+        return new LoadProvider(LoadAverageService.findProvider(RequestLoadAverage.class.getName()).orElse(null));
     }
 
     @GraphQLField
     @GraphQLDescription("Get JCR Sessions load")
-    public LoadValue getSessions() {
-        return new LoadValue(JCRSessionLoadAverage.getInstance());
+    public LoadProvider getSessions() {
+        return new LoadProvider(LoadAverageService.findProvider(JCRSessionLoadAverage.class.getName()).orElse(null));
     }
 
     @GraphQLField
     @GraphQLDescription("Get JCR Node Cache load across active sessions")
-    public LoadValue getCachedNodes() {
-        return new LoadValue(JCRNodeCacheLoadAverage.getInstance());
+    public LoadProvider getCachedNodes() {
+        return new LoadProvider(LoadAverageService.findProvider(JCRNodeCacheLoadAverage.class.getName()).orElse(null));
     }
 }

src/main/java/org/jahia/modules/sam/graphql/LoadValue.java → src/main/java/org/jahia/modules/sam/graphql/LoadProvider.java

@@ -3,43 +3,50 @@
 import graphql.annotations.annotationTypes.GraphQLDescription;
 import graphql.annotations.annotationTypes.GraphQLField;
 import graphql.annotations.annotationTypes.GraphQLName;
-import org.jahia.utils.LoadAverage;
+import org.jahia.modules.sam.load.LoadAverageProvider;
 
-@GraphQLDescription("Load value")
-public class LoadValue {
+@GraphQLDescription("Load provider")
+public class LoadProvider {
 
     @GraphQLDescription("Interval expressed in minutes")
     public enum LoadInterval {
         ONE, FIVE, FIFTEEN
     }
 
-    private LoadAverage loadAverage;
+    private LoadAverageProvider loadProvider;
 
-    public LoadValue(LoadAverage loadAverage) {
-        this.loadAverage = loadAverage;
+    public LoadProvider(LoadAverageProvider loadProvider) {
+        this.loadProvider = loadProvider;
     }
 
     @GraphQLField
     @GraphQLDescription("Instantaneous count")
-    public int getCount() {
-        return (int) loadAverage.getCount();
+    public double getCount() {
+        return loadProvider.getValue();
     }
 
+    @GraphQLField
+    @GraphQLDescription("Load Entry")
+    public String getEntry() {
+        return loadProvider.getAverage().toString();
+    }
+
+
     @GraphQLField
     @GraphQLDescription("Exponential moving average")
     public double getAverage(
             @GraphQLName("interval") @GraphQLDescription("Interval between collection of load metrics") LoadInterval interval) {
         if (interval == null) {
-            return loadAverage.getFifteenMinuteLoad();
+            return loadProvider.getAverage().getFifteenMinuteLoad();
         }
         switch (interval) {
             case ONE:
-                return loadAverage.getOneMinuteLoad();
+                return loadProvider.getAverage().getOneMinuteLoad();
             case FIVE:
-                return loadAverage.getFiveMinuteLoad();
+                return loadProvider.getAverage().getFiveMinuteLoad();
             case FIFTEEN:
             default:
-                return loadAverage.getFifteenMinuteLoad();
+                return loadProvider.getAverage().getFifteenMinuteLoad();
         }
     }
 

src/main/java/org/jahia/modules/sam/healthcheck/probes/ServerLoadProbe.java

@@ -4,10 +4,14 @@
 import org.jahia.modules.sam.Probe;
 import org.jahia.modules.sam.ProbeSeverity;
 import org.jahia.modules.sam.ProbeStatus;
-import org.jahia.utils.JCRNodeCacheLoadAverage;
-import org.jahia.utils.JCRSessionLoadAverage;
-import org.jahia.utils.RequestLoadAverage;
+import org.jahia.modules.sam.load.LoadAverageService;
+import org.jahia.modules.sam.load.LoadAverageValue;
+import org.jahia.modules.sam.load.provider.JCRNodeCacheLoadAverage;
+import org.jahia.modules.sam.load.provider.JCRSessionLoadAverage;
+import org.jahia.modules.sam.load.provider.RequestLoadAverage;
+import org.jahia.modules.sam.load.provider.ThreadLoadAverage;
 import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -24,31 +28,49 @@ public class ServerLoadProbe implements Probe {
     private int sessionLoadRedThreshold = 70;
     private int nodeCacheLoadYellowThreshold = 1000;
     private int nodeCacheLoadRedThreshold = 2000;
+    private int threadLoadYellowThreshold = 1000;
+    private int threadLoadRedThreshold = 1500;
 
     private static final String REQUEST_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY = "requestLoadYellowThreshold";
     private static final String REQUEST_LOAD_RED_THRESHOLD_CONFIG_PROPERTY = "requestLoadRedThreshold";
     private static final String SESSION_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY = "sessionLoadYellowThreshold";
     private static final String SESSION_LOAD_RED_THRESHOLD_CONFIG_PROPERTY = "sessionLoadRedThreshold";
     private static final String NODECACHE_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY = "nodeCacheLoadYellowThreshold";
     private static final String NODECACHE_LOAD_RED_THRESHOLD_CONFIG_PROPERTY = "nodeCacheLoadRedThreshold";
+    private static final String THREAD_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY = "threadLoadYellowThreshold";
+    private static final String THREAD_LOAD_RED_THRESHOLD_CONFIG_PROPERTY = "theadLoadRedThreshold";
+
+    @Reference
+    private LoadAverageService loadAverageService;
 
     @Override
     public ProbeStatus getStatus() {
 
-        double oneMinuteRequestLoadAverage = RequestLoadAverage.getInstance().getOneMinuteLoad();
-        double oneMinuteCurrentSessionLoad = JCRSessionLoadAverage.getInstance().getOneMinuteLoad();
-        double oneMinuteNodeCacheLoad = JCRNodeCacheLoadAverage.getInstance().getOneMinuteLoad();
+        double oneMinuteRequestLoadAverage =
+                loadAverageService.findValue(RequestLoadAverage.class.getName()).orElse(LoadAverageValue.EMPTY).getOneMinuteLoad();
+        double oneMinuteCurrentSessionLoad =
+                loadAverageService.findValue(JCRSessionLoadAverage.class.getName()).orElse(LoadAverageValue.EMPTY).getOneMinuteLoad();
+        double oneMinuteNodeCacheLoad =
+                loadAverageService.findValue(JCRNodeCacheLoadAverage.class.getName()).orElse(LoadAverageValue.EMPTY).getOneMinuteLoad();
+        double oneMinuteThreadLoad =
+                loadAverageService.findValue(ThreadLoadAverage.class.getName()).orElse(LoadAverageValue.EMPTY).getOneMinuteLoad();
 
         logger.debug("requestYellowThreshold: {}, requestRedThreshold: {}, sessionYellowThreshold: {}, sessionRedThreshold: {}",
                 requestLoadYellowThreshold,
                 requestLoadRedThreshold,
                 sessionLoadYellowThreshold,
                 sessionLoadRedThreshold);
 
-        if (oneMinuteRequestLoadAverage < requestLoadYellowThreshold && oneMinuteCurrentSessionLoad < sessionLoadYellowThreshold && oneMinuteNodeCacheLoad < nodeCacheLoadYellowThreshold) {
+        if (oneMinuteRequestLoadAverage < requestLoadYellowThreshold
+                && oneMinuteCurrentSessionLoad < sessionLoadYellowThreshold
+                && oneMinuteNodeCacheLoad < nodeCacheLoadYellowThreshold
+                && oneMinuteThreadLoad < threadLoadYellowThreshold) {
             return new ProbeStatus("Serverload is normal", ProbeStatus.Health.GREEN);
         }
-        if (oneMinuteRequestLoadAverage < requestLoadRedThreshold && oneMinuteCurrentSessionLoad < sessionLoadRedThreshold && oneMinuteNodeCacheLoad < nodeCacheLoadRedThreshold) {
+        if (oneMinuteRequestLoadAverage < requestLoadRedThreshold
+                && oneMinuteCurrentSessionLoad < sessionLoadRedThreshold
+                && oneMinuteNodeCacheLoad < nodeCacheLoadRedThreshold
+                && oneMinuteThreadLoad < threadLoadRedThreshold) {
             return new ProbeStatus("Serverload is above normal", ProbeStatus.Health.YELLOW);
         }
 
@@ -90,5 +112,11 @@ public void setConfig(Map<String, Object> config) {
         if (config.containsKey(NODECACHE_LOAD_RED_THRESHOLD_CONFIG_PROPERTY) && !StringUtils.isEmpty(String.valueOf(config.containsKey(NODECACHE_LOAD_RED_THRESHOLD_CONFIG_PROPERTY)))) {
             nodeCacheLoadRedThreshold = Integer.parseInt(String.valueOf(config.get(NODECACHE_LOAD_RED_THRESHOLD_CONFIG_PROPERTY)));
         }
+        if (config.containsKey(THREAD_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY) && !StringUtils.isEmpty(String.valueOf(config.containsKey(THREAD_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY)))) {
+            threadLoadYellowThreshold = Integer.parseInt(String.valueOf(config.get(THREAD_LOAD_YELLOW_THRESHOLD_CONFIG_PROPERTY)));
+        }
+        if (config.containsKey(THREAD_LOAD_RED_THRESHOLD_CONFIG_PROPERTY) && !StringUtils.isEmpty(String.valueOf(config.containsKey(THREAD_LOAD_RED_THRESHOLD_CONFIG_PROPERTY)))) {
+            threadLoadRedThreshold = Integer.parseInt(String.valueOf(config.get(THREAD_LOAD_RED_THRESHOLD_CONFIG_PROPERTY)));
+        }
     }
 }

src/main/java/org/jahia/modules/sam/load/LoadAverageProvider.java

@@ -0,0 +1,137 @@
+/*
+ * ==========================================================================================
+ * =                   JAHIA'S DUAL LICENSING - IMPORTANT INFORMATION                       =
+ * ==========================================================================================
+ *
+ *                                 http://www.jahia.com
+ *
+ *     Copyright (C) 2002-2023 Jahia Solutions Group SA. All rights reserved.
+ *
+ *     THIS FILE IS AVAILABLE UNDER TWO DIFFERENT LICENSES:
+ *     1/Apache2 OR 2/JSEL
+ *
+ *     1/ Apache2
+ *     ==================================================================================
+ *
+ *     Copyright (C) 2002-2023 Jahia Solutions Group SA. All rights reserved.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ *
+ *     2/ JSEL - Commercial and Supported Versions of the program
+ *     ===================================================================================
+ *
+ *     IF YOU DECIDE TO CHOOSE THE JSEL LICENSE, YOU MUST COMPLY WITH THE FOLLOWING TERMS:
+ *
+ *     Alternatively, commercial and supported versions of the program - also known as
+ *     Enterprise Distributions - must be used in accordance with the terms and conditions
+ *     contained in a separate written agreement between you and Jahia Solutions Group SA.
+ *
+ *     If you are unsure which license is appropriate for your use,
+ *     please contact the sales department at sales@jahia.com.
+ */
+package org.jahia.modules.sam.load;
+
+import org.jahia.tools.jvm.ThreadMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.jahia.modules.sam.load.LoadAverageValue.*;
+
+/**
+ * Abstract class that defines a specific average in the LoadAverageMonitor
+ * This class makes it easy to calculate a load average, using an average calculation like the following formula:
+ * load(t) = load(t – 1) e^(-5/60m) + n (1 – e^(-5/60m))
+ * where n = what we are evaluating over time (number of active threads, requests, etc...)
+ * and m = time in minutes over which to perform the average
+ *
+ */
+public abstract class LoadAverageProvider implements Runnable {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(LoadAverageProvider.class);
+
+    private final LoadAverageValue average;
+    private boolean threadDumpOnHighLoad;
+    private double loggingTriggerValue;
+    private double threadDumpTriggerValue;
+
+    public LoadAverageProvider(String name) {
+        average = new LoadAverageValue(name);
+        loggingTriggerValue = 0.0;
+        threadDumpTriggerValue = 0.0;
+    }
+
+    public abstract String getName();
+
+    public abstract double getValue();
+
+    public boolean isThreadDumpOnHighLoad() {
+        return threadDumpOnHighLoad;
+    }
+
+    public void setThreadDumpOnHighLoad(boolean threadDumpOnHighLoad) {
+        this.threadDumpOnHighLoad = threadDumpOnHighLoad;
+    }
+
+    public double getLoggingTriggerValue() {
+        return loggingTriggerValue;
+    }
+
+    public void setLoggingTriggerValue(double loggingTriggerValue) {
+        this.loggingTriggerValue = loggingTriggerValue;
+    }
+
+    public double getThreadDumpTriggerValue() {
+        return threadDumpTriggerValue;
+    }
+
+    public void setThreadDumpTriggerValue(double threadDumpTriggerValue) {
+        this.threadDumpTriggerValue = threadDumpTriggerValue;
+    }
+
+    public LoadAverageValue getAverage() {
+        return average;
+    }
+
+    public void run() {
+        if (getAverage().getLastRun() > 0) {
+            double calcFreqDouble = (System.currentTimeMillis() - getAverage().getLastRun()) / 1000d;
+            average.setOneMinuteLoad(average.getOneMinuteLoad() *
+                    Math.exp(-calcFreqDouble / (60.0 * ONE_MINUTE)) + getValue() * (1 - Math.exp(-calcFreqDouble / (60.0 * ONE_MINUTE))));
+            average.setFiveMinuteLoad(average.getFiveMinuteLoad() *
+                    Math.exp(-calcFreqDouble / (60.0 * FIVE_MINUTES)) + getValue() * (1 - Math.exp(-calcFreqDouble / (60.0 * FIVE_MINUTES))));
+            average.setFifteenMinuteLoad(average.getFifteenMinuteLoad() *
+                    Math.exp(-calcFreqDouble / (60.0 * FIFTEEN_MINUTES)) + getValue() * (1 - Math.exp(-calcFreqDouble / (60.0 * FIFTEEN_MINUTES))));
+            tickCallback();
+        }
+        getAverage().setLastRun(System.currentTimeMillis());
+    }
+
+    //TODO This callback is not the best option to trigger some log or dump about load average
+    // the load average should only be used to gather information about the system load and populate values
+    // a specific probe should exist to define a higher level of aggregation and trigger actions
+    // thus, an internal probe polling Executor should be able to check some probes (PollableProbe) and apply actions regarding thresholds
+    // this way, all trigger information included in LoadAverage should be removed to a specific probe
+    public void tickCallback() {
+        if (getLoggingTriggerValue() > 0 && average.getOneMinuteLoad() > getLoggingTriggerValue()) {
+            LOGGER.info(average.toString());
+            if (isThreadDumpOnHighLoad() && average.getOneMinuteLoad() > getThreadDumpTriggerValue()) {
+                ThreadMonitor.getInstance().dumpThreadInfo(false, true);
+            }
+        }
+    }
+
+    public String getInfo() {
+        return average.toString();
+    }
+}

src/main/java/org/jahia/modules/sam/load/LoadAverageService.java

@@ -0,0 +1,39 @@
+/*
+ * ==========================================================================================
+ * =                            JAHIA'S ENTERPRISE DISTRIBUTION                             =
+ * ==========================================================================================
+ *
+ *                                  http://www.jahia.com
+ *
+ * JAHIA'S ENTERPRISE DISTRIBUTIONS LICENSING - IMPORTANT INFORMATION
+ * ==========================================================================================
+ *
+ *     Copyright (C) 2002-2024 Jahia Solutions Group. All rights reserved.
+ *
+ *     This file is part of a Jahia's Enterprise Distribution.
+ *
+ *     Jahia's Enterprise Distributions must be used in accordance with the terms
+ *     contained in the Jahia Solutions Group Terms & Conditions as well as
+ *     the Jahia Sustainable Enterprise License (JSEL).
+ *
+ *     For questions regarding licensing, support, production usage...
+ *     please contact our team at sales@jahia.com or go to http://www.jahia.com/license.
+ *
+ * ==========================================================================================
+ */
+package org.jahia.modules.sam.load;
+
+import java.util.Optional;
+
+/**
+ * @author Jerome Blanchard
+ */
+public interface LoadAverageService {
+
+    Optional<LoadAverageValue> findValue(String providerClassname);
+
+    Optional<LoadAverageProvider> findProvider(String providerClassname);
+
+    String display();
+
+}