Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cleanup JVM info and stats #10553

Closed
wants to merge 1 commit into from
Closed

Cleanup JVM info and stats #10553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 9 additions & 9 deletions src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
View file
Open in desktop
Expand Up @@ -548,13 +548,13 @@ public void addNodeInfoStats(NodeInfo nodeInfo, NodeStats nodeStats) {
if (js == null) {
return;
}
if (js.threads() != null) {
threads += js.threads().count();
if (js.getThreads() != null) {
threads += js.getThreads().getCount();
}
maxUptime = Math.max(maxUptime, js.uptime().millis());
if (js.mem() != null) {
heapUsed += js.mem().getHeapUsed().bytes();
heapMax += js.mem().getHeapMax().bytes();
maxUptime = Math.max(maxUptime, js.getUptime().millis());
if (js.getMem() != null) {
heapUsed += js.getMem().getHeapUsed().bytes();
heapMax += js.getMem().getHeapMax().bytes();
}
}

Expand Down Expand Up @@ -640,9 +640,9 @@ public static class JvmVersion implements Streamable {

JvmVersion(JvmInfo jvmInfo) {
version = jvmInfo.version();
vmName = jvmInfo.vmName();
vmVersion = jvmInfo.vmVersion();
vmVendor = jvmInfo.vmVendor();
vmName = jvmInfo.getVmName();
vmVersion = jvmInfo.getVmVersion();
vmVendor = jvmInfo.getVmVendor();
}

JvmVersion() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
View file
Open in desktop
Expand Up @@ -182,7 +182,7 @@ public static void main(String[] args) {
}

// warn if running using the client VM
if (JvmInfo.jvmInfo().vmName().toLowerCase(Locale.ROOT).contains("client")) {
if (JvmInfo.jvmInfo().getVmName().toLowerCase(Locale.ROOT).contains("client")) {
ESLogger logger = Loggers.getLogger(Bootstrap.class);
logger.warn("jvm uses the client vm, make sure to run `java` with the server vm for best performance by adding `-server` to the command line");
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
View file
Open in desktop
Expand Up @@ -168,9 +168,9 @@ public NettyHttpServerTransport(Settings settings, NetworkService networkService
this.detailedErrorsEnabled = settings.getAsBoolean(SETTING_HTTP_DETAILED_ERRORS_ENABLED, true);

long defaultReceiverPredictor = 512 * 1024;
if (JvmInfo.jvmInfo().mem().directMemoryMax().bytes() > 0) {
if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
// we can guess a better default...
long l = (long) ((0.3 * JvmInfo.jvmInfo().mem().directMemoryMax().bytes()) / workerCount);
long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / workerCount);
defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
}

Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java
View file
Open in desktop
Expand Up @@ -77,7 +77,7 @@ public IndexingMemoryController(Settings settings, ThreadPool threadPool, Indice
String indexingBufferSetting = this.settings.get("indices.memory.index_buffer_size", "10%");
if (indexingBufferSetting.endsWith("%")) {
double percent = Double.parseDouble(indexingBufferSetting.substring(0, indexingBufferSetting.length() - 1));
indexingBuffer = new ByteSizeValue((long) (((double) JvmInfo.jvmInfo().mem().heapMax().bytes()) * (percent / 100)));
indexingBuffer = new ByteSizeValue((long) (((double) JvmInfo.jvmInfo().getMem().getHeapMax().bytes()) * (percent / 100)));
ByteSizeValue minIndexingBuffer = this.settings.getAsBytesSize("indices.memory.min_index_buffer_size", new ByteSizeValue(48, ByteSizeUnit.MB));
ByteSizeValue maxIndexingBuffer = this.settings.getAsBytesSize("indices.memory.max_index_buffer_size", null);

Expand All @@ -99,7 +99,7 @@ public IndexingMemoryController(Settings settings, ThreadPool threadPool, Indice
String translogBufferSetting = this.settings.get("indices.memory.translog_buffer_size", "1%");
if (translogBufferSetting.endsWith("%")) {
double percent = Double.parseDouble(translogBufferSetting.substring(0, translogBufferSetting.length() - 1));
translogBuffer = new ByteSizeValue((long) (((double) JvmInfo.jvmInfo().mem().heapMax().bytes()) * (percent / 100)));
translogBuffer = new ByteSizeValue((long) (((double) JvmInfo.jvmInfo().getMem().getHeapMax().bytes()) * (percent / 100)));
ByteSizeValue minTranslogBuffer = this.settings.getAsBytesSize("indices.memory.min_translog_buffer_size", new ByteSizeValue(256, ByteSizeUnit.KB));
ByteSizeValue maxTranslogBuffer = this.settings.getAsBytesSize("indices.memory.max_translog_buffer_size", null);

Expand Down
90 changes: 14 additions & 76 deletions src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
View file
Open in desktop
Expand Up @@ -208,76 +208,40 @@ public int versionUpdatePack() {
}
}

public String vmName() {
return vmName;
}

public String getVmName() {
return vmName;
}

public String vmVersion() {
return vmVersion;
return this.vmName;
}

public String getVmVersion() {
return vmVersion;
}

public String vmVendor() {
return vmVendor;
return this.vmVersion;
}

public String getVmVendor() {
return vmVendor;
}

public long startTime() {
return startTime;
return this.vmVendor;
}

public long getStartTime() {
return startTime;
}

public Mem mem() {
return mem;
return this.startTime;
}

public Mem getMem() {
return mem();
}

public String[] inputArguments() {
return inputArguments;
return this.mem;
}

public String[] getInputArguments() {
return inputArguments;
}

public String bootClassPath() {
return bootClassPath;
return this.inputArguments;
}

public String getBootClassPath() {
return bootClassPath;
}

public String classPath() {
return classPath;
return this.bootClassPath;
}

public String getClassPath() {
return classPath;
}

public Map<String, String> systemProperties() {
return systemProperties;
return this.classPath;
}

public Map<String, String> getSystemProperties() {
return systemProperties;
return this.systemProperties;
}

@Override
Expand Down Expand Up @@ -396,50 +360,24 @@ public static class Mem implements Streamable, Serializable {
Mem() {
}

public ByteSizeValue heapInit() {
return new ByteSizeValue(heapInit);
}

public ByteSizeValue getHeapInit() {
return heapInit();
}

public ByteSizeValue heapMax() {
return new ByteSizeValue(heapMax);
return new ByteSizeValue(heapInit);
}

public ByteSizeValue getHeapMax() {
return heapMax();
}

public ByteSizeValue nonHeapInit() {
return new ByteSizeValue(nonHeapInit);
return new ByteSizeValue(heapMax);
}

public ByteSizeValue getNonHeapInit() {
return nonHeapInit();
}

public ByteSizeValue nonHeapMax() {
return new ByteSizeValue(nonHeapMax);
return new ByteSizeValue(nonHeapInit);
}

public ByteSizeValue getNonHeapMax() {
return nonHeapMax();
}

public ByteSizeValue directMemoryMax() {
return new ByteSizeValue(directMemoryMax);
return new ByteSizeValue(nonHeapMax);
}

public ByteSizeValue getDirectMemoryMax() {
return directMemoryMax();
}

public static Mem readMem(StreamInput in) throws IOException {
Mem mem = new Mem();
mem.readFrom(in);
return mem;
return new ByteSizeValue(directMemoryMax);
}

@Override
Expand Down
50 changes: 11 additions & 39 deletions src/main/java/org/elasticsearch/monitor/jvm/JvmMonitorService.java
View file
Open in desktop
Expand Up @@ -29,9 +29,7 @@
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.threadpool.ThreadPool;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;

import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
Expand Down Expand Up @@ -107,7 +105,7 @@ public JvmMonitorService(Settings settings, ThreadPool threadPool) {

this.gcThresholds = gcThresholds.immutableMap();

logger.debug("enabled [{}], last_gc_enabled [{}], interval [{}], gc_threshold [{}]", enabled, JvmStats.isLastGcEnabled(), interval, this.gcThresholds);
logger.debug("enabled [{}], interval [{}], gc_threshold [{}]", enabled, interval, this.gcThresholds);
}

@Override
Expand All @@ -133,18 +131,14 @@ protected void doClose() throws ElasticsearchException {
private class JvmMonitor implements Runnable {

private JvmStats lastJvmStats = jvmStats();

private long seq = 0;

private final Set<DeadlockAnalyzer.Deadlock> lastSeenDeadlocks = new HashSet<>();

public JvmMonitor() {
}

@Override
public void run() {
try {
// monitorDeadlock();
monitorLongGc();
} catch (Throwable t) {
logger.debug("failed to monitor", t);
Expand All @@ -155,8 +149,8 @@ private synchronized void monitorLongGc() {
seq++;
JvmStats currentJvmStats = jvmStats();

for (int i = 0; i < currentJvmStats.gc().collectors().length; i++) {
GarbageCollector gc = currentJvmStats.gc().collectors()[i];
for (int i = 0; i < currentJvmStats.getGc().getCollectors().length; i++) {
GarbageCollector gc = currentJvmStats.getGc().getCollectors()[i];
GarbageCollector prevGc = lastJvmStats.gc.collectors[i];

// no collection has happened
Expand All @@ -169,61 +163,39 @@ private synchronized void monitorLongGc() {
continue;
}

GcThreshold gcThreshold = gcThresholds.get(gc.name());
GcThreshold gcThreshold = gcThresholds.get(gc.getName());
if (gcThreshold == null) {
gcThreshold = gcThresholds.get("default");
}

if (gc.lastGc() != null && prevGc.lastGc() != null) {
GarbageCollector.LastGc lastGc = gc.lastGc();
if (lastGc.startTime == prevGc.lastGc().startTime()) {
// we already handled this one...
continue;
}
// Ignore any duration > 1hr; getLastGcInfo occasionally returns total crap
if (lastGc.duration().hoursFrac() > 1) {
continue;
}
if (lastGc.duration().millis() > gcThreshold.warnThreshold) {
logger.warn("[last_gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], reclaimed [{}], leaving [{}][{}]/[{}]",
gc.name(), seq, gc.getCollectionCount(), lastGc.duration(), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max());
} else if (lastGc.duration().millis() > gcThreshold.infoThreshold) {
logger.info("[last_gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], reclaimed [{}], leaving [{}]/[{}]",
gc.name(), seq, gc.getCollectionCount(), lastGc.duration(), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max());
} else if (lastGc.duration().millis() > gcThreshold.debugThreshold && logger.isDebugEnabled()) {
logger.debug("[last_gc][{}][{}][{}] duration [{}], collections [{}], total [{}]/[{}], reclaimed [{}], leaving [{}]/[{}]",
gc.name(), seq, gc.getCollectionCount(), lastGc.duration(), collections, TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastGc.reclaimed(), lastGc.afterUsed(), lastGc.max());
}
}

long avgCollectionTime = collectionTime / collections;

if (avgCollectionTime > gcThreshold.warnThreshold) {
logger.warn("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.timestamp() - lastJvmStats.timestamp()), TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax(), buildPools(lastJvmStats, currentJvmStats));
gc.getName(), seq, gc.getCollectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.getTimestamp() - lastJvmStats.getTimestamp()), TimeValue.timeValueMillis(collectionTime), gc.getCollectionTime(), lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), JvmInfo.jvmInfo().getMem().getHeapMax(), buildPools(lastJvmStats, currentJvmStats));
} else if (avgCollectionTime > gcThreshold.infoThreshold) {
logger.info("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.timestamp() - lastJvmStats.timestamp()), TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax(), buildPools(lastJvmStats, currentJvmStats));
gc.getName(), seq, gc.getCollectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.getTimestamp() - lastJvmStats.getTimestamp()), TimeValue.timeValueMillis(collectionTime), gc.getCollectionTime(), lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), JvmInfo.jvmInfo().getMem().getHeapMax(), buildPools(lastJvmStats, currentJvmStats));
} else if (avgCollectionTime > gcThreshold.debugThreshold && logger.isDebugEnabled()) {
logger.debug("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
gc.name(), seq, gc.collectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.timestamp() - lastJvmStats.timestamp()), TimeValue.timeValueMillis(collectionTime), gc.collectionTime(), lastJvmStats.mem().heapUsed(), currentJvmStats.mem().heapUsed(), JvmInfo.jvmInfo().mem().heapMax(), buildPools(lastJvmStats, currentJvmStats));
gc.getName(), seq, gc.getCollectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.getTimestamp() - lastJvmStats.getTimestamp()), TimeValue.timeValueMillis(collectionTime), gc.getCollectionTime(), lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), JvmInfo.jvmInfo().getMem().getHeapMax(), buildPools(lastJvmStats, currentJvmStats));
}
}
lastJvmStats = currentJvmStats;
}

private String buildPools(JvmStats prev, JvmStats current) {
StringBuilder sb = new StringBuilder();
for (JvmStats.MemoryPool currentPool : current.mem()) {
for (JvmStats.MemoryPool currentPool : current.getMem()) {
JvmStats.MemoryPool prevPool = null;
for (JvmStats.MemoryPool pool : prev.mem()) {
for (JvmStats.MemoryPool pool : prev.getMem()) {
if (pool.getName().equals(currentPool.getName())) {
prevPool = pool;
break;
}
}
sb.append("{[").append(currentPool.name())
.append("] [").append(prevPool == null ? "?" : prevPool.used()).append("]->[").append(currentPool.used()).append("]/[").append(currentPool.getMax()).append("]}");
sb.append("{[").append(currentPool.getName())
.append("] [").append(prevPool == null ? "?" : prevPool.getUsed()).append("]->[").append(currentPool.getUsed()).append("]/[").append(currentPool.getMax()).append("]}");
}
return sb.toString();
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
View file
Open in desktop
Expand Up @@ -51,7 +51,7 @@ public JvmInfo info() {
}

public synchronized JvmStats stats() {
if ((System.currentTimeMillis() - jvmStats.timestamp()) > refreshInterval.millis()) {
if ((System.currentTimeMillis() - jvmStats.getTimestamp()) > refreshInterval.millis()) {
jvmStats = JvmStats.jvmStats();
}
return jvmStats;
Expand Down