From 91e1b3bb6bbfffc62401025a24231cd388992d7c Mon Sep 17 00:00:00 2001 From: Nileema Shingte Date: Mon, 29 Feb 2016 11:30:56 -0800 Subject: [PATCH] Add workaround for CodeCache not being collected effectively This problem is seen in jdk8u45+ where the CodeCache becomes full causing the JIT compiler to be disabled. We'd seen this problem earlier with Java 7 and implemented this as a workaround. Reviving the old commit with minor changes. This reverts commit da9df1eb7f797f8e287fe8e51dcdcbd983d2a85e. --- .../presto/server/CodeCacheGcTrigger.java | 76 +++++++++++++++++++ .../facebook/presto/server/PrestoServer.java | 3 + 2 files changed, 79 insertions(+) create mode 100644 presto-main/src/main/java/com/facebook/presto/server/CodeCacheGcTrigger.java diff --git a/presto-main/src/main/java/com/facebook/presto/server/CodeCacheGcTrigger.java b/presto-main/src/main/java/com/facebook/presto/server/CodeCacheGcTrigger.java new file mode 100644 index 000000000000..c7051b9a5ae6 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/server/CodeCacheGcTrigger.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.server; + +import io.airlift.log.Logger; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryPoolMXBean; +import java.util.concurrent.atomic.AtomicBoolean; + +final class CodeCacheGcTrigger +{ + private static final Logger log = Logger.get(CodeCacheGcTrigger.class); + private static final AtomicBoolean installed = new AtomicBoolean(); + + private CodeCacheGcTrigger() {} + + public static void installCodeCacheGcTrigger() + { + if (installed.getAndSet(true)) { + return; + } + + // Hack to work around bugs in java 8 (8u45+) related to code cache management. + // See http://openjdk.5641.n7.nabble.com/JIT-stops-compiling-after-a-while-java-8u45-td259603.html for more info. + MemoryPoolMXBean codeCacheMbean = findCodeCacheMBean(); + + Thread gcThread = new Thread(() -> { + while (!Thread.currentThread().isInterrupted()) { + long used = codeCacheMbean.getUsage().getUsed(); + long max = codeCacheMbean.getUsage().getMax(); + + if (used > 0.95 * max) { + log.error("Code Cache is more than 95% full. JIT may stop working."); + } + if (used > 0.7 * max) { + // Due to some obscure bug in hotspot (java 8), once the code cache fills up the JIT stops compiling + // By forcing a GC, we let the code cache evictor make room before the cache fills up. + log.info("Triggering GC to avoid Code Cache eviction bugs"); + System.gc(); + } + + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }); + gcThread.setDaemon(true); + gcThread.setName("Code-Cache-GC-Trigger"); + gcThread.start(); + } + + private static MemoryPoolMXBean findCodeCacheMBean() + { + for (MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) { + if (bean.getName().equals("Code Cache")) { + return bean; + } + } + throw new RuntimeException("Could not obtain a reference to the 'Code Cache' MemoryPoolMXBean"); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 99c68965e45c..de138dc25cba 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -53,6 +53,7 @@ import java.util.Set; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.LEGACY_NETWORK_TOPOLOGY; +import static com.facebook.presto.server.CodeCacheGcTrigger.installCodeCacheGcTrigger; import static com.facebook.presto.server.ConditionalModule.installModuleIf; import static com.facebook.presto.server.PrestoSystemRequirements.verifyJvmRequirements; import static com.facebook.presto.server.PrestoSystemRequirements.verifySystemTimeIsReasonable; @@ -142,6 +143,8 @@ public void run() injector.getInstance(Announcer.class).start(); log.info("======== SERVER STARTED ========"); + + installCodeCacheGcTrigger(); } catch (Throwable e) { log.error(e);