Skip to content

Commit

Permalink
Pluggable resource leak detector
Browse files Browse the repository at this point in the history
Allow users of Netty to plug in their own leak detector for the purpose
of instrumentation.

Motivation:

We are rolling out a large Netty deployment and want to be able to
track the amount of leaks we're seeing in production via custom
instrumentation. In order to achieve this today, I had to plug in a
custom `ByteBufAllocator` into the bootstrap and have it initialize a
custom `ResourceLeakDetector`. Due to these classes mostly being marked
`final` or having private or static methods, a lot of the code had to
be copy-pasted and it's quite ugly.

Modifications:

* I've added a static loader method for the `ResourceLeakDetector` in
`AbstractByteBuf` that tries to instantiate the class passed in via the
`-Dio.netty.customResourceLeakDetector`, otherwise falling back to the
default one.
* I've modified `ResourceLeakDetector` to be non-final and to have the
reporting broken out in to methods that can be overridden.

Result:

You can instrument leaks in your application by just adding something
like the following:

```java
public class InstrumentedResourceLeakDetector<T> extends
ResourceLeakDetector<T> {

    @monitor("InstanceLeakCounter")
    private final AtomicInteger instancesLeakCounter;

    @monitor("LeakCounter")
    private final AtomicInteger leakCounter;

    public InstrumentedResourceLeakDetector(Class<T> resource) {
        super(resource);
        this.instancesLeakCounter = new AtomicInteger();
        this.leakCounter = new AtomicInteger();
    }

    @OverRide
    protected void reportTracedLeak(String records) {
        super.reportTracedLeak(records);
        leakCounter.incrementAndGet();
    }

    @OverRide
    protected void reportUntracedLeak() {
        super.reportUntracedLeak();
        leakCounter.incrementAndGet();
    }

    @OverRide
    protected void reportInstancesLeak() {
        super.reportInstancesLeak();
        instancesLeakCounter.incrementAndGet();
    }
}
```
  • Loading branch information
artgon authored and normanmaurer committed Jun 20, 2016
1 parent f68517d commit cad22bb
Show file tree
Hide file tree
Showing 3 changed files with 163 additions and 15 deletions.
4 changes: 3 additions & 1 deletion buffer/src/main/java/io/netty/buffer/AbstractByteBuf.java
Expand Up @@ -17,6 +17,7 @@

import io.netty.util.IllegalReferenceCountException;
import io.netty.util.ResourceLeakDetector;
import io.netty.util.ResourceLeakDetectorFactory;
import io.netty.util.internal.PlatformDependent;
import io.netty.util.internal.StringUtil;
import io.netty.util.internal.SystemPropertyUtil;
Expand Down Expand Up @@ -49,7 +50,8 @@ public abstract class AbstractByteBuf extends ByteBuf {
}
}

static final ResourceLeakDetector<ByteBuf> leakDetector = new ResourceLeakDetector<ByteBuf>(ByteBuf.class);
static final ResourceLeakDetector<ByteBuf> leakDetector =
ResourceLeakDetectorFactory.instance().newResourceLeakDetector(ByteBuf.class);

int readerIndex;
int writerIndex;
Expand Down
52 changes: 38 additions & 14 deletions common/src/main/java/io/netty/util/ResourceLeakDetector.java
Expand Up @@ -34,7 +34,7 @@
import static io.netty.util.internal.StringUtil.NEWLINE;
import static io.netty.util.internal.StringUtil.simpleClassName;

public final class ResourceLeakDetector<T> {
public class ResourceLeakDetector<T> {

private static final String PROP_LEVEL_OLD = "io.netty.leakDetectionLevel";
private static final String PROP_LEVEL = "io.netty.leakDetection.level";
Expand Down Expand Up @@ -234,9 +234,7 @@ private void reportLeak(Level level) {
// Report too many instances.
int samplingInterval = level == Level.PARANOID? 1 : this.samplingInterval;
if (active * samplingInterval > maxActive && loggedTooManyActive.compareAndSet(false, true)) {
logger.error("LEAK: You are creating too many " + resourceType + " instances. " +
resourceType + " is a shared resource that must be reused across the JVM," +
"so that only a few instances are created.");
reportInstancesLeak(resourceType);
}

// Detect and report previous leaks.
Expand All @@ -256,22 +254,48 @@ private void reportLeak(Level level) {
String records = ref.toString();
if (reportedLeaks.putIfAbsent(records, Boolean.TRUE) == null) {
if (records.isEmpty()) {
logger.error("LEAK: {}.release() was not called before it's garbage-collected. " +
"Enable advanced leak reporting to find out where the leak occurred. " +
"To enable advanced leak reporting, " +
"specify the JVM option '-D{}={}' or call {}.setLevel() " +
"See http://netty.io/wiki/reference-counted-objects.html for more information.",
resourceType, PROP_LEVEL, Level.ADVANCED.name().toLowerCase(), simpleClassName(this));
reportUntracedLeak(resourceType);
} else {
logger.error(
"LEAK: {}.release() was not called before it's garbage-collected. " +
"See http://netty.io/wiki/reference-counted-objects.html for more information.{}",
resourceType, records);
reportTracedLeak(resourceType, records);
}
}
}
}

/**
* This method is called when a traced leak is detected. It can be overridden for tracking how many times leaks
* have been detected.
*/
protected void reportTracedLeak(String resourceType, String records) {
logger.error(
"LEAK: {}.release() was not called before it's garbage-collected. " +
"See http://netty.io/wiki/reference-counted-objects.html for more information.{}",
resourceType, records);
}

/**
* This method is called when an untraced leak is detected. It can be overridden for tracking how many times leaks
* have been detected.
*/
protected void reportUntracedLeak(String resourceType) {
logger.error("LEAK: {}.release() was not called before it's garbage-collected. " +
"Enable advanced leak reporting to find out where the leak occurred. " +
"To enable advanced leak reporting, " +
"specify the JVM option '-D{}={}' or call {}.setLevel() " +
"See http://netty.io/wiki/reference-counted-objects.html for more information.",
resourceType, PROP_LEVEL, Level.ADVANCED.name().toLowerCase(), simpleClassName(this));
}

/**
* This method is called when instance leaks are detected. It can be overridden for tracking how many times leaks
* have been detected.
*/
protected void reportInstancesLeak(String resourceType) {
logger.error("LEAK: You are creating too many " + resourceType + " instances. " +
resourceType + " is a shared resource that must be reused across the JVM," +
"so that only a few instances are created.");
}

private final class DefaultResourceLeak extends PhantomReference<Object> implements ResourceLeak {
private final String creationRecord;
private final Deque<String> lastRecords = new ArrayDeque<String>();
Expand Down
122 changes: 122 additions & 0 deletions common/src/main/java/io/netty/util/ResourceLeakDetectorFactory.java
@@ -0,0 +1,122 @@
/*
* Copyright 2016 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package io.netty.util;

import io.netty.util.internal.ObjectUtil;
import io.netty.util.internal.PlatformDependent;
import io.netty.util.internal.SystemPropertyUtil;
import io.netty.util.internal.logging.InternalLogger;
import io.netty.util.internal.logging.InternalLoggerFactory;

import java.lang.reflect.Constructor;
import java.security.AccessController;
import java.security.PrivilegedAction;

/**
* This static factory should be used to load {@link ResourceLeakDetector}s as needed
*/
public abstract class ResourceLeakDetectorFactory {
private static final InternalLogger logger = InternalLoggerFactory.getInstance(ResourceLeakDetectorFactory.class);

private static volatile ResourceLeakDetectorFactory factoryInstance = new DefaultResourceLeakDetectorFactory();

/**
* Get the singleton instance of this factory class.
*
* @return - the current {@link ResourceLeakDetectorFactory}
*/
public static ResourceLeakDetectorFactory instance() {
return factoryInstance;
}

/**
* Set the factory's singleton instance. This has to be called before the static initializer of the
* {@link ResourceLeakDetector} is called by all the callers of this factory. That is, before initializing a
* Netty Bootstrap.
*
* @param factory - the instance that will become the current {@link ResourceLeakDetectorFactory}'s singleton
*/
public static void setResourceLeakDetectorFactory(ResourceLeakDetectorFactory factory) {
factoryInstance = ObjectUtil.checkNotNull(factory, "factory");
}

/**
* Returns a new instance of a {@link ResourceLeakDetector} with the given resource class.
*
* @param resource - the resource class used to initialize the {@link ResourceLeakDetector}
* @param <T> - the type of the resource class
* @return - a new instance of {@link ResourceLeakDetector}
*/
public abstract <T> ResourceLeakDetector<T> newResourceLeakDetector(final Class<T> resource);

/**
* Default implementation that loads custom leak detector via system property
*/
private static final class DefaultResourceLeakDetectorFactory extends ResourceLeakDetectorFactory {

private final String customLeakDetector;
private final Constructor customClassConstructor;

public DefaultResourceLeakDetectorFactory() {
this.customLeakDetector = AccessController.doPrivileged(new PrivilegedAction<String>() {
@Override
public String run() {
return SystemPropertyUtil.get("io.netty.customResourceLeakDetector");
}
});

this.customClassConstructor = customClassConstructor();
}

private Constructor customClassConstructor() {
try {
if (customLeakDetector != null) {
final Class<?> detectorClass = Class.forName(customLeakDetector, true,
PlatformDependent.getSystemClassLoader());

if (ResourceLeakDetector.class.isAssignableFrom(detectorClass)) {
return detectorClass.getConstructor(Class.class);
} else {
logger.error("Class {} does not inherit from ResourceLeakDetector.", customLeakDetector);
}
}
} catch (Throwable t) {
logger.error("Could not load custom resource leak detector class provided: " + customLeakDetector, t);
}
return null;
}

@Override
public <T> ResourceLeakDetector<T> newResourceLeakDetector(final Class<T> resource) {
try {
if (customClassConstructor != null) {
ResourceLeakDetector<T> leakDetector =
(ResourceLeakDetector<T>) customClassConstructor.newInstance(resource);
logger.debug("Loaded custom ResourceLeakDetector: {}", customLeakDetector);
return leakDetector;
}
} catch (Throwable t) {
logger.error("Could not load custom resource leak detector provided: {} with the given resource: {}",
customLeakDetector, resource, t);
}

ResourceLeakDetector<T> resourceLeakDetector = new ResourceLeakDetector<T>(resource);
logger.debug("Loaded default ResourceLeakDetector: {}", resourceLeakDetector);
return resourceLeakDetector;
}
}
}

0 comments on commit cad22bb

Please sign in to comment.