Skip to content

Commit

Permalink
Try to load native linux libraries with matching classifier first (#9411
Browse files Browse the repository at this point in the history 
)

Motivation:

Users' runtime systems may have incompatible dynamic libraries to the ones our
tcnative wrappers link to. Unfortunately, we cannot determine and catch these
scenarios (in which the JVM crashes) but we can make a more educated guess on
what library to load and try to find one that works better before crashing.

Modifications:

1) Build dynamically linked openSSL builds for more OSs (netty-tcnative)
2) Load native linux libraries with matching classifier (first)

Result:

More developers / users can use the dynamically-linked native libraries.
  • Loading branch information
@NicoK @normanmaurer
NicoK authored and normanmaurer committed Aug 12, 2019
1 parent bcf6d56 commit 8d9cea2
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 4 deletions.
82 changes: 82 additions & 0 deletions common/src/main/java/io/netty/util/internal/PlatformDependent.java
View file
Expand Up @@ -15,6 +15,7 @@
*/
package io.netty.util.internal;

import io.netty.util.CharsetUtil;
import io.netty.util.internal.logging.InternalLogger;
import io.netty.util.internal.logging.InternalLoggerFactory;
import org.jctools.queues.MpscArrayQueue;
Expand All @@ -28,19 +29,28 @@
import org.jctools.util.Pow2;
import org.jctools.util.UnsafeAccess;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Queue;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentMap;
Expand Down Expand Up @@ -97,6 +107,10 @@ public final class PlatformDependent {
private static final String NORMALIZED_ARCH = normalizeArch(SystemPropertyUtil.get("os.arch", ""));
private static final String NORMALIZED_OS = normalizeOs(SystemPropertyUtil.get("os.name", ""));

// keep in sync with maven's pom.xml via os.detection.classifierWithLikes!
private static final String[] ALLOWED_LINUX_OS_CLASSIFIERS = {"fedora", "suse", "arch"};
private static final Set<String> LINUX_OS_CLASSIFIERS;

private static final int ADDRESS_SIZE = addressSize0();
private static final boolean USE_DIRECT_BUFFER_NO_CLEANER;
private static final AtomicLong DIRECT_MEMORY_COUNTER;
Expand Down Expand Up @@ -196,6 +210,50 @@ public Random current() {
"Unless explicitly requested, heap buffer will always be preferred to avoid potential system " +
"instability.");
}

// For specifications, see https://www.freedesktop.org/software/systemd/man/os-release.html
final String[] OS_RELEASE_FILES = {"/etc/os-release", "/usr/lib/os-release"};
final String LINUX_ID_PREFIX = "ID=";
final String LINUX_ID_LIKE_PREFIX = "ID_LIKE=";
Set<String> allowedClassifiers = new HashSet<String>(Arrays.asList(ALLOWED_LINUX_OS_CLASSIFIERS));
allowedClassifiers = Collections.unmodifiableSet(allowedClassifiers);
Set<String> availableClassifiers = new LinkedHashSet<String>();

for (String osReleaseFileName : OS_RELEASE_FILES) {
final File file = new File(osReleaseFileName);
if (file.exists()) {
BufferedReader reader = null;
try {
reader = new BufferedReader(
new InputStreamReader(
new FileInputStream(file), CharsetUtil.UTF_8));

String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith(LINUX_ID_PREFIX)) {
String id = normalizeOsReleaseVariableValue(line.substring(LINUX_ID_PREFIX.length()));
addClassifier(allowedClassifiers, availableClassifiers, id);
} else if (line.startsWith(LINUX_ID_LIKE_PREFIX)) {
line = normalizeOsReleaseVariableValue(line.substring(LINUX_ID_LIKE_PREFIX.length()));
addClassifier(allowedClassifiers, availableClassifiers, line.split("[ ]+"));
}
}
} catch (IOException ignored) {
// Ignore
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException ignored) {
// Ignore
}
}
}
// specification states we should only fall back if /etc/os-release does not exist
break;
}
}
LINUX_OS_CLASSIFIERS = Collections.unmodifiableSet(availableClassifiers);
}

public static boolean hasDirectBufferNoCleanerConstructor() {
Expand Down Expand Up @@ -1274,6 +1332,30 @@ public static String normalizedOs() {
return NORMALIZED_OS;
}

public static Set<String> normalizedLinuxClassifiers() {
return LINUX_OS_CLASSIFIERS;
}

/**
* Adds only those classifier strings to <tt>dest</tt> which are present in <tt>allowed</tt>.
*
* @param allowed allowed classifiers
* @param dest destination set
* @param maybeClassifiers potential classifiers to add
*/
private static void addClassifier(Set<String> allowed, Set<String> dest, String... maybeClassifiers) {
for (String id : maybeClassifiers) {
if (allowed.contains(id)) {
dest.add(id);
}
}
}

private static String normalizeOsReleaseVariableValue(String value) {
// Variable assignment values may be enclosed in double or single quotes.
return value.trim().replaceAll("[\"']", "");
}

private static String normalize(String value) {
return value.toLowerCase(Locale.US).replaceAll("[^a-z0-9]+", "");
}
Expand Down
16 changes: 13 additions & 3 deletions handler/src/main/java/io/netty/handler/ssl/OpenSsl.java
View file
Expand Up @@ -552,15 +552,25 @@ private static void loadTcNative() throws Exception {
String os = PlatformDependent.normalizedOs();
String arch = PlatformDependent.normalizedArch();

Set<String> libNames = new LinkedHashSet<String>(4);
Set<String> libNames = new LinkedHashSet<String>(5);
String staticLibName = "netty_tcnative";

// First, try loading the platform-specific library. Platform-specific
// libraries will be available if using a tcnative uber jar.
libNames.add(staticLibName + "_" + os + '_' + arch);
if ("linux".equalsIgnoreCase(os)) {
// Fedora SSL lib so naming (libssl.so.10 vs libssl.so.1.0.0)..
Set<String> classifiers = PlatformDependent.normalizedLinuxClassifiers();
for (String classifier : classifiers) {
libNames.add(staticLibName + "_" + os + '_' + arch + "_" + classifier);
}
// generic arch-dependent library
libNames.add(staticLibName + "_" + os + '_' + arch);

// Fedora SSL lib so naming (libssl.so.10 vs libssl.so.1.0.0).
// note: should already be included from the classifiers but if not, we use this as an
// additional fallback option here
libNames.add(staticLibName + "_" + os + '_' + arch + "_fedora");
} else {
libNames.add(staticLibName + "_" + os + '_' + arch);
}
libNames.add(staticLibName + "_" + arch);
libNames.add(staticLibName);
Expand Down
3 changes: 2 additions & 1 deletion pom.xml
View file
Expand Up @@ -296,8 +296,9 @@
<argLine.javaProperties>-D_</argLine.javaProperties>
<!-- Configure the os-maven-plugin extension to expand the classifier on -->
<!-- Fedora-"like" systems. This is currently only used for the netty-tcnative dependency -->
<os.detection.classifierWithLikes>fedora</os.detection.classifierWithLikes>
<osmaven.version>1.6.2</osmaven.version>
<!-- keep in sync with PlatformDependent#ALLOWED_LINUX_OS_CLASSIFIERS -->
<os.detection.classifierWithLikes>fedora,suse,arch</os.detection.classifierWithLikes>
<tcnative.artifactId>netty-tcnative</tcnative.artifactId>
<tcnative.version>2.0.25.Final</tcnative.version>
<tcnative.classifier>${os.detected.classifier}</tcnative.classifier>
Expand Down

0 comments on commit 8d9cea2

Please sign in to comment.