Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DRILL-6349: Drill JDBC driver fails on Java 1.9+ with NoClassDefFoundError: sun/misc/VM #1446

Closed
wants to merge 1 commit into from
Closed

DRILL-6349: Drill JDBC driver fails on Java 1.9+ with NoClassDefFoundError: sun/misc/VM #1446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion common/src/main/java/org/apache/drill/common/config/DrillConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Properties;
import java.util.concurrent.TimeUnit;

import io.netty.util.internal.PlatformDependent;
import org.apache.drill.common.exceptions.DrillConfigurationException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.scanner.ClassPathScanner;
Expand All @@ -46,7 +47,7 @@ public class DrillConfig extends NestedConfig {
private final ImmutableList<String> startupArguments;

@SuppressWarnings("restriction")
private static final long MAX_DIRECT_MEMORY = sun.misc.VM.maxDirectMemory();
private static final long MAX_DIRECT_MEMORY = PlatformDependent.maxDirectMemory();

@VisibleForTesting
public DrillConfig(Config config) {
Expand Down
3 changes: 2 additions & 1 deletion contrib/storage-jdbc/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@
<!-- Because the JDBC tests are somewhat heavyweight, we only run them in the 'verify' phase -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.18.1</version>
<version>2.22.0</version>
<configuration>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please also set <useSystemClassLoader>false</useSystemClassLoader> for maven-failsafe-plugin, since it also fails.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@vvysotskyi , done

<forkCount combine.self="override">1</forkCount>
<systemPropertyVariables>
Expand All @@ -115,6 +115,7 @@
<includes>
<include>**/*IT.java</include>
</includes>
<useSystemClassLoader>false</useSystemClassLoader>
</configuration>
<executions>
<execution>
Expand Down
11 changes: 11 additions & 0 deletions distribution/src/resources/drill-config.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,17 @@ export DRILLBIT_CODE_CACHE_SIZE=${DRILLBIT_CODE_CACHE_SIZE:-"1G"}
export DRILLBIT_OPTS="-Xms$DRILL_HEAP -Xmx$DRILL_HEAP -XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"
export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:ReservedCodeCacheSize=$DRILLBIT_CODE_CACHE_SIZE -Ddrill.exec.enable-epoll=false"

# Check that java is newer than 1.8
"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.8" > /dev/null
if [ $? -gt 0 ]; then
# Allow reflective access on Java 9+
export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.base/sun.nio.ch=ALL-UNNAMED"
export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.base/java.nio=ALL-UNNAMED"
export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED"
export DRILLBIT_OPTS="$DRILLBIT_OPTS --illegal-access=permit"
fi


# Under YARN, the log directory is usually YARN-provided. Replace any
# value that may have been set in drill-env.sh.
Expand Down
12 changes: 12 additions & 0 deletions distribution/src/resources/sqlline.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,18 @@ set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% -Dlog.path="%DRILL_LOG_DIR%\sq
SET JAVA_CMD=%JAVA_HOME%\bin\%JAVA_EXE%
if "%JAVA_HOME%" == "" (set JAVA_CMD=%JAVA_EXE%)
set ERROR_CODE=0

rem Check that java is newer than 1.8
"%JAVA_CMD%" -version 2>&1 | findstr "1.8" > nul 2>&1
if errorlevel 1 (
rem allow reflective access on Java 9+
set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.base/java.lang=ALL-UNNAMED
set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.base/sun.nio.ch=ALL-UNNAMED
set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.base/java.nio=ALL-UNNAMED
set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED
set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --illegal-access=permit
)

set SQLLINE_CALL=sqlline.SqlLine -ac org.apache.drill.exec.client.DrillSqlLineApplication -d org.apache.drill.jdbc.Driver
if NOT "test%QUERY%"=="test" (
echo %QUERY% | "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" %SQLLINE_CALL% %DRILL_ARGS%
Expand Down
20 changes: 7 additions & 13 deletions exec/java-exec/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,28 +64,22 @@
<version>${kerby.version}</version>
<scope>test</scope>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.ow2.asm</groupId> -->
<!-- <artifactId>asm-util</artifactId> -->
<!-- <version>5.0.3</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-debug-all</artifactId>
<version>5.0.3</version>
<!-- <scope>test</scope> -->
<artifactId>asm-commons</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-util</artifactId>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see any new dependencies being introduced as part of the PR, why is this necessary? The same for all other newly introduced dependencies.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New asm version supports jdk 10, but there asm-debug-all jar was removed (see https://gitlab.ow2.org/asm/asm/commit/4c449bea1e3fb980238c61d60103fe1be76e77e5), so new dependencies, where classes used by Drill are available were introduced.

If I understand correctly, jaxb-api and activation dependencies were added due to http://openjdk.java.net/jeps/320.
@oleg-zinovev, Is it make sence to move these two dependencies to the root pom?

Also, please remove commented out lines introduced in previous commits and move version for asm dependencies to the property in the root pom.

<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.apache.drill.contrib.data</groupId>
<artifactId>tpch-sample-data</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.ow2.asm</groupId> -->
<!-- <artifactId>asm-commons</artifactId> -->
<!-- <version>5.0.3</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion exec/java-exec/src/main/java/org/apache/drill/exec/compile/CompilationConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@ public class CompilationConfig {
* Never use asm.Opcodes values directly in calls that require them. Use ASM_OPCODES
* instead, so that we can change it here once for all references.
*/
public final static int ASM_API_VERSION = Opcodes.ASM5;
public final static int ASM_API_VERSION = Opcodes.ASM7;
}
2 changes: 2 additions & 0 deletions .../java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import java.util.List;

import org.apache.drill.exec.compile.CompilationConfig;
import org.objectweb.asm.Opcodes;
import org.objectweb.asm.Type;
import org.objectweb.asm.tree.AbstractInsnNode;
Expand All @@ -40,6 +41,7 @@ public class ReplacingInterpreter extends BasicInterpreter {
private final List<ReplacingBasicValue> valueList;

public ReplacingInterpreter(final String className, final List<ReplacingBasicValue> valueList) {
super(CompilationConfig.ASM_API_VERSION);
this.className = className;
this.valueList = valueList;
}
Expand Down
4 changes: 2 additions & 2 deletions exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ public static MappingSet getDefaultMapping() {
// from the JDK Modifier values to the JCodeModel JMod values: the
// values are different.

int mods = JMod.PRIVATE + JMod.FINAL;
int mods = JMod.FINAL;
if ((innerClass.getModifiers() & Modifier.STATIC) != 0) {
mods += JMod.STATIC;
}
Expand Down Expand Up @@ -370,7 +370,7 @@ private boolean createNestedClass() {
// all new fields will be declared in the class from innerClassGenerator
if (innerClassGenerator == null) {
try {
JDefinedClass innerClazz = clazz._class(JMod.PRIVATE, clazz.name() + "0");
JDefinedClass innerClazz = clazz._class(JMod.NONE, clazz.name() + "0");
innerClassGenerator = new ClassGenerator<>(codeGenerator, mappings, sig, evaluationVisitor, innerClazz, model, optionManager);
} catch (JClassAlreadyExistsException e) {
throw new DrillRuntimeException(e);
Expand Down
2 changes: 1 addition & 1 deletion .../java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ protected void createPartitionSublists() {

// build a list of DFSDirPartitionLocation.
for (final List<String> dirs : dirToFileMap.keySet()) {
locations.add( new DFSDirPartitionLocation((String [])dirs.toArray(), dirToFileMap.get(dirs)));
locations.add( new DFSDirPartitionLocation(dirs.toArray(new String[dirs.size()]), dirToFileMap.get(dirs)));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this change necessary? As I already mentioned, I don't see why is it necessary to go back and forth between an array and a list in this method, so I would prefer not to introduce an unrelated change into the PR and handle it in a separate PR/JIRA.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://bugs.java.com/view_bug.do?bug_id=6260652
Result type of Arrays.ArrayList::toArray() was changed from T[] to Object[] in JDK 9+.
So, FileSystemPartitionDescriptor::createPartitionSublists() doesn't work if Drill running on JDK9+.

}

locationSuperList = Lists.partition(locations, PartitionDescriptor.PARTITION_BATCH_SIZE);
Expand Down
3 changes: 0 additions & 3 deletions exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,16 @@

import mockit.Mock;
import mockit.MockUp;
import mockit.integration.junit4.JMockit;
import org.apache.calcite.util.Util;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.runner.RunWith;

import java.nio.charset.Charset;

import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertThat;

@RunWith(JMockit.class)
public class TestUtf8SupportInQueryString extends BaseTestQuery {

@Test
Expand Down
4 changes: 0 additions & 4 deletions exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,13 @@
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;

import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;

import mockit.integration.junit4.JMockit;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.hasItem;

@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestCastFunctions extends BaseTestQuery {

Expand Down
3 changes: 0 additions & 3 deletions exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
import org.apache.drill.shaded.guava.com.google.common.io.Files;
import mockit.integration.junit4.JMockit;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.util.DrillFileUtils;
Expand All @@ -37,12 +36,10 @@
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestDateFunctions extends PopUnitTestBase {

Expand Down
3 changes: 0 additions & 3 deletions exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,13 @@
*/
package org.apache.drill.exec.fn.impl.testing;

import mockit.integration.junit4.JMockit;
import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
Expand All @@ -37,7 +35,6 @@
import static org.hamcrest.CoreMatchers.startsWith;
import static org.junit.Assert.assertThat;

@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestDateConversions extends BaseTestQuery {

Expand Down
3 changes: 0 additions & 3 deletions exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,16 @@

import org.apache.drill.shaded.guava.com.google.common.base.Joiner;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import mockit.integration.junit4.JMockit;
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.io.File;
import java.io.PrintWriter;
import java.util.List;

@RunWith(JMockit.class)
@Category(SqlTest.class)
public class TestConstantFolding extends PlanTestBase {

Expand Down
4 changes: 2 additions & 2 deletions ...ava-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ public void testCreateViewInWSWithNoPermissionsForQueryUser() throws Exception {

final String query = "CREATE VIEW " + viewName + " AS SELECT " +
"c_custkey, c_nationkey FROM cp.`tpch/customer.parquet` ORDER BY c_custkey;";
final String expErrorMsg = "PERMISSION ERROR: Permission denied: user=drillTestUser2, access=WRITE, inode=\"/drill_test_grp_0_755/";
final String expErrorMsg = "PERMISSION ERROR: Permission denied: user=drillTestUser2, access=WRITE, inode=\"/drill_test_grp_0_755";
errorMsgTestHelper(query, expErrorMsg);

// SHOW TABLES is expected to return no records as view creation fails above.
Expand Down Expand Up @@ -351,7 +351,7 @@ public void testCreateTableInWSWithNoPermissionsForQueryUser() throws Exception

thrown.expect(UserRemoteException.class);
thrown.expectMessage(containsString("Permission denied: user=drillTestUser2, " +
"access=WRITE, inode=\"/drill_test_grp_0_755/"));
"access=WRITE, inode=\"/drill_test_grp_0_755"));

test("CREATE TABLE %s AS SELECT c_custkey, c_nationkey " +
"FROM cp.`tpch/customer.parquet` ORDER BY c_custkey", tableName);
Expand Down
4 changes: 0 additions & 4 deletions exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,12 @@
import org.apache.drill.test.TestBuilder;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;

import org.apache.drill.shaded.guava.com.google.common.collect.Lists;

import mockit.integration.junit4.JMockit;

/**
* Unit tests for Avro record reader.
*/
@RunWith(JMockit.class)
public class AvroFormatTest extends BaseTestQuery {

// XXX
Expand Down
3 changes: 0 additions & 3 deletions ...ava-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.vector.complex.writer;

import mockit.integration.junit4.JMockit;
import static org.junit.Assert.assertEquals;

import java.nio.file.Files;
Expand All @@ -30,9 +29,7 @@
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;

@RunWith(JMockit.class)
public class TestExtendedTypes extends BaseTestQuery {
@BeforeClass
public static void setupTestFiles() {
Expand Down
4 changes: 3 additions & 1 deletion exec/java-exec/src/test/resources/drill-udf/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,14 @@
<includes>
<include>${include.files}</include>
</includes>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<version>3.0.1</version>
<configuration>
<finalName>${jar.finalName}</finalName>
<includes>
Expand Down
5 changes: 4 additions & 1 deletion exec/jdbc-all/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.18.1</version>
<version>2.22.0</version>
<executions>
<execution>
<goals>
Expand All @@ -268,6 +268,7 @@
<app.class.path>${app.class.path}</app.class.path>
<project.version>${project.version}</project.version>
</systemPropertyVariables>
<useSystemClassLoader>false</useSystemClassLoader>
</configuration>
</plugin>

Expand Down Expand Up @@ -333,6 +334,7 @@
<exclude>commons-beanutils:commons-beanutils-core:jar:*</exclude>
<exclude>commons-beanutils:commons-beanutils:jar:*</exclude>
<exclude>io.netty:netty-tcnative:jar:*</exclude>
<exclude>org.honton.chas.hocon:jackson-dataformat-hocon:*</exclude>
</excludes>
</artifactSet>
<relocations>
Expand Down Expand Up @@ -648,6 +650,7 @@
<exclude>commons-io:commons-io</exclude>
<exclude>commons-beanutils:commons-beanutils-core:jar:*</exclude>
<exclude>commons-beanutils:commons-beanutils:jar:*</exclude>
<exclude>org.honton.chas.hocon:jackson-dataformat-hocon:*</exclude>
</excludes>
</artifactSet>
<relocations>
Expand Down