DRILL-6349: Drill JDBC driver fails on Java 1.9+ with NoClassDefFound…

…Error: sun/misc/VM closes apache#1446
mattpollack · Feb 25, 2019 · 272b715 · 272b715
1 parent d4c9da7
commit 272b715
Show file tree

Hide file tree

Showing 24 changed files with 123 additions and 68 deletions.
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -27,6 +27,7 @@
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
+import io.netty.util.internal.PlatformDependent;
 import org.apache.drill.common.exceptions.DrillConfigurationException;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.scanner.ClassPathScanner;
@@ -46,7 +47,7 @@ public class DrillConfig extends NestedConfig {
   private final ImmutableList<String> startupArguments;
 
   @SuppressWarnings("restriction")
-  private static final long MAX_DIRECT_MEMORY = sun.misc.VM.maxDirectMemory();
+  private static final long MAX_DIRECT_MEMORY = PlatformDependent.maxDirectMemory();
 
   @VisibleForTesting
   public DrillConfig(Config config) {

diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml
@@ -104,7 +104,7 @@
         <!-- Because the JDBC tests are somewhat heavyweight, we only run them in the 'verify' phase -->
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-failsafe-plugin</artifactId>
-        <version>2.18.1</version>
+        <version>2.22.0</version>
         <configuration>
           <forkCount combine.self="override">1</forkCount>
           <systemPropertyVariables>
@@ -115,6 +115,7 @@
           <includes>
             <include>**/*IT.java</include>
           </includes>
+          <useSystemClassLoader>false</useSystemClassLoader>
         </configuration>
         <executions>
           <execution>

diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh
@@ -302,6 +302,17 @@ export DRILLBIT_CODE_CACHE_SIZE=${DRILLBIT_CODE_CACHE_SIZE:-"1G"}
 export DRILLBIT_OPTS="-Xms$DRILL_HEAP -Xmx$DRILL_HEAP -XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"
 export DRILLBIT_OPTS="$DRILLBIT_OPTS -XX:ReservedCodeCacheSize=$DRILLBIT_CODE_CACHE_SIZE -Ddrill.exec.enable-epoll=false"
 
+# Check that java is newer than 1.8
+"$JAVA" -version 2>&1 | grep "version" | egrep -e "1\.8" > /dev/null
+if [ $? -gt 0 ]; then
+  # Allow reflective access on Java 9+
+  export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
+  export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.base/sun.nio.ch=ALL-UNNAMED"
+  export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.base/java.nio=ALL-UNNAMED"
+  export DRILLBIT_OPTS="$DRILLBIT_OPTS --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED"
+  export DRILLBIT_OPTS="$DRILLBIT_OPTS --illegal-access=permit"
+fi
+
 
 # Under YARN, the log directory is usually YARN-provided. Replace any
 # value that may have been set in drill-env.sh.

diff --git a/distribution/src/resources/sqlline.bat b/distribution/src/resources/sqlline.bat
@@ -205,6 +205,18 @@ set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% -Dlog.path="%DRILL_LOG_DIR%\sq
 SET JAVA_CMD=%JAVA_HOME%\bin\%JAVA_EXE%
 if "%JAVA_HOME%" == "" (set JAVA_CMD=%JAVA_EXE%)
 set ERROR_CODE=0
+
+rem Check that java is newer than 1.8
+"%JAVA_CMD%" -version 2>&1 | findstr "1.8" > nul  2>&1
+if errorlevel 1 (
+  rem allow reflective access on Java 9+
+  set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.base/java.lang=ALL-UNNAMED
+  set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.base/sun.nio.ch=ALL-UNNAMED
+  set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.base/java.nio=ALL-UNNAMED
+  set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED
+  set DRILL_SHELL_JAVA_OPTS=%DRILL_SHELL_JAVA_OPTS% --illegal-access=permit
+)
+
 set SQLLINE_CALL=sqlline.SqlLine -ac org.apache.drill.exec.client.DrillSqlLineApplication -d org.apache.drill.jdbc.Driver
 if NOT "test%QUERY%"=="test" (
   echo %QUERY% | "%JAVA_CMD%" %DRILL_SHELL_JAVA_OPTS% %DRILL_JAVA_OPTS% -cp "%DRILL_CP%" %SQLLINE_CALL% %DRILL_ARGS%

diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
@@ -64,28 +64,22 @@
       <version>${kerby.version}</version>
       <scope>test</scope>
     </dependency>
-    <!-- <dependency> -->
-    <!-- <groupId>org.ow2.asm</groupId> -->
-    <!-- <artifactId>asm-util</artifactId> -->
-    <!-- <version>5.0.3</version> -->
-    <!-- </dependency> -->
     <dependency>
       <groupId>org.ow2.asm</groupId>
-      <artifactId>asm-debug-all</artifactId>
-      <version>5.0.3</version>
-      <!-- <scope>test</scope> -->
+      <artifactId>asm-commons</artifactId>
+      <version>${asm.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.ow2.asm</groupId>
+      <artifactId>asm-util</artifactId>
+      <version>${asm.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.drill.contrib.data</groupId>
       <artifactId>tpch-sample-data</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
-    <!-- <dependency> -->
-    <!-- <groupId>org.ow2.asm</groupId> -->
-    <!-- <artifactId>asm-commons</artifactId> -->
-    <!-- <version>5.0.3</version> -->
-    <!-- </dependency> -->
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-pool2</artifactId>

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CompilationConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CompilationConfig.java
@@ -24,5 +24,5 @@ public class CompilationConfig {
    * Never use asm.Opcodes values directly in calls that require them. Use ASM_OPCODES
    * instead, so that we can change it here once for all references.
    */
-  public final static int ASM_API_VERSION = Opcodes.ASM5;
+  public final static int ASM_API_VERSION = Opcodes.ASM7;
 }
diff --git a/.../java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java b/.../java-exec/src/main/java/org/apache/drill/exec/compile/bytecode/ReplacingInterpreter.java
@@ -19,6 +19,7 @@
 
 import java.util.List;
 
+import org.apache.drill.exec.compile.CompilationConfig;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 import org.objectweb.asm.tree.AbstractInsnNode;
@@ -40,6 +41,7 @@ public class ReplacingInterpreter extends BasicInterpreter {
   private final List<ReplacingBasicValue> valueList;
 
   public ReplacingInterpreter(final String className, final List<ReplacingBasicValue> valueList) {
+    super(CompilationConfig.ASM_API_VERSION);
     this.className = className;
     this.valueList = valueList;
   }

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
@@ -161,7 +161,7 @@ public static MappingSet getDefaultMapping() {
       // from the JDK Modifier values to the JCodeModel JMod values: the
       // values are different.
 
-      int mods = JMod.PRIVATE + JMod.FINAL;
+      int mods = JMod.FINAL;
       if ((innerClass.getModifiers() & Modifier.STATIC) != 0) {
         mods += JMod.STATIC;
       }
@@ -370,7 +370,7 @@ private boolean createNestedClass() {
       // all new fields will be declared in the class from innerClassGenerator
       if (innerClassGenerator == null) {
         try {
-          JDefinedClass innerClazz = clazz._class(JMod.PRIVATE, clazz.name() + "0");
+          JDefinedClass innerClazz = clazz._class(JMod.NONE, clazz.name() + "0");
           innerClassGenerator = new ClassGenerator<>(codeGenerator, mappings, sig, evaluationVisitor, innerClazz, model, optionManager);
         } catch (JClassAlreadyExistsException e) {
           throw new DrillRuntimeException(e);

diff --git a/.../java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java b/.../java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
@@ -180,7 +180,7 @@ protected void createPartitionSublists() {
 
     // build a list of DFSDirPartitionLocation.
     for (final List<String> dirs : dirToFileMap.keySet()) {
-      locations.add( new DFSDirPartitionLocation((String [])dirs.toArray(), dirToFileMap.get(dirs)));
+      locations.add( new DFSDirPartitionLocation(dirs.toArray(new String[dirs.size()]), dirToFileMap.get(dirs)));
     }
 
     locationSuperList = Lists.partition(locations, PartitionDescriptor.PARTITION_BATCH_SIZE);

diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java b/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java
@@ -19,19 +19,16 @@
 
 import mockit.Mock;
 import mockit.MockUp;
-import mockit.integration.junit4.JMockit;
 import org.apache.calcite.util.Util;
 import org.apache.drill.common.exceptions.UserRemoteException;
 import org.apache.drill.test.BaseTestQuery;
 import org.junit.Test;
-import org.junit.runner.RunWith;
 
 import java.nio.charset.Charset;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.junit.Assert.assertThat;
 
-@RunWith(JMockit.class)
 public class TestUtf8SupportInQueryString extends BaseTestQuery {
 
   @Test

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
@@ -38,17 +38,13 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
 
 import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
 import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
 
-import mockit.integration.junit4.JMockit;
-
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.hasItem;
 
-@RunWith(JMockit.class)
 @Category({UnlikelyTest.class, SqlFunctionTest.class})
 public class TestCastFunctions extends BaseTestQuery {
 

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
@@ -19,7 +19,6 @@
 
 import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
 import org.apache.drill.shaded.guava.com.google.common.io.Files;
-import mockit.integration.junit4.JMockit;
 import org.apache.drill.categories.SqlFunctionTest;
 import org.apache.drill.categories.UnlikelyTest;
 import org.apache.drill.common.util.DrillFileUtils;
@@ -37,12 +36,10 @@
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
 import java.util.List;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-@RunWith(JMockit.class)
 @Category({UnlikelyTest.class, SqlFunctionTest.class})
 public class TestDateFunctions extends PopUnitTestBase {
 

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
@@ -17,15 +17,13 @@
  */
 package org.apache.drill.exec.fn.impl.testing;
 
-import mockit.integration.junit4.JMockit;
 import org.apache.drill.test.BaseTestQuery;
 import org.apache.drill.categories.SqlFunctionTest;
 import org.apache.drill.categories.UnlikelyTest;
 import org.apache.drill.common.exceptions.UserException;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
@@ -37,7 +35,6 @@
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.junit.Assert.assertThat;
 
-@RunWith(JMockit.class)
 @Category({UnlikelyTest.class, SqlFunctionTest.class})
 public class TestDateConversions extends BaseTestQuery {
 

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
@@ -19,19 +19,16 @@
 
 import org.apache.drill.shaded.guava.com.google.common.base.Joiner;
 import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import mockit.integration.junit4.JMockit;
 import org.apache.drill.PlanTestBase;
 import org.apache.drill.categories.SqlTest;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
 import java.io.File;
 import java.io.PrintWriter;
 import java.util.List;
 
-@RunWith(JMockit.class)
 @Category(SqlTest.class)
 public class TestConstantFolding extends PlanTestBase {
 

diff --git a/...ava-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java b/...ava-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
@@ -280,7 +280,7 @@ public void testCreateViewInWSWithNoPermissionsForQueryUser() throws Exception {
 
     final String query = "CREATE VIEW " + viewName + " AS SELECT " +
         "c_custkey, c_nationkey FROM cp.`tpch/customer.parquet` ORDER BY c_custkey;";
-    final String expErrorMsg = "PERMISSION ERROR: Permission denied: user=drillTestUser2, access=WRITE, inode=\"/drill_test_grp_0_755/";
+    final String expErrorMsg = "PERMISSION ERROR: Permission denied: user=drillTestUser2, access=WRITE, inode=\"/drill_test_grp_0_755";
     errorMsgTestHelper(query, expErrorMsg);
 
     // SHOW TABLES is expected to return no records as view creation fails above.
@@ -351,7 +351,7 @@ public void testCreateTableInWSWithNoPermissionsForQueryUser() throws Exception
 
     thrown.expect(UserRemoteException.class);
     thrown.expectMessage(containsString("Permission denied: user=drillTestUser2, " +
-        "access=WRITE, inode=\"/drill_test_grp_0_755/"));
+        "access=WRITE, inode=\"/drill_test_grp_0_755"));
 
     test("CREATE TABLE %s AS SELECT c_custkey, c_nationkey " +
         "FROM cp.`tpch/customer.parquet` ORDER BY c_custkey", tableName);

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
@@ -56,16 +56,12 @@
 import org.apache.drill.test.TestBuilder;
 import org.junit.Assert;
 import org.junit.Test;
-import org.junit.runner.RunWith;
 
 import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
 
-import mockit.integration.junit4.JMockit;
-
 /**
  * Unit tests for Avro record reader.
  */
-@RunWith(JMockit.class)
 public class AvroFormatTest extends BaseTestQuery {
 
   // XXX

diff --git a/...ava-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java b/...ava-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
@@ -17,7 +17,6 @@
  */
 package org.apache.drill.exec.vector.complex.writer;
 
-import mockit.integration.junit4.JMockit;
 import static org.junit.Assert.assertEquals;
 
 import java.nio.file.Files;
@@ -30,9 +29,7 @@
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.junit.runner.RunWith;
 
-@RunWith(JMockit.class)
 public class TestExtendedTypes extends BaseTestQuery {
   @BeforeClass
   public static void setupTestFiles() {

diff --git a/exec/java-exec/src/test/resources/drill-udf/pom.xml b/exec/java-exec/src/test/resources/drill-udf/pom.xml
@@ -63,12 +63,14 @@
           <includes>
             <include>${include.files}</include>
           </includes>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-source-plugin</artifactId>
-        <version>2.4</version>
+        <version>3.0.1</version>
         <configuration>
           <finalName>${jar.finalName}</finalName>
           <includes>

diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
@@ -241,7 +241,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-failsafe-plugin</artifactId>
-        <version>2.18.1</version>
+        <version>2.22.0</version>
         <executions>
           <execution>
             <goals>
@@ -268,6 +268,7 @@
             <app.class.path>${app.class.path}</app.class.path>
             <project.version>${project.version}</project.version>
           </systemPropertyVariables>
+          <useSystemClassLoader>false</useSystemClassLoader>
         </configuration>
       </plugin>
 
@@ -333,6 +334,7 @@
               <exclude>commons-beanutils:commons-beanutils-core:jar:*</exclude>
               <exclude>commons-beanutils:commons-beanutils:jar:*</exclude>
               <exclude>io.netty:netty-tcnative:jar:*</exclude>
+              <exclude>org.honton.chas.hocon:jackson-dataformat-hocon:*</exclude>
             </excludes>
           </artifactSet>
           <relocations>
@@ -648,6 +650,7 @@
                     <exclude>commons-io:commons-io</exclude>
                     <exclude>commons-beanutils:commons-beanutils-core:jar:*</exclude>
                     <exclude>commons-beanutils:commons-beanutils:jar:*</exclude>
+                    <exclude>org.honton.chas.hocon:jackson-dataformat-hocon:*</exclude>
                   </excludes>
                 </artifactSet>
                 <relocations>