DRILL-3987: (CLEANUP) Final cleanups to get complete working build/di…

…stribution - small cleanups - move Hook to drill-adbc - update distribution assembly to include new modules This closes #250
apache · Nov 13, 2015 · 494492f · 494492f
1 parent 44dea43
commit 494492f
Show file tree

Hide file tree

Showing 32 changed files with 305 additions and 86 deletions.
diff --git a/common/pom.xml b/common/pom.xml
@@ -129,28 +129,6 @@
           </arguments>
         </configuration>
       </plugin>
-      <plugin>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.15</version>
-        <configuration>
-          <useSystemClassLoader>false</useSystemClassLoader>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.antlr</groupId>
-        <artifactId>antlr3-maven-plugin</artifactId>
-        <version>3.4</version>
-        <configuration>
-
-        </configuration>
-        <executions>
-          <execution>
-            <goals>
-              <goal>antlr</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
 
   </build>

diff --git a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
@@ -87,6 +87,15 @@ private static ScanResult loadExcept(URL ignored) {
       }
     }
     if (result != null) {
+      if (logger.isInfoEnabled()) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(format("Loaded prescanned packages %s from locations:\n", result.getScannedPackages()));
+        for (URL u : preScanned) {
+          sb.append('\t');
+          sb.append(u.toExternalForm());
+          sb.append('\n');
+        }
+      }
       logger.info(format("Loaded prescanned packages %s from locations %s", result.getScannedPackages(), preScanned));
       return result;
     } else {

diff --git a/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java b/common/src/main/java/org/apache/drill/exec/metrics/DrillMetrics.java
@@ -37,9 +37,9 @@
 public class DrillMetrics {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetrics.class);
 
-  public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.exec.metrics.jmx.enabled";
-  public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.exec.metrics.log.enabled";
-  public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.exec.metrics.log.interval";
+  public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.metrics.jmx.enabled";
+  public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.metrics.log.enabled";
+  public static final String METRICS_LOG_OUTPUT_INTERVAL = "drill.metrics.log.interval";
 
   static final DrillConfig config = DrillConfig.create();
 

diff --git a/common/src/main/resources/drill-module.conf b/common/src/main/resources/drill-module.conf
@@ -29,5 +29,17 @@ drill {
       org.apache.drill.exec.store.mock,
       org.apache.drill.common.logical
     ]
-  }
+  },
+
+  metrics : {
+    context: "drillbit",
+    jmx: {
+      enabled : true
+    },
+    log: {
+      enabled : false,
+      interval : 60
+    }
+  },
+
 }
diff --git a/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java b/common/src/test/java/org/apache/drill/common/exceptions/TestUserException.java
@@ -26,7 +26,8 @@
  * Test various use cases when creating user exceptions
  */
 public class TestUserException {
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUserException.class);
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
+      .getLogger("--ignore.as.this.is.for.testing.exceptions--");
 
   private Exception wrap(UserException uex, int numWraps) {
     Exception ex = uex;

diff --git a/.../storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java b/.../storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
@@ -191,7 +191,7 @@ public static class Decimal28 extends HiveFieldConverter {
     public Decimal28(int precision, int scale, FragmentContext context) {
       holder.scale = scale;
       holder.precision = precision;
-      holder.buffer = context.getManagedBuffer(Decimal28SparseHolder.nDecimalDigits * DecimalUtility.integerSize);
+      holder.buffer = context.getManagedBuffer(Decimal28SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE);
       holder.start = 0;
     }
 
@@ -210,7 +210,7 @@ public static class Decimal38 extends HiveFieldConverter {
     public Decimal38(int precision, int scale, FragmentContext context) {
       holder.scale = scale;
       holder.precision = precision;
-      holder.buffer = context.getManagedBuffer(Decimal38SparseHolder.nDecimalDigits * DecimalUtility.integerSize);
+      holder.buffer = context.getManagedBuffer(Decimal38SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE);
       holder.start = 0;
     }
 

diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
@@ -231,7 +231,7 @@ public static void populateVector(final ValueVector vector, final DrillBuf manag
       }
 
       case DECIMAL28SPARSE: {
-        final int needSpace = Decimal28SparseHolder.nDecimalDigits * DecimalUtility.integerSize;
+      final int needSpace = Decimal28SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE;
         Preconditions.checkArgument(managedBuffer.capacity() > needSpace,
             String.format("Not sufficient space in given managed buffer. Need %d bytes, buffer has %d bytes",
                 needSpace, managedBuffer.capacity()));
@@ -252,7 +252,7 @@ public static void populateVector(final ValueVector vector, final DrillBuf manag
       }
 
       case DECIMAL38SPARSE: {
-        final int needSpace = Decimal38SparseHolder.nDecimalDigits * DecimalUtility.integerSize;
+      final int needSpace = Decimal38SparseHolder.nDecimalDigits * DecimalUtility.INTEGER_SIZE;
         Preconditions.checkArgument(managedBuffer.capacity() > needSpace,
             String.format("Not sufficient space in given managed buffer. Need %d bytes, buffer has %d bytes",
                 needSpace, managedBuffer.capacity()));

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -29,6 +29,31 @@
       <artifactId>drill-protocol</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.drill</groupId>
+      <artifactId>drill-logical</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.drill.exec</groupId>
+      <artifactId>vector</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.drill.memory</groupId>
+      <artifactId>drill-memory-base</artifactId>
+      <version>${project.version}</version>
+    </dependency>    
+    <dependency>
+      <groupId>org.apache.drill.memory</groupId>
+      <artifactId>drill-memory-impl</artifactId>
+      <version>${project.version}</version>
+    </dependency> 
+    <dependency>
+      <groupId>org.apache.drill.exec</groupId>
+      <artifactId>drill-rpc</artifactId>
+      <version>${project.version}</version>
+    </dependency>    
     <dependency>
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>drill-java-exec</artifactId>

diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
@@ -86,6 +86,11 @@
         <include>org.apache.drill.exec:drill-jdbc:jar</include>
         <include>org.apache.drill:drill-protocol:jar</include>
         <include>org.apache.drill:drill-common:jar</include>
+        <include>org.apache.drill:drill-logical:jar</include>
+        <include>org.apache.drill.exec:vector:jar</include>
+        <include>org.apache.drill.memory:drill-memory-base:jar</include>
+        <include>org.apache.drill.memory:drill-memory-impl:jar</include>
+        <include>org.apache.drill.exec:drill-rpc:jar</include>
         <include>org.apache.drill.exec:drill-java-exec:jar</include>
         <include>org.apache.drill.contrib.storage-hive:drill-storage-hive-core</include>
         <include>org.apache.drill.contrib.storage-hive:drill-hive-exec-shaded</include>

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java
@@ -22,7 +22,7 @@
 import org.apache.drill.exec.expr.ClassGenerator;
 import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
 import org.apache.drill.exec.physical.impl.project.ProjectRecordBatch;
-import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
+import org.apache.drill.exec.record.VectorAccessibleComplexWriter;
 import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
 
 import com.sun.codemodel.JBlock;
@@ -54,13 +54,13 @@ protected HoldingContainer generateEvalBody(ClassGenerator<?> g, HoldingContaine
 
     JVar complexWriter = g.declareClassField("complexWriter", g.getModel()._ref(ComplexWriter.class));
 
-    JClass cwClass = g.getModel().ref(ComplexWriterImpl.class);
 
     JInvocation container = g.getMappingSet().getOutgoing().invoke("getOutgoingContainer");
 
     //Default name is "col", if not passed in a reference name for the output vector.
     String refName = ref == null? "col" : ref.getRootSegment().getPath();
 
+    JClass cwClass = g.getModel().ref(VectorAccessibleComplexWriter.class);
     g.getSetupBlock().assign(complexWriter, cwClass.staticInvoke("getWriter").arg(refName).arg(container));
 
     JClass projBatchClass = g.getModel().ref(ProjectRecordBatch.class);

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorAccessibleComplexWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorAccessibleComplexWriter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.MapVector;
+import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+
+public class VectorAccessibleComplexWriter extends MapVector {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorAccessibleComplexWriter.class);
+
+  private final VectorContainer vc;
+
+  public VectorAccessibleComplexWriter(VectorContainer vc) {
+    super("", null, null);
+    this.vc = vc;
+  }
+
+  @Override
+  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+    final ValueVector v = vc.addOrGet(name, type, clazz);
+    putChild(name, v);
+    return this.typeify(v, clazz);
+
+  }
+
+  public static ComplexWriter getWriter(String name, VectorContainer container) {
+    VectorAccessibleComplexWriter vc = new VectorAccessibleComplexWriter(container);
+    return new ComplexWriterImpl(name, vc);
+  }
+}
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
@@ -82,16 +82,6 @@ drill.exec: {
       }
     }
   },
-  metrics : {
-    context: "drillbit",
-    jmx: {
-      enabled : true
-    },
-    log: {
-      enabled : false,
-      interval : 60
-    }
-  },
   zk: {
   connect: "localhost:2181",
   root: "drill",
@@ -164,7 +154,6 @@ drill.exec: {
     }
   },
   memory: {
-    top.max: 1000000000000,
     operator: {
       max: 20000000000,
       initial: 10000000
@@ -175,7 +164,6 @@ drill.exec: {
     }
   },
   debug: {
-    error_on_leak: true,
     return_error_for_failure_in_cancelled_fragments: false
   }
 }

diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
@@ -17,21 +17,24 @@
  */
 package org.apache.drill;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.exception.OutOfMemoryException;
 import org.apache.drill.exec.memory.TopLevelAllocator;
 import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
 import org.apache.drill.exec.testing.Controls;
 import org.apache.drill.exec.testing.ControlsInjectionUtil;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /**
  * Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause an OUT_OF_MEMORY outcome to
  * be propagated downstream. Make sure the proper "memory error" message is sent to the client.
  */
+@Ignore("Need to add exception site in memory layer that doesn't depend on other Drill code.")
 public class TestAllocationException extends BaseTestQuery {
 
   private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true";

diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
@@ -18,6 +18,7 @@
 package org.apache.drill;
 
 import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
 import org.apache.drill.exec.proto.CoordinationProtos;
 import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
 import org.apache.drill.exec.testing.ControlsInjectionUtil;
@@ -44,7 +45,7 @@ private void testSingleMode(String fileName) throws Exception{
       + "\"desc\":\"" + "next-allocate" + "\","
       + "\"nSkip\":0,"
       + "\"nFire\":1,"
-      + "\"exceptionClass\":\"" + "org.apache.drill.exec.memory.OutOfMemoryException" + "\""
+        + "\"exceptionClass\":\"" + OutOfMemoryException.class.getName() + "\""
       + "}]}";
     ControlsInjectionUtil.setControls(client, controlsString);
 

diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
@@ -69,7 +69,7 @@ public void x() throws Exception{
     ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
 
     v.addExpr(e2,  g.getRoot());
-    System.out.println(g.generateAndGet());
+    logger.debug(g.generateAndGet());
   }
 
   private LogicalExpression getExpr(String expr) throws Exception{

diff --git a/.../src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java b/.../src/test/java/org/apache/drill/exec/test/Drill2130JavaExecHamcrestConfigurationTest.java
@@ -17,14 +17,17 @@
  */
 package org.apache.drill.exec.test;
 
-import org.junit.Test;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
-import static org.hamcrest.CoreMatchers.equalTo;
+
+import org.junit.Test;
 
 
 public class Drill2130JavaExecHamcrestConfigurationTest {
 
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
+      .getLogger(Drill2130JavaExecHamcrestConfigurationTest.class);
   @SuppressWarnings("unused")
   private org.hamcrest.MatcherAssert forCompileTimeCheckForNewEnoughHamcrest;
 
@@ -36,9 +39,8 @@ public void testJUnitHamcrestMatcherFailureWorks() {
     catch ( NoSuchMethodError e ) {
       fail( "Class search path seems broken re new JUnit and old Hamcrest."
              + "  Got NoSuchMethodError;  e: " + e );
-    }
-    catch ( AssertionError e ) {
-      System.out.println( "Class path seems fine re new JUnit vs. old Hamcrest."
+    } catch (AssertionError e) {
+      logger.debug("Class path seems fine re new JUnit vs. old Hamcrest."
                           + " (Got AssertionError, not NoSuchMethodError.)" );
     }
   }

diff --git a/...va-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java b/...va-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
@@ -17,11 +17,13 @@
  */
 package org.apache.drill.exec.work.fragment;
 
+import static org.junit.Assert.fail;
+
 import org.apache.drill.BaseTestQuery;
 import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.exception.OutOfMemoryException;
 import org.apache.drill.exec.proto.CoordinationProtos;
 import org.apache.drill.exec.testing.ControlsInjectionUtil;
-import static org.junit.Assert.fail;
 import org.junit.Test;
 
 /**
@@ -44,7 +46,7 @@ public void testCancelNonRunningFragments() throws Exception{
       + "\"desc\":\"" + "next-allocate" + "\","
       + "\"nSkip\":0,"
       + "\"nFire\":1,"
-      + "\"exceptionClass\":\"" + "org.apache.drill.exec.memory.OutOfMemoryException" + "\""
+        + "\"exceptionClass\":\"" + OutOfMemoryException.class.getName() + "\""
       + "}]}";
     ControlsInjectionUtil.setControls(client, controlsString);
 

diff --git a/exec/java-exec/src/test/resources/drill-oom-xsort.conf b/exec/java-exec/src/test/resources/drill-oom-xsort.conf
@@ -6,7 +6,6 @@ drill: {
   classpath.scanning: {
     packages += "org.apache.drill.exec.expr.fn.impl"
   },
-  exec: {
   memory: {
     fragment: {
       max: 50000000,
@@ -17,5 +16,4 @@ drill: {
       initial: 2000000
     }
   }
-  }
 }