DRILL-4134: Add new allocator

common/src/main/java/org/apache/drill/common/AutoCloseablePointer.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+/**
+ * A class similar to Pointer<>, but with features unique to holding
+ * AutoCloseable pointers. The AutoCloseablePointer<> must be closed
+ * when it will no longer be used.
+ *
+ * <p>If you're familiar with C++/Boost's shared_ptr<>, you might recognize
+ * some of the features here.</p>
+ *
+ * @param <T> type of the pointer
+ */
+public final class AutoCloseablePointer<T extends AutoCloseable> implements AutoCloseable {
+  private T pointer;
+
+  /**
+   * Constructor for a null-valued pointer.
+   */
+  public AutoCloseablePointer() {
+    pointer = null;
+  }
+
+  /**
+   * Constructor for a pointer value.
+   *
+   * @param pointer the initial pointer value
+   */
+  public AutoCloseablePointer(final T pointer) {
+    this.pointer = pointer;
+  }
+
+  @Override
+  public void close() throws Exception {
+    assign(null);
+  }
+
+  /**
+   * Get the raw pointer out for use.
+   *
+   * @return the raw pointer
+   */
+  public T get() {
+    return pointer;
+  }
+
+  /**
+   * The caller adopts the pointer; the holder is set to
+   * null, and will no longer be responsible for close()ing this pointer.
+   *
+   * @return the pointer being adopted; may be null
+   */
+  public T adopt() {
+    final T p = pointer;
+    pointer = null;
+    return p;
+  }
+
+  /**
+   * Assign a new pointer to this holder. Any currently held pointer
+   * will first be closed. If closing the currently held pointer throws
+   * an exception, the new pointer is still assigned, and the holder must still
+   * be closed to close that.
+   *
+   * <p>This makes it convenient to assign a new pointer without having to check
+   * for a previous value and worry about cleaning it up; this does all that.</p>
+   *
+   * @param newP the new pointer to hold
+   * @throws Exception any exception thrown by closing the currently held
+   *   pointer
+   */
+  public void assign(final T newP) throws Exception {
+    try {
+      if (pointer != null) {
+        pointer.close();
+      }
+    } finally {
+      pointer = newP;
+    }
+  }
+
+  /**
+   * Like {@link #assign(AutoCloseable)}, except that any exception thrown
+   * by closing the previously held pointer is wrapped with (an unchecked)
+   * {@link RuntimeException}.
+   *
+   * @param newP the new pointer to hold
+   */
+  public void assignNoChecked(final T newP) {
+    try {
+      assign(newP);
+    } catch(final Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

common/src/main/java/org/apache/drill/common/DrillCloseables.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Provides additional functionality to Guava's Closeables.
+ */
+public class DrillCloseables {
+  /**
+   * Constructor. Prevents construction for class of static utilities.
+   */
+  private DrillCloseables() {
+  }
+
+  /**
+   * Close() a {@see java.io.Closeable} without throwing a (checked)
+   * {@see java.io.IOException}. This wraps the close() call with a
+   * try-catch that will rethrow an IOException wrapped with a
+   * {@see java.lang.RuntimeException}, providing a way to call close()
+   * without having to do the try-catch everywhere or propagate the IOException.
+   *
+   * <p>Guava has deprecated {@see com.google.common.io.Closeables.closeQuietly()}
+   * as described in
+   * {@link https://code.google.com/p/guava-libraries/issues/detail?id=1118}.
+   *
+   * @param closeable the Closeable to close
+   * @throws RuntimeException if an IOException occurs; the IOException is
+   *   wrapped by the RuntimeException
+   */
+  public static void closeNoChecked(final Closeable closeable) {
+    try {
+      closeable.close();
+    } catch(final IOException e) {
+      throw new RuntimeException("IOException while closing", e);
+    }
+  }
+}

common/src/main/java/org/apache/drill/common/HistoricalLog.java

@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+import java.util.LinkedList;
+
+import org.slf4j.Logger;
+
+/**
+ * Utility class that can be used to log activity within a class
+ * for later logging and debugging. Supports recording events and
+ * recording the stack at the time they occur.
+ */
+public class HistoricalLog {
+  private static class Event {
+    private final String note; // the event text
+    private final StackTrace stackTrace; // where the event occurred
+
+    public Event(final String note) {
+      this.note = note;
+      stackTrace = new StackTrace();
+    }
+  }
+
+  private final LinkedList<Event> history = new LinkedList<>();
+  private final String idString; // the formatted id string
+  private Event firstEvent; // the first stack trace recorded
+  private final int limit; // the limit on the number of events kept
+
+  /**
+   * Constructor. The format string will be formatted and have its arguments
+   * substituted at the time this is called.
+   *
+   * @param idStringFormat {@link String#format} format string that can be used
+   *     to identify this object in a log. Including some kind of unique identifier
+   *     that can be associated with the object instance is best.
+   * @param args for the format string, or nothing if none are required
+   */
+  public HistoricalLog(final String idStringFormat, Object... args) {
+    this(Integer.MAX_VALUE, idStringFormat, args);
+  }
+
+  /**
+   * Constructor. The format string will be formatted and have its arguments
+   * substituted at the time this is called.
+   *
+   * <p>This form supports the specification of a limit that will limit the
+   * number of historical entries kept (which keeps down the amount of memory
+   * used). With the limit, the first entry made is always kept (under the
+   * assumption that this is the creation site of the object, which is usually
+   * interesting), and then up to the limit number of entries are kept after that.
+   * Each time a new entry is made, the oldest that is not the first is dropped.
+   * </p>
+   *
+   * @param limit the maximum number of historical entries that will be kept,
+   *   not including the first entry made
+   * @param idStringFormat {@link String#format} format string that can be used
+   *     to identify this object in a log. Including some kind of unique identifier
+   *     that can be associated with the object instance is best.
+   * @param args for the format string, or nothing if none are required
+   */
+  public HistoricalLog(final int limit, final String idStringFormat, Object... args) {
+    this.limit = limit;
+    this.idString = String.format(idStringFormat, args);
+  }
+
+  /**
+   * Record an event. Automatically captures the stack trace at the time this is
+   * called. The format string will be formatted and have its arguments substituted
+   * at the time this is called.
+   *
+   * @param noteFormat {@link String#format} format string that describes the event
+   * @param args for the format string, or nothing if none are required
+   */
+  public synchronized void recordEvent(final String noteFormat, Object... args) {
+    final String note = String.format(noteFormat, args);
+    final Event event = new Event(note);
+    if (firstEvent == null) {
+      firstEvent = event;
+    }
+    if (history.size() == limit) {
+      history.removeFirst();
+    }
+    history.add(event);
+  }
+
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   */
+  public void buildHistory(final StringBuilder sb) {
+    buildHistory(sb, null);
+  }
+
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   * @param additional an extra string that will be written between the identifying
+   *     information and the history; often used for a current piece of state
+   */
+  public synchronized void buildHistory(final StringBuilder sb, final CharSequence additional) {
+    sb.append('\n')
+        .append(idString);
+
+    if (additional != null) {
+      sb.append('\n')
+          .append(additional)
+          .append('\n');
+    }
+
+    sb.append(" event log\n");
+
+    if (firstEvent != null) {
+      sb.append("  ")
+          .append(firstEvent.note)
+          .append('\n');
+      firstEvent.stackTrace.writeToBuilder(sb, 4);
+
+      for(final Event event : history) {
+        if (event == firstEvent) {
+          continue;
+        }
+
+        sb.append("  ")
+            .append(event.note)
+            .append('\n');
+
+        event.stackTrace.writeToBuilder(sb, 4);
+      }
+    }
+  }
+
+  /**
+   * Write the history of this object to the given {@link Logger}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param logger {@link Logger} to write to
+   * @param additional an extra string that will be written between the identifying
+   *     information and the history; often used for a current piece of state
+   */
+  public void logHistory(final Logger logger, final CharSequence additional) {
+    final StringBuilder sb = new StringBuilder();
+    buildHistory(sb, additional);
+    logger.debug(sb.toString());
+  }
+
+  /**
+   * Write the history of this object to the given {@link Logger}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param logger {@link Logger} to write to
+   */
+  public void logHistory(final Logger logger) {
+    logHistory(logger, null);
+  }
+}

exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java

@@ -92,12 +92,8 @@ public QueryContext(final UserSession session, final DrillbitContext drillbitCon
     queryContextInfo = Utilities.createQueryContextInfo(session.getDefaultSchemaName());
     contextInformation = new ContextInformation(session.getCredentials(), queryContextInfo);
 
-    try {
-      allocator = drillbitContext.getAllocator().getChildAllocator(null, plannerSettings.getInitialPlanningMemorySize(),
-          plannerSettings.getPlanningMemoryLimit(), false);
-    } catch (OutOfMemoryException e) {
-      throw new DrillRuntimeException("Error creating off-heap allocator for planning context.",e);
-    }
+    allocator = drillbitContext.getAllocator().getChildAllocator(null, plannerSettings.getInitialPlanningMemorySize(),
+        plannerSettings.getPlanningMemoryLimit(), false);
     // TODO(DRILL-1942) the new allocator has this capability built-in, so this can be removed once that is available
     bufferManager = new BufferManagerImpl(this.allocator);
     viewExpansionContext = new ViewExpansionContext(this);

exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java

@@ -22,6 +22,8 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.calcite.rel.RelFieldCollation.Direction;
+import org.apache.drill.common.DrillAutoCloseables;
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
 import org.apache.drill.common.expression.LogicalExpression;
@@ -59,7 +61,6 @@
 import org.apache.drill.exec.record.selection.SelectionVector4;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.complex.AbstractContainerVector;
-import org.apache.calcite.rel.RelFieldCollation.Direction;
 
 import com.google.common.base.Stopwatch;
 import com.sun.codemodel.JConditional;
@@ -323,7 +324,7 @@ private void purge() throws SchemaChangeException {
       builder.getSv4().clear();
       selectionVector4.clear();
     } finally {
-      builder.close();
+      DrillAutoCloseables.closeNoChecked(builder);
     }
     logger.debug("Took {} us to purge", watch.elapsed(TimeUnit.MICROSECONDS));
   }