DRILL-4142: Query/Node memory maximums

common/src/main/java/org/apache/drill/common/AutoCloseablePointer.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+/**
+ * A class similar to Pointer<>, but with features unique to holding
+ * AutoCloseable pointers. The AutoCloseablePointer<> must be closed
+ * when it will no longer be used.
+ *
+ * <p>If you're familiar with C++/Boost's shared_ptr<>, you might recognize
+ * some of the features here.</p>
+ *
+ * @param <T> type of the pointer
+ */
+public final class AutoCloseablePointer<T extends AutoCloseable> implements AutoCloseable {
+  private T pointer;
+
+  /**
+   * Constructor for a null-valued pointer.
+   */
+  public AutoCloseablePointer() {
+    pointer = null;
+  }
+
+  /**
+   * Constructor for a pointer value.
+   *
+   * @param pointer the initial pointer value
+   */
+  public AutoCloseablePointer(final T pointer) {
+    this.pointer = pointer;
+  }
+
+  @Override
+  public void close() throws Exception {
+    assign(null);
+  }
+
+  /**
+   * Get the raw pointer out for use.
+   *
+   * @return the raw pointer
+   */
+  public T get() {
+    return pointer;
+  }
+
+  /**
+   * The caller adopts the pointer; the holder is set to
+   * null, and will no longer be responsible for close()ing this pointer.
+   *
+   * @return the pointer being adopted; may be null
+   */
+  public T adopt() {
+    final T p = pointer;
+    pointer = null;
+    return p;
+  }
+
+  /**
+   * Assign a new pointer to this holder. Any currently held pointer
+   * will first be closed. If closing the currently held pointer throws
+   * an exception, the new pointer is still assigned, and the holder must still
+   * be closed to close that.
+   *
+   * <p>This makes it convenient to assign a new pointer without having to check
+   * for a previous value and worry about cleaning it up; this does all that.</p>
+   *
+   * @param newP the new pointer to hold
+   * @throws Exception any exception thrown by closing the currently held
+   *   pointer
+   */
+  public void assign(final T newP) throws Exception {
+    try {
+      if (pointer != null) {
+        pointer.close();
+      }
+    } finally {
+      pointer = newP;
+    }
+  }
+
+  /**
+   * Like {@link #assign(AutoCloseable)}, except that any exception thrown
+   * by closing the previously held pointer is wrapped with (an unchecked)
+   * {@link RuntimeException}.
+   *
+   * @param newP the new pointer to hold
+   */
+  public void assignNoChecked(final T newP) {
+    try {
+      assign(newP);
+    } catch(final Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

common/src/main/java/org/apache/drill/common/AutoCloseables.java

@@ -47,7 +47,7 @@ public static void close(final AutoCloseable ac, final Logger logger) {
     }
   }
 
-  public static void close(AutoCloseable[] ac) throws Exception {
+  public static void close(AutoCloseable... ac) throws Exception {
     Exception topLevelException = null;
     for (AutoCloseable closeable : ac) {
       try {

common/src/main/java/org/apache/drill/common/DrillCloseables.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Provides additional functionality to Guava's Closeables.
+ */
+public class DrillCloseables {
+  /**
+   * Constructor. Prevents construction for class of static utilities.
+   */
+  private DrillCloseables() {
+  }
+
+  /**
+   * Close() a {@see java.io.Closeable} without throwing a (checked)
+   * {@see java.io.IOException}. This wraps the close() call with a
+   * try-catch that will rethrow an IOException wrapped with a
+   * {@see java.lang.RuntimeException}, providing a way to call close()
+   * without having to do the try-catch everywhere or propagate the IOException.
+   *
+   * <p>Guava has deprecated {@see com.google.common.io.Closeables.closeQuietly()}
+   * as described in
+   * {@link https://code.google.com/p/guava-libraries/issues/detail?id=1118}.
+   *
+   * @param closeable the Closeable to close
+   * @throws RuntimeException if an IOException occurs; the IOException is
+   *   wrapped by the RuntimeException
+   */
+  public static void closeNoChecked(final Closeable closeable) {
+    try {
+      closeable.close();
+    } catch(final IOException e) {
+      throw new RuntimeException("IOException while closing", e);
+    }
+  }
+}

common/src/main/java/org/apache/drill/common/HistoricalLog.java

@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+
+import org.slf4j.Logger;
+
+/**
+ * Utility class that can be used to log activity within a class
+ * for later logging and debugging. Supports recording events and
+ * recording the stack at the time they occur.
+ */
+public class HistoricalLog {
+  private static class Event {
+    private final String note; // the event text
+    private final StackTrace stackTrace; // where the event occurred
+
+    public Event(final String note) {
+      this.note = note;
+      stackTrace = new StackTrace();
+    }
+  }
+
+  private final LinkedList<Event> history = new LinkedList<>();
+  private final String idString; // the formatted id string
+  private Event firstEvent; // the first stack trace recorded
+  private final int limit; // the limit on the number of events kept
+
+  /**
+   * Constructor. The format string will be formatted and have its arguments
+   * substituted at the time this is called.
+   *
+   * @param idStringFormat {@link String#format} format string that can be used
+   *     to identify this object in a log. Including some kind of unique identifier
+   *     that can be associated with the object instance is best.
+   * @param args for the format string, or nothing if none are required
+   */
+  public HistoricalLog(final String idStringFormat, Object... args) {
+    this(Integer.MAX_VALUE, idStringFormat, args);
+  }
+
+  /**
+   * Constructor. The format string will be formatted and have its arguments
+   * substituted at the time this is called.
+   *
+   * <p>This form supports the specification of a limit that will limit the
+   * number of historical entries kept (which keeps down the amount of memory
+   * used). With the limit, the first entry made is always kept (under the
+   * assumption that this is the creation site of the object, which is usually
+   * interesting), and then up to the limit number of entries are kept after that.
+   * Each time a new entry is made, the oldest that is not the first is dropped.
+   * </p>
+   *
+   * @param limit the maximum number of historical entries that will be kept,
+   *   not including the first entry made
+   * @param idStringFormat {@link String#format} format string that can be used
+   *     to identify this object in a log. Including some kind of unique identifier
+   *     that can be associated with the object instance is best.
+   * @param args for the format string, or nothing if none are required
+   */
+  public HistoricalLog(final int limit, final String idStringFormat, Object... args) {
+    this.limit = limit;
+    this.idString = String.format(idStringFormat, args);
+  }
+
+  /**
+   * Record an event. Automatically captures the stack trace at the time this is
+   * called. The format string will be formatted and have its arguments substituted
+   * at the time this is called.
+   *
+   * @param noteFormat {@link String#format} format string that describes the event
+   * @param args for the format string, or nothing if none are required
+   */
+  public synchronized void recordEvent(final String noteFormat, Object... args) {
+    final String note = String.format(noteFormat, args);
+    final Event event = new Event(note);
+    if (firstEvent == null) {
+      firstEvent = event;
+    }
+    if (history.size() == limit) {
+      history.removeFirst();
+    }
+    history.add(event);
+  }
+
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   */
+  public void buildHistory(final StringBuilder sb, boolean includeStackTrace) {
+    buildHistory(sb, 0, includeStackTrace);
+  }
+
+  /**
+   * Write the history of this object to the given {@link StringBuilder}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param sb {@link StringBuilder} to write to
+   * @param additional an extra string that will be written between the identifying
+   *     information and the history; often used for a current piece of state
+   */
+
+  /**
+   *
+   * @param sb
+   * @param indexLevel
+   * @param includeStackTrace
+   */
+  public synchronized void buildHistory(final StringBuilder sb, int indent, boolean includeStackTrace) {
+    final char[] indentation = new char[indent];
+    final char[] innerIndentation = new char[indent + 2];
+    Arrays.fill(indentation, ' ');
+    Arrays.fill(innerIndentation, ' ');
+
+    sb.append(indentation)
+        .append("event log for: ")
+        .append(idString)
+        .append('\n');
+
+
+    if (firstEvent != null) {
+      sb.append(innerIndentation)
+          .append(firstEvent.note)
+          .append('\n');
+      if (includeStackTrace) {
+        firstEvent.stackTrace.writeToBuilder(sb, indent + 2);
+      }
+
+      for(final Event event : history) {
+        if (event == firstEvent) {
+          continue;
+        }
+        sb.append(innerIndentation)
+            .append("  ")
+            .append(event.note)
+            .append('\n');
+
+        if (includeStackTrace) {
+          event.stackTrace.writeToBuilder(sb, indent + 2);
+          sb.append('\n');
+        }
+      }
+    }
+  }
+
+  /**
+   * Write the history of this object to the given {@link Logger}. The history
+   * includes the identifying string provided at construction time, and all the recorded
+   * events with their stack traces.
+   *
+   * @param logger {@link Logger} to write to
+   */
+  public void logHistory(final Logger logger) {
+    final StringBuilder sb = new StringBuilder();
+    buildHistory(sb, 0, true);
+    logger.debug(sb.toString());
+  }
+}

common/src/main/java/org/apache/drill/common/StackTrace.java

@@ -36,13 +36,14 @@ public StackTrace() {
 
   /**
    * Write the stack trace to a StringBuilder.
-   *
-   * @param sb where to write it
-   * @param indent how many spaces to indent each line
+   * @param sb
+   *          where to write it
+   * @param indent
+   *          how many double spaces to indent each line
    */
   public void writeToBuilder(final StringBuilder sb, final int indent) {
     // create the indentation string
-    final char[] indentation = new char[indent];
+    final char[] indentation = new char[indent * 2];
     Arrays.fill(indentation, ' ');
 
     // write the stack trace in standard Java format

distribution/pom.xml

@@ -44,11 +44,6 @@
       <artifactId>drill-memory-base</artifactId>
       <version>${project.version}</version>
     </dependency>    
-    <dependency>
-      <groupId>org.apache.drill.memory</groupId>
-      <artifactId>drill-memory-impl</artifactId>
-      <version>${project.version}</version>
-    </dependency> 
     <dependency>
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>drill-rpc</artifactId>

distribution/src/assemble/bin.xml

@@ -89,7 +89,6 @@
         <include>org.apache.drill:drill-logical:jar</include>
         <include>org.apache.drill.exec:vector:jar</include>
         <include>org.apache.drill.memory:drill-memory-base:jar</include>
-        <include>org.apache.drill.memory:drill-memory-impl:jar</include>
         <include>org.apache.drill.exec:drill-rpc:jar</include>
         <include>org.apache.drill.exec:drill-java-exec:jar</include>
         <include>org.apache.drill.contrib.storage-hive:drill-storage-hive-core</include>