Permalink
Browse files

DRILL-334: Subdivide Drillbit control and data messages. Add support …

…for socket backpressure. Add TopLevel and Child memory allocator with debug mode to capture memory leaks. Various memory leak fixes to get build to complete.
  • Loading branch information...
jacques-n committed Jan 3, 2014
1 parent db02036 commit fb3543367b25b59359b233bd483f21bdfe37a1af
Showing with 13,559 additions and 9,124 deletions.
  1. +26 −0 common/src/main/java/org/apache/drill/common/config/DrillConfig.java
  2. +1 −1 exec/bufferl/src/main/java/io/netty/buffer/PooledUnsafeDirectByteBufL.java
  3. +18 −18 exec/java-exec/pom.xml
  4. +4 −3 exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
  5. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedCache.java
  6. +5 −5 exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMap.java
  7. +3 −3 exec/java-exec/src/main/java/org/apache/drill/exec/cache/DistributedMultiMap.java
  8. +33 −29 exec/java-exec/src/main/java/org/apache/drill/exec/cache/HazelCache.java
  9. +31 −33 exec/java-exec/src/main/java/org/apache/drill/exec/cache/LocalCache.java
  10. +2 −3 exec/java-exec/src/main/java/org/apache/drill/exec/cache/ProtoBufImpl.java
  11. +2 −6 exec/java-exec/src/main/java/org/apache/drill/exec/cache/ProtoBufWrap.java
  12. +3 −1 exec/java-exec/src/main/java/org/apache/drill/exec/cache/VectorAccessibleSerializable.java
  13. +5 −5 exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
  14. +3 −5 exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
  15. +858 −0 exec/java-exec/src/main/java/org/apache/drill/exec/memory/AccountingByteBuf.java
  16. +192 −0 exec/java-exec/src/main/java/org/apache/drill/exec/memory/Accountor.java
  17. +112 −0 exec/java-exec/src/main/java/org/apache/drill/exec/memory/AtomicRemainder.java
  18. +14 −28 exec/java-exec/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
  19. +0 −65 exec/java-exec/src/main/java/org/apache/drill/exec/memory/DirectBufferAllocator.java
  20. +51 −0 exec/java-exec/src/main/java/org/apache/drill/exec/memory/OutOfMemoryException.java
  21. +161 −0 exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
  22. +4 −5 exec/java-exec/src/main/java/org/apache/drill/exec/metrics/SingleThreadNestedCounter.java
  23. +101 −54 exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
  24. +62 −0 exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentStats.java
  25. +26 −0 exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java
  26. +10 −3 exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
  27. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedPartitionSender.java
  28. +4 −4 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
  29. +4 −0 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
  30. +27 −9 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
  31. +25 −28 ...ava/org/apache/drill/exec/{work/foreman/TunnelManager.java → physical/impl/SendingAccountor.java}
  32. +15 −6 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
  33. +9 −2 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
  34. +10 −8 ...ec/src/main/java/org/apache/drill/exec/physical/impl/broadcastsender/BroadcastSenderRootExec.java
  35. +8 −1 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
  36. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatchBuilder.java
  37. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/limit/LimitRecordBatch.java
  38. +9 −0 ...java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
  39. +250 −180 ...main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
  40. +35 −16 ...a-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
  41. +23 −8 ...ec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
  42. +22 −21 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
  43. +22 −14 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
  44. +7 −0 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
  45. +9 −1 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate4.java
  46. +5 −1 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
  47. +11 −23 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java
  48. +5 −0 ...ec/src/main/java/org/apache/drill/exec/physical/impl/validate/IteratorValidatorBatchIterator.java
  49. +3 −3 exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
  50. 0 {protocol → exec/java-exec}/src/main/java/org/apache/drill/exec/proto/helper/QueryIdHelper.java
  51. +14 −4 exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
  52. +8 −2 exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
  53. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/record/FragmentWritableBatch.java
  54. +2 −2 exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
  55. +10 −2 exec/java-exec/src/main/java/org/apache/drill/exec/record/RawFragmentBatch.java
  56. +2 −0 exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
  57. +3 −1 exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
  58. +10 −7 exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
  59. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
  60. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
  61. +12 −8 exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
  62. +11 −12 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClient.java
  63. +19 −6 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicClientWithConnection.java
  64. +51 −22 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java
  65. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/CoordinationQueue.java
  66. +5 −10 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/{bit → }/FutureBitCommand.java
  67. +7 −11 ...ec/src/main/java/org/apache/drill/exec/rpc/{bit/ListeningBitCommand.java → ListeningCommand.java}
  68. +22 −4 .../java/org/apache/drill/exec/rpc/{ZeroCopyProtobufLengthDecoder.java → ProtobufLengthDecoder.java}
  69. +242 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ReconnectingConnection.java
  70. +64 −24 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
  71. +85 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/ResettableBarrier.java
  72. +0 −2 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/Response.java
  73. +20 −10 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
  74. +3 −3 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/{bit/BitCommand.java → RpcCommand.java}
  75. +8 −3 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
  76. +0 −99 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/bit/BitClient.java
  77. +0 −189 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/bit/BitConnectionManager.java
  78. +0 −59 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/bit/ListenerPool.java
  79. +2 −2 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/{bit → control}/AvailabilityListener.java
  80. +11 −19 .../java-exec/src/main/java/org/apache/drill/exec/rpc/{bit → control}/ConnectionManagerRegistry.java
  81. +105 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlClient.java
  82. +28 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlCommand.java
  83. +37 −46 ...src/main/java/org/apache/drill/exec/rpc/{bit/BitConnection.java → control/ControlConnection.java}
  84. +60 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlConnectionManager.java
  85. +40 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/ControlProtobufLengthDecoder.java
  86. +9 −11 ...c/src/main/java/org/apache/drill/exec/rpc/{bit/BitRpcConfig.java → control/ControlRpcConfig.java}
  87. +31 −26 ...va-exec/src/main/java/org/apache/drill/exec/rpc/{bit/BitServer.java → control/ControlServer.java}
  88. +16 −71 ...va-exec/src/main/java/org/apache/drill/exec/rpc/{bit/BitTunnel.java → control/ControlTunnel.java}
  89. +6 −16 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/{bit/BitCom.java → control/Controller.java}
  90. +14 −27 ...-exec/src/main/java/org/apache/drill/exec/rpc/{bit/BitComImpl.java → control/ControllerImpl.java}
  91. +8 −8 ...ache/drill/exec/rpc/{bit/BitComDefaultInstanceHandler.java → control/DefaultInstanceHandler.java}
  92. +22 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/SendProgress.java
  93. +113 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/control/WorkEventBus.java
  94. +55 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/BitServerConnection.java
  95. +100 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClient.java
  96. +80 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataClientConnection.java
  97. +63 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java
  98. +53 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionManager.java
  99. +58 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataDefaultInstanceHandler.java
  100. +38 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataProtobufLengthDecoder.java
  101. +31 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandler.java
  102. +64 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataResponseHandlerImpl.java
  103. +42 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataRpcConfig.java
  104. +128 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataServer.java
  105. +94 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataTunnel.java
  106. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/{bit → data}/SendProgress.java
  107. +10 −8 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java
  108. +38 −0 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserProtobufLengthDecoder.java
  109. +20 −5 exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java
  110. +8 −1 exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
  111. +3 −3 exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
  112. +23 −8 exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
  113. +22 −11 exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java
  114. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
  115. +35 −0 exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java
  116. +0 −2 exec/java-exec/src/main/java/org/apache/drill/exec/util/AtomicState.java
  117. +0 −1 exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
  118. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/work/{foreman → }/ErrorHelper.java
  119. +1 −2 exec/java-exec/src/main/java/org/apache/drill/exec/work/QueryWorkUnit.java
  120. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/work/StatusProvider.java
  121. +33 −16 exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
  122. +6 −8 ...va/org/apache/drill/exec/work/batch/{AbstractFragmentCollector.java → AbstractDataCollector.java}
  123. +0 −211 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/BitComHandlerImpl.java
  124. +150 −0 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/ControlHandlerImpl.java
  125. +7 −7 .../src/main/java/org/apache/drill/exec/work/batch/{BitComHandler.java → ControlMessageHandler.java}
  126. +6 −6 ...va-exec/src/main/java/org/apache/drill/exec/work/batch/{BatchCollector.java → DataCollector.java}
  127. +13 −15 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
  128. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/MergingCollector.java
  129. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/PartitionedCollector.java
  130. +19 −8 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/RawBatchBuffer.java
  131. +17 −16 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
  132. +33 −13 exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
  133. +8 −7 exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
  134. +1 −1 exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/FragmentStatusListener.java
  135. +29 −22 ...rc/main/java/org/apache/drill/exec/work/foreman/{RunningFragmentManager.java → QueryManager.java}
  136. +39 −0 exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/RootStatusReporter.java
  137. +8 −8 ...ache/drill/exec/work/{AbstractFragmentRunnerListener.java → fragment/AbstractStatusReporter.java}
  138. +27 −15 ...c/src/main/java/org/apache/drill/exec/work/{FragmentRunner.java → fragment/FragmentExecutor.java}
  139. +14 −9 ...in/java/org/apache/drill/exec/work/fragment/{IncomingFragmentHandler.java → FragmentManager.java}
  140. +17 −17 ...va/org/apache/drill/exec/work/fragment/{RemoteFragmentHandler.java → NonRootFragmentManager.java}
  141. +8 −9 .../apache/drill/exec/work/{RemoteFragmentRunnerListener.java → fragment/NonRootStatusReporter.java}
  142. +13 −10 ...n/java/org/apache/drill/exec/work/fragment/{LocalFragmentHandler.java → RootFragmentManager.java}
  143. +6 −3 ...main/java/org/apache/drill/exec/work/{FragmentRunnerListener.java → fragment/StatusReporter.java}
  144. +2 −2 exec/java-exec/src/main/java/org/apache/drill/exec/work/user/UserWorker.java
  145. +6 −3 exec/java-exec/src/main/resources/drill-module.conf
  146. +6 −5 exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
  147. +8 −7 exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
  148. +2 −2 exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
  149. +8 −1 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
  150. +19 −19 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
  151. +4 −2 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java
  152. +9 −18 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java
  153. +105 −72 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
  154. +12 −10 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
  155. +4 −2 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
  156. +6 −4 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
  157. +10 −8 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
  158. +7 −5 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
  159. +1 −1 ...ava-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
  160. +7 −8 ...est/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java
  161. +4 −2 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
  162. +6 −4 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
  163. +4 −2 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
  164. +4 −2 .../java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
  165. +4 −2 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
  166. +4 −2 exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
  167. +2 −4 exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java
  168. +5 −7 exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java
  169. +2 −2 exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
  170. +110 −38 exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java
  171. +7 −7 exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
  172. +9 −4 exec/java-exec/src/test/resources/logback.xml
  173. +5,438 −0 protocol/src/main/java/org/apache/drill/exec/proto/BitControl.java
  174. +2,228 −0 protocol/src/main/java/org/apache/drill/exec/proto/BitData.java
  175. +160 −79 protocol/src/main/java/org/apache/drill/exec/proto/CoordinationProtos.java
  176. +212 −6,566 protocol/src/main/java/org/apache/drill/exec/proto/ExecProtos.java
  177. +102 −330 protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java
  178. +136 −45 protocol/src/main/java/org/apache/drill/exec/proto/UserProtos.java
  179. +83 −0 protocol/src/main/protobuf/BitControl.proto
  180. +34 −0 protocol/src/main/protobuf/BitData.proto
  181. +3 −2 protocol/src/main/protobuf/Coordination.proto
  182. +0 −83 protocol/src/main/protobuf/ExecutionProtos.proto
  183. +3 −2 protocol/src/main/protobuf/User.proto
  184. +18 −17 protocol/src/main/protobuf/UserBitShared.proto
  185. +1 −1 sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.common.config;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
import java.net.URL;
import java.util.Collection;
import java.util.List;
@@ -35,20 +37,26 @@
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
public final class DrillConfig extends NestedConfig{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfig.class);
private final ObjectMapper mapper;
+ private final ImmutableList<String> startupArguments;
+ @SuppressWarnings("restriction") private static final long MAX_DIRECT_MEMORY = sun.misc.VM.maxDirectMemory();
@SuppressWarnings("unchecked")
private volatile List<Queue<Object>> sinkQueues = new CopyOnWriteArrayList<Queue<Object>>(new Queue[1]);
+
+ @SuppressWarnings("restriction")
@VisibleForTesting
public DrillConfig(Config config) {
super(config);
+
mapper = new ObjectMapper();
SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
.addDeserializer(LogicalExpression.class, new LogicalExpression.De(this));
@@ -61,8 +69,16 @@ public DrillConfig(Config config) {
mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
mapper.registerSubtypes(StorageEngineConfigBase.getSubTypes(this));
+ RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
+ this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());
};
+
+
+
+ public List<String> getStartupArguments(){
+ return startupArguments;
+ }
/**
* Create a DrillConfig object using the default config file name
@@ -155,4 +171,14 @@ public ObjectMapper getMapper(){
public String toString(){
return this.root().render();
}
+
+ public static void main(String[] args) throws Exception{
+ //"-XX:MaxDirectMemorySize"
+ DrillConfig config = DrillConfig.create();
+
+ }
+
+ public static long getMaxDirectMemory(){
+ return MAX_DIRECT_MEMORY;
+ }
}
@@ -30,7 +30,7 @@
import java.nio.channels.GatheringByteChannel;
import java.nio.channels.ScatteringByteChannel;
-final class PooledUnsafeDirectByteBufL extends PooledByteBufL<ByteBuffer> {
+public final class PooledUnsafeDirectByteBufL extends PooledByteBufL<ByteBuffer> {
private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
View
@@ -1,21 +1,16 @@
<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>exec-parent</artifactId>
@@ -36,6 +31,11 @@
<artifactId>asm-commons</artifactId>
<version>4.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-pool2</artifactId>
+ <version>2.1</version>
+ </dependency>
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
@@ -273,7 +273,7 @@
</resource>
<resource>
<directory>target/generated-sources</directory>
-<!-- <include>*/org</include> -->
+ <!-- <include>*/org</include> -->
<filtering>true</filtering>
</resource>
</resources>
@@ -47,8 +47,9 @@
public static final String TRACE_DUMP_FILESYSTEM = "drill.exec.trace.filesystem";
public static final String TEMP_DIRECTORIES = "drill.exec.tmp.directories";
public static final String TEMP_FILESYSTEM = "drill.exec.tmp.filesystem";
- public static final String SPOOLING_BUFFER_IMPL = "drill.exec.spooling.impl";
- public static final String SPOOLING_BUFFER_DELETE = "drill.exec.spooling.delete";
- public static final String SPOOLING_BUFFER_MEMORY = "drill.exec.spooling.size";
+ public static final String INCOMING_BUFFER_IMPL = "drill.exec.buffer.impl";
+ public static final String INCOMING_BUFFER_SIZE = "drill.exec.buffer.size"; // incoming buffer size (number of batches)
+ public static final String SPOOLING_BUFFER_DELETE = "drill.exec.buffer.spooling.delete";
+ public static final String SPOOLING_BUFFER_MEMORY = "drill.exec.buffer.spooling.size";
}
@@ -20,8 +20,8 @@
import java.io.Closeable;
import org.apache.drill.exec.exception.DrillbitStartupException;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
public interface DistributedCache extends Closeable{
@@ -19,10 +19,10 @@
import java.util.concurrent.TimeUnit;
-public interface DistributedMap<V> {
+public interface DistributedMap<V extends DrillSerializable> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DistributedMap.class);
- public DrillSerializable get(String key);
- public void put(String key, DrillSerializable value);
- public void putIfAbsent(String key, DrillSerializable value);
- public void putIfAbsent(String key, DrillSerializable value, long ttl, TimeUnit timeUnit);
+ public V get(String key);
+ public void put(String key, V value);
+ public void putIfAbsent(String key, V value);
+ public void putIfAbsent(String key, V value, long ttl, TimeUnit timeUnit);
}
@@ -19,8 +19,8 @@
import java.util.Collection;
-public interface DistributedMultiMap<V> {
+public interface DistributedMultiMap<V extends DrillSerializable> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DistributedMultiMap.class);
- public Collection<DrillSerializable> get(String key);
- public void put(String key, DrillSerializable value);
+ public Collection<V> get(String key);
+ public void put(String key, V value);
}
@@ -22,23 +22,28 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.hazelcast.config.SerializerConfig;
-import com.hazelcast.core.*;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.cache.ProtoBufImpl.HWorkQueueStatus;
import org.apache.drill.exec.cache.ProtoBufImpl.HandlePlan;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.BitControl.WorkQueueStatus;
import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
-import org.apache.drill.exec.proto.ExecProtos.PlanFragment;
-import org.apache.drill.exec.proto.ExecProtos.WorkQueueStatus;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Lists;
import com.hazelcast.config.Config;
-import org.apache.drill.exec.server.DrillbitContext;
+import com.hazelcast.config.SerializerConfig;
+import com.hazelcast.core.DuplicateInstanceNameException;
+import com.hazelcast.core.Hazelcast;
+import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.core.IAtomicLong;
+import com.hazelcast.core.IMap;
+import com.hazelcast.core.ITopic;
+import com.hazelcast.core.Message;
+import com.hazelcast.core.MessageListener;
public class HazelCache implements DistributedCache {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HazelCache.class);
@@ -67,8 +72,9 @@ public void onMessage(Message<HWorkQueueStatus> wrapped) {
public void run() {
Config c = new Config();
- SerializerConfig sc = new SerializerConfig().setImplementation(new HCVectorAccessibleSerializer(allocator))
- .setTypeClass(VectorAccessibleSerializable.class);
+ SerializerConfig sc = new SerializerConfig() //
+ .setImplementation(new HCVectorAccessibleSerializer(allocator)) //
+ .setTypeClass(VectorAccessibleSerializable.class);
c.setInstanceName(instanceName);
c.getSerializationConfig().addSerializerConfig(sc);
instance = getInstanceOrCreateNew(c);
@@ -119,64 +125,62 @@ public void storeFragment(PlanFragment fragment) {
@Override
public <V extends DrillSerializable> DistributedMultiMap<V> getMultiMap(Class<V> clazz) {
- return new HCDistributedMultiMapImpl(this.instance.getMultiMap(clazz.toString()), clazz);
+ com.hazelcast.core.MultiMap<String, V> mmap = this.instance.getMultiMap(clazz.toString());
+ return new HCDistributedMultiMapImpl<V>(mmap, clazz);
}
@Override
public <V extends DrillSerializable> DistributedMap<V> getMap(Class<V> clazz) {
- return new HCDistributedMapImpl(this.instance.getMap(clazz.toString()), clazz);
+ IMap<String, V> imap = this.instance.getMap(clazz.toString());
+ return new HCDistributedMapImpl<V>(imap, clazz);
}
@Override
public Counter getCounter(String name) {
return new HCCounterImpl(this.instance.getAtomicLong(name));
}
- public static class HCDistributedMapImpl<V> implements DistributedMap<V> {
- private IMap<String, DrillSerializable> m;
- private Class<V> clazz;
+ public static class HCDistributedMapImpl<V extends DrillSerializable> implements DistributedMap<V> {
+ private IMap<String, V> m;
- public HCDistributedMapImpl(IMap m, Class<V> clazz) {
+ public HCDistributedMapImpl(IMap<String, V> m, Class<V> clazz) {
this.m = m;
- this.clazz = clazz;
}
- public DrillSerializable get(String key) {
+ public V get(String key) {
return m.get(key);
}
- public void put(String key, DrillSerializable value) {
+ public void put(String key, V value) {
m.put(key, value);
}
- public void putIfAbsent(String key, DrillSerializable value) {
+ public void putIfAbsent(String key, V value) {
m.putIfAbsent(key, value);
}
- public void putIfAbsent(String key, DrillSerializable value, long ttl, TimeUnit timeunit) {
+ public void putIfAbsent(String key, V value, long ttl, TimeUnit timeunit) {
m.putIfAbsent(key, value, ttl, timeunit);
}
}
- public static class HCDistributedMultiMapImpl<V> implements DistributedMultiMap<V> {
- private com.hazelcast.core.MultiMap<String, DrillSerializable> mmap;
- private Class<V> clazz;
+ public static class HCDistributedMultiMapImpl<V extends DrillSerializable> implements DistributedMultiMap<V> {
+ private com.hazelcast.core.MultiMap<String, V> mmap;
- public HCDistributedMultiMapImpl(com.hazelcast.core.MultiMap mmap, Class<V> clazz) {
+ public HCDistributedMultiMapImpl(com.hazelcast.core.MultiMap<String, V> mmap, Class<V> clazz) {
this.mmap = mmap;
- this.clazz = clazz;
}
- public Collection<DrillSerializable> get(String key) {
- List<DrillSerializable> list = Lists.newArrayList();
- for (DrillSerializable v : mmap.get(key)) {
+ public Collection<V> get(String key) {
+ List<V> list = Lists.newArrayList();
+ for (V v : mmap.get(key)) {
list.add(v);
}
return list;
}
@Override
- public void put(String key, DrillSerializable value) {
+ public void put(String key, V value) {
mmap.put(key, value);
}
}
Oops, something went wrong.

0 comments on commit fb35433

Please sign in to comment.