TEIID-5525

adding a flag to optionally enforce the single buffer max increasing the default estimate of max size
teiid · Nov 1, 2018 · 233c88b · 233c88b
1 parent 28cdf90
commit 233c88b
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 3 deletions.
diff --git a/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java b/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
@@ -198,6 +198,7 @@ final class BatchManagerImpl implements BatchManager, Serializer<List<? extends
 		private long currentSize;
 		private long rowsSampled;
         private boolean removed;
+        private boolean sizeWarning;
 
 		private BatchManagerImpl(Long newID, Class<?>[] types) {
 			this.id = newID;
@@ -296,8 +297,14 @@ private void updateEstimates(long sizeEstimate, boolean remove) throws TeiidComp
             }
             currentSize += sizeEstimate;
             if (!remove && currentSize > maxBatchManagerSizeEstimate) {
-                this.remove();
-                throw new TeiidComponentException(QueryPlugin.Event.TEIID31261, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31261, maxBatchManagerSizeEstimate, id));
+                if (enforceMaxBatchManagerSizeEstimate) {
+                    this.remove();
+                    throw new TeiidComponentException(QueryPlugin.Event.TEIID31261, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31261, maxBatchManagerSizeEstimate, id));
+                }
+                if (!sizeWarning) {
+                    LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31261, maxBatchManagerSizeEstimate, id));
+                    sizeWarning = true;
+                }
             }
             CommandContext threadLocalContext = CommandContext.getThreadLocalContext();
             if (threadLocalContext != null) {
@@ -523,6 +530,7 @@ protected boolean removeEldestEntry(Map.Entry<Long,BatchSoftReference> eldest) {
 
     private long maxFileStoreLength = Long.MAX_VALUE;
     private long maxBatchManagerSizeEstimate = Long.MAX_VALUE;
+    private boolean enforceMaxBatchManagerSizeEstimate = true;
     private long maxSessionBatchManagerSizeEstimate = Long.MAX_VALUE;
 
     public BufferManagerImpl() {
@@ -723,9 +731,13 @@ public void initialize() throws TeiidComponentException {
 		if (this.storageManager != null) {
     		long max = this.storageManager.getMaxStorageSpace();
             this.maxFileStoreLength  = (max/maxActivePlans)>>2;
-            this.maxBatchManagerSizeEstimate = (long)(.8*((((long)this.getMaxReserveKB())<<10) + max + cache.getMemoryBufferSpace())/Math.sqrt(maxActivePlans));
+		    //note the increase of the storage / memory buffer values here to normalize to heap estimates 
+		    //batches in serialized form are much more compact
+
+            this.maxBatchManagerSizeEstimate = (long)(.8*((((long)this.getMaxReserveKB())<<10) + (max<<3) + (cache.getMemoryBufferSpace()<<3))/Math.sqrt(maxActivePlans));
             if (this.options != null) {
                 this.maxSessionBatchManagerSizeEstimate = this.options.getMaxSessionBufferSizeEstimate();
+                this.enforceMaxBatchManagerSizeEstimate = this.options.isEnforceSingleMaxBufferSizeEstimate();
             }
             this.maxBatchManagerSizeEstimate = Math.min(maxBatchManagerSizeEstimate, maxSessionBatchManagerSizeEstimate);
 		}
@@ -1444,5 +1456,10 @@ public void setMaxBatchManagerSizeEstimate(
             long maxBatchManagerSizeEstimate) {
         this.maxBatchManagerSizeEstimate = maxBatchManagerSizeEstimate;
     }
+
+	public void setEnforceMaxBatchManagerSizeEstimate(
+            boolean enforceMaxBatchManagerSizeEstimate) {
+        this.enforceMaxBatchManagerSizeEstimate = enforceMaxBatchManagerSizeEstimate;
+    }
 
 }
diff --git a/engine/src/main/java/org/teiid/query/util/Options.java b/engine/src/main/java/org/teiid/query/util/Options.java
@@ -38,6 +38,7 @@ public class Options {
 	public static final String AGGRESSIVE_JOIN_GROUPING = "org.teiid.aggressiveJoinGrouping"; //$NON-NLS-1$
 	public static final String MAX_SESSION_BUFFER_SIZE_ESTIMATE = "org.teiid.maxSessionBufferSizeEstimate"; //$NON-NLS-1$
 	public static final String TRACING_WITH_ACTIVE_SPAN_ONLY = "org.teiid.tracingWithActiveSpanOnly"; //$NON-NLS-1$
+	public static final String ENFORCE_SINGLE_MAX_BUFFER_SIZE_ESTIMATE = "org.teiid.enforceSingleMaxBufferSizeEstimate"; //$NON-NLS-1$
 
 	private Properties properties;
 	private boolean subqueryUnnestDefault = false;
@@ -51,6 +52,7 @@ public class Options {
 	private boolean aggressiveJoinGrouping = true;
 	private long maxSessionBufferSizeEstimate = Long.MAX_VALUE;
 	private boolean tracingWithActiveSpanOnly = true;
+	private boolean enforceSingleMaxBufferSizeEstimate = true;
 
 	public Properties getProperties() {
 		return properties;
@@ -205,4 +207,19 @@ public Options tracingWithActiveSpanOnly(boolean b) {
 	    this.tracingWithActiveSpanOnly = b;
 	    return this;
 	}
+
+	public boolean isEnforceSingleMaxBufferSizeEstimate() {
+        return enforceSingleMaxBufferSizeEstimate;
+    }
+
+	public void setEnforceSingleMaxBufferSizeEstimate(
+            boolean enforceSingleMaxBufferSizeEstimate) {
+        this.enforceSingleMaxBufferSizeEstimate = enforceSingleMaxBufferSizeEstimate;
+    }
+
+	public Options enforceSingleMaxBufferSizeEstimate(
+            boolean b) {
+        this.enforceSingleMaxBufferSizeEstimate = b;
+        return this;
+    }
 }
diff --git a/engine/src/test/java/org/teiid/query/processor/TestTempTables.java b/engine/src/test/java/org/teiid/query/processor/TestTempTables.java
@@ -739,4 +739,20 @@ private void helpTestDelete() throws Exception {
         fail();
     }
 
+	@Test public void testLargeNoException() throws Exception {
+        BufferManagerImpl bm = BufferManagerFactory.getTestBufferManager(20480, 256);
+        bm.setMaxBatchManagerSizeEstimate(100000);
+        bm.setEnforceMaxBatchManagerSizeEstimate(false);
+        FakeDataManager fdm = new FakeDataManager();
+        TestProcessor.sampleData1(fdm);
+        setUp(RealMetadataFactory.example1Cached(), fdm, bm);
+
+        execute("create local temporary table x (e1 string, e2 string)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
+        execute("insert into x (e2, e1) values (1, 1)", new List[] {Arrays.asList(1)}); //$NON-NLS-1$
+
+        for (int i = 0; i < 10; i++) {
+            execute("insert into x (e2, e1) select * from x", new List[] {Arrays.asList(1<<i)}); //$NON-NLS-1$
+        }
+    }
+
 }