TEIID-5525: add a flag to revert to the prior behavior (TEIID-4557) (…

…adding a flag to optionally enforce the single buffer max, increasing the default estimate of max size, adding details about the system property to the message)

build/kits/jboss-as7/overlay/docs/teiid/teiid-releasenotes.html

@@ -230,6 +230,12 @@ <h4 class="western">from ${project.version}</h4>
 <li/>
 	<p style="margin-bottom: 0in">
 <a href='https://issues.jboss.org/browse/TEIID-5410'>TEIID-5410</a> - Incorrect result from timestampadd with 1 billion or more fractional seconds (correcting timestampadd behavior)
+<li/>
+	<p style="margin-bottom: 0in">
+<a href='https://issues.jboss.org/browse/TEIID-5461'>TEIID-5461</a> - adding parsing/pushdown support for the window frame clause, adding a validation of RANGE and int bounds
+<li/>
+	<p style="margin-bottom: 0in">
+<a href='https://issues.jboss.org/browse/TEIID-5525'>TEIID-5525</a> - add a flag to revert to the prior behavior (TEIID-4557) (adding a flag to optionally enforce the single buffer max, increasing the default estimate of max size, adding details about the system property to the message)
 </ul>
 
 <h4 class="western">from 8.12.16.6_4</h4>  

engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java

@@ -189,6 +189,7 @@ final class BatchManagerImpl implements BatchManager, Serializer<List<? extends
 		private long currentSize;
 		private long rowsSampled;
         private boolean removed;
+        private boolean sizeWarning;
 
 		private BatchManagerImpl(Long newID, Class<?>[] types) {
 			this.id = newID;
@@ -290,8 +291,14 @@ private void updateEstimates(long sizeEstimate, boolean remove) throws TeiidComp
             }
             currentSize += sizeEstimate;
             if (!remove && currentSize > maxBatchManagerSizeEstimate) {
-                this.remove();
-                throw new TeiidComponentException(QueryPlugin.Event.TEIID31261, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31261, maxBatchManagerSizeEstimate, id));
+                if (enforceMaxBatchManagerSizeEstimate) {
+                    this.remove();
+                    throw new TeiidComponentException(QueryPlugin.Event.TEIID31261, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31261, maxBatchManagerSizeEstimate, id));
+                }
+                if (!sizeWarning) {
+                    LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31292, maxBatchManagerSizeEstimate, id));
+                    sizeWarning = true;
+                }
             }
             CommandContext threadLocalContext = CommandContext.getThreadLocalContext();
             if (threadLocalContext != null) {
@@ -516,6 +523,7 @@ protected boolean removeEldestEntry(Map.Entry<Long,BatchSoftReference> eldest) {
 
     private long maxFileStoreLength = Long.MAX_VALUE;
     private long maxBatchManagerSizeEstimate = Long.MAX_VALUE;
+    private boolean enforceMaxBatchManagerSizeEstimate = false;
     private long maxSessionBatchManagerSizeEstimate = Long.MAX_VALUE;
 
 	public BufferManagerImpl() {
@@ -703,9 +711,13 @@ public void initialize() throws TeiidComponentException {
 		if (this.storageManager != null) {
     		long max = this.storageManager.getMaxStorageSpace();
             this.maxFileStoreLength  = (max/maxActivePlans)>>2;
-            this.maxBatchManagerSizeEstimate = (long)(.8*((((long)this.getMaxReserveKB())<<10) + max + cache.getMemoryBufferSpace())/Math.sqrt(maxActivePlans));
+		    //note the increase of the storage / memory buffer values here to normalize to heap estimates 
+		    //batches in serialized form are much more compact
+
+            this.maxBatchManagerSizeEstimate = (long)(.8*((((long)this.getMaxReserveKB())<<10) + (max<<3) + (cache.getMemoryBufferSpace()<<3))/Math.sqrt(maxActivePlans));
             if (this.options != null) {
                 this.maxSessionBatchManagerSizeEstimate = this.options.getMaxSessionBufferSizeEstimate();
+                this.enforceMaxBatchManagerSizeEstimate = this.options.isEnforceSingleMaxBufferSizeEstimate();
             }
             this.maxBatchManagerSizeEstimate = Math.min(maxBatchManagerSizeEstimate, maxSessionBatchManagerSizeEstimate);
 		}
@@ -1417,5 +1429,10 @@ public void setMaxBatchManagerSizeEstimate(
             long maxBatchManagerSizeEstimate) {
         this.maxBatchManagerSizeEstimate = maxBatchManagerSizeEstimate;
     }
+
+	public void setEnforceMaxBatchManagerSizeEstimate(
+            boolean enforceMaxBatchManagerSizeEstimate) {
+        this.enforceMaxBatchManagerSizeEstimate = enforceMaxBatchManagerSizeEstimate;
+    }
 
 }

engine/src/main/java/org/teiid/query/QueryPlugin.java

@@ -638,6 +638,7 @@ public static enum Event implements BundleUtil.Event{
         TEIID31285, 
         TEIID31286,
         TEIID31287, 
-        TEIID31288
+        TEIID31288,
+        TEIID31292
 	}
 }

engine/src/main/java/org/teiid/query/util/Options.java

@@ -41,6 +41,7 @@ public class Options {
 	public static final String AGGRESSIVE_JOIN_GROUPING = "org.teiid.aggressiveJoinGrouping"; //$NON-NLS-1$
 	public static final String MAX_SESSION_BUFFER_SIZE_ESTIMATE = "org.teiid.maxSessionBufferSizeEstimate"; //$NON-NLS-1$
 	public static final String ASSUME_MATCHING_COLLATION = "org.teiid.assumeMatchingCollation"; //$NON-NLS-1$
+	public static final String ENFORCE_SINGLE_MAX_BUFFER_SIZE_ESTIMATE = "org.teiid.enforceSingleMaxBufferSizeEstimate"; //$NON-NLS-1$
 
 	private Properties properties;
 	private boolean subqueryUnnestDefault = false;
@@ -53,6 +54,7 @@ public class Options {
 	private boolean aggressiveJoinGrouping = true;
 	private long maxSessionBufferSizeEstimate = Long.MAX_VALUE;
 	private boolean assumeMatchingCollation = true;
+	private boolean enforceSingleMaxBufferSizeEstimate = false;
 
 	public Properties getProperties() {
 		return properties;
@@ -194,5 +196,19 @@ public Options assumeMatchingCollation(boolean b) {
 		this.assumeMatchingCollation = b;
 		return this;
 	}
-
+
+	public boolean isEnforceSingleMaxBufferSizeEstimate() {
+        return enforceSingleMaxBufferSizeEstimate;
+    }
+
+	public void setEnforceSingleMaxBufferSizeEstimate(
+            boolean enforceSingleMaxBufferSizeEstimate) {
+        this.enforceSingleMaxBufferSizeEstimate = enforceSingleMaxBufferSizeEstimate;
+    }
+
+	public Options enforceSingleMaxBufferSizeEstimate(
+            boolean b) {
+        this.enforceSingleMaxBufferSizeEstimate = b;
+        return this;
+    }
 }

engine/src/main/resources/org/teiid/query/i18n.properties

@@ -1569,7 +1569,8 @@ TEIID31174=Result set or count exceeds the maximum integer value.
 
 TEIID31175=Array type expected for {0} but was {1}.
 TEIID31560=Max size of temporary lob exceeded: {0}
-TEIID31261=Max estimated size {0} for a single operation/table id {1} has been exceeded.  The server may need to increase the amount of disk or memory available, or decrease the number of max active plans.
+TEIID31261=Max estimated size {0} for a single operation/table id {1} has been exceeded.  The server may need to increase the amount of disk or memory available, or decrease the number of max active plans.  The system property org.teiid.enforceSingleMaxBufferSizeEstimate may be set to false to turn this into a warning.
+TEIID31292=Max estimated size {0} for a single operation/table id {1} has been exceeded.  The server may need to increase the amount of disk or memory available, or decrease the number of max active plans.  The system property org.teiid.enforceSingleMaxBufferSizeEstimate may be set to true to turn this into an error.
 
 TEIID31262=Max estimated size {0} for a single session {1} has been exceeded by operation/table id {2}.  The server may need to increase the amount of disk or memory available, or decrease the number of max active plans.
 TEIID31263=Error loading functions from {0}: {1}

engine/src/test/java/org/teiid/common/buffer/impl/TestBufferManagerImpl.java

@@ -99,6 +99,7 @@ public long getMaxStorageSpace() {
         });
         bufferManager.setMaxReserveKB(10);
         bufferManager.setMaxActivePlans(20);
+        bufferManager.setEnforceMaxBatchManagerSizeEstimate(true);
         bufferManager.initialize();
         TupleBuffer tb = bufferManager.createTupleBuffer(Arrays.asList(new ElementSymbol("x", null, String.class)), "x", TupleSourceType.PROCESSOR);
         //fill one batch, which should then exceed the max

engine/src/test/java/org/teiid/query/processor/TestTempTables.java

@@ -693,6 +693,7 @@ private void helpTestDelete() throws Exception {
 	@Test public void testLargeException() throws Exception {
 	    BufferManagerImpl bm = BufferManagerFactory.getTestBufferManager(20480, 256);
 	    bm.setMaxBatchManagerSizeEstimate(100000);
+	    bm.setEnforceMaxBatchManagerSizeEstimate(true);
 	    FakeDataManager fdm = new FakeDataManager();
         TestProcessor.sampleData1(fdm);
 	    setUp(RealMetadataFactory.example1Cached(), fdm, bm);
@@ -711,4 +712,20 @@ private void helpTestDelete() throws Exception {
         fail();
     }
 
+	@Test public void testLargeNoException() throws Exception {
+        BufferManagerImpl bm = BufferManagerFactory.getTestBufferManager(20480, 256);
+        bm.setMaxBatchManagerSizeEstimate(100000);
+        bm.setEnforceMaxBatchManagerSizeEstimate(false);
+        FakeDataManager fdm = new FakeDataManager();
+        TestProcessor.sampleData1(fdm);
+        setUp(RealMetadataFactory.example1Cached(), fdm, bm);
+
+        execute("create local temporary table x (e1 string, e2 string)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
+        execute("insert into x (e2, e1) values (1, 1)", new List[] {Arrays.asList(1)}); //$NON-NLS-1$
+
+        for (int i = 0; i < 10; i++) {
+            execute("insert into x (e2, e1) select * from x", new List[] {Arrays.asList(1<<i)}); //$NON-NLS-1$
+        }
+    }
+
 }