TEIID-5422: Buffer cleaner running too often in some cases (changing …

…the cleaning interval to prevent thrashing)

build/kits/jboss-as7/overlay/docs/teiid/teiid-releasenotes.html

@@ -243,6 +243,9 @@ <h4 class="western">from ${project.version}</h4>
 	<p style="margin-bottom: 0in">
 <a href='https://issues.jboss.org/browse/TEIID-5417'>TEIID-5417</a> - Performance degradation with large in predicate against temp table (correcting array typing and producing a set of constants from the
     dependent criteria processor for temp tables)
+<li/>
+	<p style="margin-bottom: 0in">
+<a href='https://issues.jboss.org/browse/TEIID-5422'>TEIID-5422</a> - Buffer cleaner running too often in some cases (changing the cleaning interval to prevent thrashing)
 </ul>
 
 <h4 class="western">from  8.12.14.6_4</h4>  

engine/src/main/java/org/teiid/common/buffer/impl/BufferFrontedFileStoreCache.java

@@ -927,7 +927,9 @@ public boolean addToCacheGroup(Long gid, Long oid) {
 		if (map == null) {
 			return false;
 		}
-		map.put(oid, null);
+		if (map.put(oid, null) != null) {
+		    throw new AssertionError("already added"); //$NON-NLS-1$
+		}
 		return true;
 	}
 

engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java

@@ -117,7 +117,9 @@ public void run() {
 				synchronized (this) {
 					impl.cleaning.set(false);
 					try {
-						this.wait(100);
+					    //wait for a while before cleanning more
+					    //we'll be woken up by a processing thread if needed
+					    this.wait(5000);
 					} catch (InterruptedException e) {
 						break;
 					}
@@ -1107,7 +1109,7 @@ private void remove(CacheEntry ce, boolean inMemory) {
 		    long result = activeBatchBytes.addAndGet(-ce.getSizeEstimate());
             assert result >= 0 || !LrfuEvictionQueue.isSuspectSize(activeBatchBytes);
 		}
-		assert activeBatchBytes.get() >= 0;
+		assert !LrfuEvictionQueue.isSuspectSize(activeBatchBytes);
 		Serializer<?> s = ce.getSerializer();
 		if (s != null) {
 			removeFromCache(s.getId(), ce.getId());

test-integration/perf/src/test/java/org/teiid/query/eval/AutoGenExecutionFactory.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags and
+ * the COPYRIGHT.txt file distributed with this work.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.teiid.query.eval;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Clob;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.teiid.core.types.ClobImpl;
+import org.teiid.core.types.DataTypeManager;
+import org.teiid.core.types.InputStreamFactory;
+import org.teiid.core.types.Streamable;
+import org.teiid.language.NamedTable;
+import org.teiid.language.QueryExpression;
+import org.teiid.query.unittest.TimestampUtil;
+import org.teiid.runtime.HardCodedExecutionFactory;
+
+class AutoGenExecutionFactory extends HardCodedExecutionFactory {
+
+    private static final Character CHAR_VAL = new Character('c');
+    private static final Byte BYTE_VAL = new Byte((byte)0);
+    private static final Clob CLOB_VAL = new ClobImpl(new InputStreamFactory() {
+        @Override
+        public InputStream getInputStream() throws IOException {
+            return new ByteArrayInputStream("hello world".getBytes(Streamable.CHARSET));
+        }
+    }, -1);
+    private static final Boolean BOOLEAN_VAL = Boolean.FALSE;
+    private static final BigInteger BIG_INTEGER_VAL = new BigInteger("0"); //$NON-NLS-1$
+    private static final java.sql.Date SQL_DATE_VAL = TimestampUtil.createDate(69, 11, 31);
+    private static final java.sql.Time TIME_VAL = new java.sql.Time(0);
+    private static final java.sql.Timestamp TIMESTAMP_VAL = new java.sql.Timestamp(0);
+
+    private static final java.sql.Timestamp IN_RANGE = TimestampUtil.createTimestamp(117, 2, 0, 0, 0, 0, 0);
+
+    static Object getValue(Class<?> type, int row, int max) {
+        if(type.equals(DataTypeManager.DefaultDataClasses.STRING)) {
+            //return "a";
+            return String.valueOf(10000000 + row%max);
+            //return "" + String.valueOf(100000+(row/4)*4); //sample;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.INTEGER)) {
+            return row%max;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.SHORT)) { 
+            return (short)row%max;    
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.LONG)) {
+            return (long)row%max;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.FLOAT)) {
+            return (float)(row%max);
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.DOUBLE)) {
+            return (double)(row%max);
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.CHAR)) {
+            return CHAR_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.BYTE)) {
+            return BYTE_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.BOOLEAN)) {
+            return BOOLEAN_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.BIG_INTEGER)) {
+            return BIG_INTEGER_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.BIG_DECIMAL)) {
+            return BigDecimal.valueOf(row%max);
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.DATE)) {
+            return SQL_DATE_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.TIME)) {
+            return TIME_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.TIMESTAMP)) {
+            if (row%3==0) {
+                return IN_RANGE;
+            }
+            return TIMESTAMP_VAL;
+        } else if(type.equals(DataTypeManager.DefaultDataClasses.CLOB)) {
+            return CLOB_VAL;
+        } else {
+            return null;
+        }
+    }
+
+    private Map<String, Integer> rowCounts = new HashMap<String, Integer>();
+
+    @Override
+    protected List<? extends List<?>> getData(QueryExpression command) {
+        List<? extends List<?>> result = super.getData(command);
+        if (result != null) {
+            return result;
+        }
+        Class<?>[] types = command.getColumnTypes();
+        String name = ((NamedTable)command.getProjectedQuery().getFrom().get(0)).getMetadataObject().getName();
+        int rowCount = rowCounts.get(name);
+        List resultList = new ArrayList();
+        for (int i = 0; i < rowCount; i++) {
+            List row = new ArrayList(); 
+            for (int j = 0; j < types.length; j++) {
+                Object value = getValue(types[j], i, Integer.MAX_VALUE);
+                row.add(value);
+            }
+            resultList.add(row);
+        }
+        return resultList;
+    }
+
+    public void addRowCount(String name, int count) {
+        this.rowCounts.put(name, count);
+    }
+}

test-integration/perf/src/test/java/org/teiid/query/eval/TestMaterializationPerformance.java

@@ -31,6 +31,8 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.teiid.adminapi.impl.ModelMetaData;
+import org.teiid.core.util.ObjectConverterUtil;
+import org.teiid.core.util.UnitTestUtil;
 import org.teiid.jdbc.AbstractQueryTest;
 import org.teiid.runtime.EmbeddedConfiguration;
 import org.teiid.runtime.EmbeddedServer;
@@ -86,4 +88,22 @@ public class TestMaterializationPerformance extends AbstractQueryTest {
 		}
 	}
 
+	@Test public void testLargeWithoutKeys() throws Exception {
+        ModelMetaData mmm = new ModelMetaData();
+        mmm.setName("test");
+        mmm.addSourceMetadata("ddl", ObjectConverterUtil.convertFileToString(UnitTestUtil.getTestDataFile("large_mat.ddl")));
+        mmm.addSourceMapping("x", "hc", null);
+        AutoGenExecutionFactory agef = new AutoGenExecutionFactory();
+        agef.addRowCount("tbl_f", 84717);
+        agef.addRowCount("tbl_y", 85248);
+        agef.addRowCount("tbl_u", 327955);
+        es.addTranslator("hc", agef);
+        es.deployVDB("test", mmm);
+        setConnection(es.getDriver().connect("jdbc:teiid:test", null));
+        for (int i = 0; i < 15; i++) {
+            execute("SELECT y.iamak, u.penog, y.bibdd, SUM((y.odpdb * f.apcmd)) FROM /*+ PRESERVE */ ((v_y AS y LEFT OUTER JOIN v_f AS f ON f.lbjaa = y.bggnl) INNER JOIN (SELECT DISTINCT hjobj, penog FROM /*+ MAKENOTDEP */ v_u) AS u ON u.hjobj = y.jmafi) WHERE icfbj BETWEEN {ts'2017-01-01 00:00:00.0'} AND {ts'2018-01-01 00:00:00.0'} GROUP BY y.iamak, u.penog, y.bibdd ORDER BY y.iamak, u.penog, y.bibdd");
+            assertEquals(28416, getRowCount());
+        }
+    }
+
 }