Fixed SmallBankLoader and YCSBLoader for failing test cases.

I forgot to call commit on the last batch in each of the new LoaderThreads.
I added a test-case for ThreadUtil from H-Store.
Fixed TestFileUtil so that it is actually executed (it didn't inherit JUnit).
Other API tweaks and clean ups...

src/com/oltpbenchmark/api/BenchmarkModule.java

@@ -277,7 +277,7 @@ public final void loadDatabase() {
      * Invoke this benchmark's database loader using the given Connection handle
      * @param conn
      */
-    protected final void loadDatabase(Connection conn) {
+    protected final void loadDatabase(final Connection conn) {
         try {
             Loader<? extends BenchmarkModule> loader = this.makeLoaderImpl(conn);
             if (loader != null) {
@@ -290,9 +290,19 @@ protected final void loadDatabase(Connection conn) {
                 // method.
                 List<? extends LoaderThread> loaderThreads = loader.createLoaderTheads();
                 if (loaderThreads != null) {
-                    ThreadUtil.runNewPool(loaderThreads, workConf.getLoaderThreads());
+                    int maxConcurrent = workConf.getLoaderThreads();
+                    assert(maxConcurrent > 0);
+                    if (LOG.isDebugEnabled())
+                        LOG.debug(String.format("Starting %d %s.LoaderThreads [maxConcurrent=%d]",
+                                                loaderThreads.size(),
+                                                loader.getClass().getSimpleName(),
+                                                maxConcurrent));
+                    ThreadUtil.runNewPool(loaderThreads, maxConcurrent);
                 } else {
-                    loader.load();    
+                    if (LOG.isDebugEnabled())
+                        LOG.debug(String.format("Using legacy %s.load() method",
+                                                loader.getClass().getSimpleName()));
+                    loader.load();
                 }
                 conn.commit();
 
@@ -305,6 +315,9 @@ protected final void loadDatabase(Connection conn) {
                                        this.benchmarkName.toUpperCase());
             throw new RuntimeException(msg, ex);
         }
+        if (LOG.isDebugEnabled())
+            LOG.debug(String.format("Finished loading the %s database",
+                                    this.getBenchmarkName().toUpperCase()));
     }
 
     /**

src/com/oltpbenchmark/api/Loader.java

@@ -127,16 +127,6 @@ public DatabaseType getDatabaseType() {
         return (this.workConf.getDBType());
     }
 
-    /**
-     * Hackishly return true if we are using the same type as we use in our unit
-     * tests
-     * 
-     * @return
-     */
-    protected final boolean isTesting() {
-        return (this.workConf.getDBType() == DatabaseType.TEST_TYPE);
-    }
-
     /**
      * Return the database's catalog
      */

src/com/oltpbenchmark/api/TransactionTypes.java

@@ -21,6 +21,7 @@
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Spliterator;
 
 import org.apache.commons.collections15.map.ListOrderedMap;
 
@@ -49,6 +50,12 @@ public int compare(TransactionType o1, TransactionType o2) {
 		} // FOR
 	}
 
+	@Override
+	public Spliterator<TransactionType> spliterator() {
+	    // TODO Auto-generated method stub
+	    return null;
+	}
+
 	public TransactionType getType(String procName) {
 	    return (this.types.get(procName.toUpperCase()));
 	}

src/com/oltpbenchmark/benchmarks/epinions/EpinionsBenchmark.java

@@ -61,7 +61,7 @@ protected List<Worker<? extends BenchmarkModule>> makeWorkersImpl(boolean verbos
             Table t = this.catalog.getTable("USER");
             assert (t != null) : "Invalid table name '" + t + "' " + this.catalog.getTables();
 
-            String userCount = SQLUtil.selectColValues(t, "u_id");
+            String userCount = SQLUtil.selectColValues(this.workConf.getDBType(), t, "u_id");
             Statement stmt = metaConn.createStatement();
             ResultSet res = stmt.executeQuery(userCount);
             ArrayList<String> user_ids = new ArrayList<String>();
@@ -73,7 +73,7 @@ protected List<Worker<? extends BenchmarkModule>> makeWorkersImpl(boolean verbos
             // LIST OF ITEMS AND
             t = this.catalog.getTable("ITEM");
             assert (t != null) : "Invalid table name '" + t + "' " + this.catalog.getTables();
-            String itemCount = SQLUtil.selectColValues(t, "i_id");
+            String itemCount = SQLUtil.selectColValues(this.workConf.getDBType(), t, "i_id");
             res = stmt.executeQuery(itemCount);
             ArrayList<String> item_ids = new ArrayList<String>();
             while (res.next()) {

src/com/oltpbenchmark/benchmarks/hyadapt/HYADAPTBenchmark.java

@@ -52,7 +52,7 @@ protected List<Worker<? extends BenchmarkModule>> makeWorkersImpl(boolean verbos
 
             Table t = this.catalog.getTable("HTABLE");
             assert (t != null) : "Invalid table name '" + t + "' " + this.catalog.getTables();
-            String userCount = SQLUtil.getCountSQL(t);
+            String userCount = SQLUtil.getCountSQL(this.workConf.getDBType(), t);
             Statement stmt = metaConn.createStatement();
             ResultSet res = stmt.executeQuery(userCount);
             int init_record_count = 0;

src/com/oltpbenchmark/benchmarks/smallbank/SmallBankLoader.java

@@ -111,24 +111,26 @@ public void load(Connection conn) throws SQLException {
                     stmtSavings.addBatch();
 
                     if (++batchSize >= SmallBankConstants.BATCH_SIZE) {
-                        this.loadTables();
+                        this.loadTables(conn);
                         batchSize = 0;
                     }
 
                 } // FOR
                 if (batchSize > 0) {
-                    this.loadTables();
+                    this.loadTables(conn);
                 }
             } catch (SQLException ex) {
                 LOG.error("Failed to load data", ex);
                 throw new RuntimeException(ex);
             }
         }
 
-        private void loadTables() throws SQLException {
+        private void loadTables(Connection conn) throws SQLException {
             this.stmtAccts.executeBatch();
             this.stmtSavings.executeBatch();
             this.stmtChecking.executeBatch();
+            conn.commit();
+
         }
     };
 

src/com/oltpbenchmark/benchmarks/ycsb/YCSBLoader.java

@@ -51,8 +51,8 @@ public List<LoaderThread> createLoaderTheads() throws SQLException {
             threads.add(new LoaderThread() {
                 @Override
                 public void load(Connection conn) throws SQLException {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug(String.format("YCSBLoadThread[%d, %d]", start, stop));
+//                    if (LOG.isDebugEnabled())
+                        LOG.info(String.format("YCSBLoadThread[%d, %d]", start, stop));
                     loadRecords(conn, start, stop);
                 }
             });
@@ -87,6 +87,7 @@ private void loadRecords(Connection conn, int start, int stop) throws SQLExcepti
         } // FOR
         if (batch > 0) {
             stmt.executeBatch();
+            conn.commit();
             if (LOG.isDebugEnabled())
                 LOG.debug(String.format("Records Loaded %d / %d", total, this.num_record));
         }

src/com/oltpbenchmark/types/DatabaseType.java

@@ -109,12 +109,6 @@ public boolean shouldIncludeColumnNames() {
     // STATIC METHODS + MEMBERS
     // ----------------------------------------------------------------
 
-    /**
-     * This is the database type that we will use in our unit tests.
-     * This should always be one of the embedded java databases
-     */
-    public static final DatabaseType TEST_TYPE = DatabaseType.HSQLDB; 
-
     protected static final Map<Integer, DatabaseType> idx_lookup = new HashMap<Integer, DatabaseType>();
     protected static final Map<String, DatabaseType> name_lookup = new HashMap<String, DatabaseType>();
     static {

src/com/oltpbenchmark/util/ExceptionHandlingRunnable.java

@@ -0,0 +1,17 @@
+package com.oltpbenchmark.util;
+
+public abstract class ExceptionHandlingRunnable implements Runnable {
+
+    public abstract void runImpl();
+
+    @Override
+    public final void run() {
+        try {
+            this.runImpl();
+        } catch (Throwable ex) {
+            Thread t = Thread.currentThread();
+            Thread.getDefaultUncaughtExceptionHandler().uncaughtException(t, ex);
+        }
+    }
+
+}

src/com/oltpbenchmark/util/SQLUtil.java

@@ -313,8 +313,8 @@ public static boolean needsColumnSize(int sqlType) {
      * @param table
      * @return SQL for select count execution
      */
-    public static String getCountSQL(Table catalog_tbl) {
-        return SQLUtil.getCountSQL(catalog_tbl, "*");
+    public static String getCountSQL(DatabaseType dbType, Table catalog_tbl) {
+        return SQLUtil.getCountSQL(dbType, catalog_tbl, "*");
     }
 
     /**
@@ -324,9 +324,9 @@ public static String getCountSQL(Table catalog_tbl) {
      * @param col
      * @return SQL for select count execution
      */
-    public static String getCountSQL(Table catalog_tbl, String col) {
-        return String.format("SELECT COUNT(%s) FROM %s",
-                             col, catalog_tbl.getEscapedName());
+    public static String getCountSQL(DatabaseType dbType, Table catalog_tbl, String col) {
+        String tableName = (dbType.shouldEscapeNames() ? catalog_tbl.getEscapedName() : catalog_tbl.getName());
+        return String.format("SELECT COUNT(%s) FROM %s", col, tableName.trim());
     }
 
 
@@ -426,7 +426,7 @@ public static String getMaxColSQL(DatabaseType dbType, Table catalog_tbl, String
         return String.format("SELECT MAX(%s) FROM %s", col, tableName);
     }
 
-    public static String selectColValues(Table catalog_tbl, String col) {
+    public static String selectColValues(DatabaseType dbType, Table catalog_tbl, String col) {
         return String.format("SELECT %s FROM %s",
                 col, catalog_tbl.getEscapedName());
     }

tests/com/oltpbenchmark/api/AbstractTestCase.java

@@ -22,35 +22,58 @@
 import java.util.List;
 import java.util.Random;
 
+import org.apache.log4j.Logger;
+
 import junit.framework.TestCase;
 
 import com.oltpbenchmark.WorkloadConfiguration;
 import com.oltpbenchmark.catalog.Catalog;
 import com.oltpbenchmark.types.DatabaseType;
 import com.oltpbenchmark.util.ClassUtil;
+import com.oltpbenchmark.util.FileUtil;
 
 public abstract class AbstractTestCase<T extends BenchmarkModule> extends TestCase {
 
+    private static final Logger LOG = Logger.getLogger(AbstractTestCase.class);
+
     // HACK
     static {
-//      org.apache.log4j.PropertyConfigurator.configure("/home/pavlo/Documents/OLTPBenchmark/OLTPBenchmark/log4j.properties");
+        String propFile = "/home/pavlo/Documents/OLTPBenchmark/oltpbench/log4j.properties";
+        if (FileUtil.exists(propFile)) {
+            org.apache.log4j.PropertyConfigurator.configure(propFile);    
+        }
     }
 
     // -----------------------------------------------------------------
 
-    // HSQLDB
-    public static final String DB_CONNECTION = "jdbc:hsqldb:mem:";
-    public static final DatabaseType DB_TYPE = DatabaseType.TEST_TYPE;
-
-    // H2
-    // public static final String DB_CONNECTION = "jdbc:h2:mem:";
-    // public static final String DB_JDBC = "org.h2.Driver";
-    // public static final DatabaseType DB_TYPE = DatabaseType.H2;
-
-    // SQLITE
-    // public static final String DB_CONNECTION = "jdbc:sqlite:/tmp/";
-    // public static final String DB_JDBC = "org.sqlite.JDBC";
-    // public static final DatabaseType DB_TYPE = DatabaseType.SQLITE;
+    /**
+     * This is the database type that we will use in our unit tests.
+     * This should always be one of the embedded java databases
+     */
+    public static final DatabaseType DB_TYPE = DatabaseType.HSQLDB;
+    public static final String DB_CONNECTION;
+    static {
+        switch (DB_TYPE) {
+            case HSQLDB: {
+                DB_CONNECTION = "jdbc:hsqldb:mem:";
+                break;
+            }
+            case H2: {
+                DB_CONNECTION = "jdbc:h2:mem:";
+                break;
+            }
+            case SQLITE: {
+                 DB_CONNECTION = "jdbc:sqlite::memory:";
+//                DB_CONNECTION = "jdbc:sqlite:/tmp/";
+                break;
+            }
+            default: {
+                LOG.warn("Unexpected testing DatabaseType '" + DB_TYPE + "'");
+                DB_CONNECTION = null;
+            }
+        } // SWITCH
+
+    }
 
     // -----------------------------------------------------------------
 

tests/com/oltpbenchmark/api/AbstractTestLoader.java

@@ -22,12 +22,16 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.log4j.Logger;
+
 import com.oltpbenchmark.catalog.Table;
 import com.oltpbenchmark.util.Histogram;
 import com.oltpbenchmark.util.SQLUtil;
 
 public abstract class AbstractTestLoader<T extends BenchmarkModule> extends AbstractTestCase<T> {
 
+    private static final Logger LOG = Logger.getLogger(AbstractTestLoader.class);
+
     protected Set<String> ignoreTables = new HashSet<String>();
 
     @SuppressWarnings("rawtypes")
@@ -58,21 +62,28 @@ public void testLoad() throws Exception {
         // All we really can do here is just invoke the loader 
         // and then check to make sure that our tables aren't empty
         this.benchmark.loadDatabase(this.conn);
-        Histogram<String> tableSizes = new Histogram<String>();
+        assertFalse("Failed to get table names for " + benchmark.getBenchmarkName().toUpperCase(),
+                    this.catalog.getTableNames().isEmpty());
+
+        LOG.debug("Computing the size of the tables");
+        Histogram<String> tableSizes = new Histogram<String>(true);
         for (String tableName : this.catalog.getTableNames()) {
             if (this.ignoreTables.contains(tableName.toUpperCase())) continue;
             Table catalog_tbl = this.catalog.getTable(tableName);
 
-            sql = SQLUtil.getCountSQL(catalog_tbl);
+            sql = SQLUtil.getCountSQL(this.workConf.getDBType(), catalog_tbl);
             result = stmt.executeQuery(sql);
             assertNotNull(result);
             boolean adv = result.next();
             assertTrue(sql, adv);
             int count = result.getInt(1);
             result.close();
+            System.out.println(sql + " => " + count);
             tableSizes.put(tableName, count);            
         } // FOR
-        System.err.println(tableSizes);
+        System.out.println("=== TABLE SIZES ===\n" + tableSizes);
+        assertFalse("Unable to compute the tables size for " + benchmark.getBenchmarkName().toUpperCase(),
+                    tableSizes.isEmpty());
 
         for (String tableName : tableSizes.values()) {
             long count = tableSizes.get(tableName);

tests/com/oltpbenchmark/benchmarks/seats/TestSEATSWorker.java

@@ -21,7 +21,7 @@
 public class TestSEATSWorker extends AbstractTestWorker<SEATSBenchmark> {
 
     static {
-        org.apache.log4j.PropertyConfigurator.configure("/home/pavlo/Documents/OLTPBenchmark/OLTPBenchmark/log4j.properties");
+        org.apache.log4j.PropertyConfigurator.configure("/home/pavlo/Documents/OLTPBenchmark/oltpbench/log4j.properties");
     }
 
     @Override

tests/com/oltpbenchmark/util/TestFileUtil.java

@@ -16,18 +16,17 @@
 
 package com.oltpbenchmark.util;
 
-import static org.junit.Assert.*;
-
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.sql.Time;
+
+import junit.framework.TestCase;
 
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-public class TestFileUtil {
+public class TestFileUtil extends TestCase {
 
     @Before
     public void setUp() throws Exception {