References to BLOB and CLOB objects now have a timeout. The configuration setting is LOB_TIMEOUT (default 5 minutes). This should avoid growing the database file if there are many queries that return BLOB or CLOB objects, and the database is not closed for a longer time.

git-svn-id: http://h2database.googlecode.com/svn/trunk@6116 e6896862-9d19-0410-bfb3-e9fa5d50e656

Author: thomas.tom.mueller@gmail.com

h2/src/main/org/h2/api/ErrorCode.java

@@ -804,6 +804,13 @@ public class ErrorCode {
      */
     public static final int VIEW_ALREADY_EXISTS_1 = 90038;
 
+    /**
+     * The error with code <code>90039</code> is thrown when
+     * trying to access a CLOB or BLOB object that timed out.
+     * See the database setting LOB_TIMEOUT.
+     */
+    public static final int LOB_CLOSED_ON_TIMEOUT_1 = 90039;
+
     /**
      * The error with code <code>90040</code> is thrown when
      * a user that is not administrator tries to execute a statement
@@ -1907,7 +1914,7 @@ public class ErrorCode {
     public static final int STEP_SIZE_MUST_NOT_BE_ZERO = 90142;
 
 
-    // next are 90039, 90051, 90056, 90110, 90122, 90143
+    // next are 90051, 90056, 90110, 90122, 90143
 
     private ErrorCode() {
         // utility class

h2/src/main/org/h2/engine/DbSettings.java

@@ -133,6 +133,15 @@ public class DbSettings extends SettingsBase {
      */
     public final boolean largeTransactions = get("LARGE_TRANSACTIONS", true);
 
+    /**
+     * Database setting <code>LOB_TIMEOUT</code> (default: 300000,
+     * which means 5 minutes).<br />
+     * The number of milliseconds a temporary LOB reference is kept until it
+     * times out. After the timeout, the LOB is no longer accessible using this
+     * reference.
+     */
+    public final int lobTimeout = get("LOB_TIMEOUT", 300000);
+
     /**
      * Database setting <code>MAX_COMPACT_COUNT</code>
      * (default: Integer.MAX_VALUE).<br />

h2/src/main/org/h2/engine/Session.java

@@ -9,6 +9,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.Random;
 
 import org.h2.api.ErrorCode;
@@ -108,6 +109,20 @@ public class Session extends SessionWithState {
     private final int queryCacheSize;
     private SmallLRUCache<String, Command> queryCache;
     private long modificationMetaID = -1;
+
+    /**
+     * Temporary LOBs from result sets. Those are kept for some time. The
+     * problem is that transactions are committed before the result is returned,
+     * and in some cases the next transaction is already started before the
+     * result is read (for example when using the server mode, when accessing
+     * metadata methods). We can't simply free those values up when starting the
+     * next transaction, because they would be removed too early.
+     */
+    private LinkedList<TimeoutValue> temporaryResultLobs;
+
+    /**
+     * The temporary LOBs that need to be removed on commit.
+     */
     private ArrayList<Value> temporaryLobs;
 
     private Transaction transaction;
@@ -497,14 +512,7 @@ public void commit(boolean ddl) {
             // (create/drop table and so on)
             database.commit(this);
         }
-        if (temporaryLobs != null) {
-            for (Value v : temporaryLobs) {
-                if (!v.isLinked()) {
-                    v.close();
-                }
-            }
-            temporaryLobs.clear();
-        }
+        removeTemporaryLobs(true);
         if (undoLog.size() > 0) {
             // commit the rows when using MVCC
             if (database.isMultiVersion()) {
@@ -536,6 +544,31 @@ public void commit(boolean ddl) {
         endTransaction();
     }
 
+    private void removeTemporaryLobs(boolean onTimeout) {
+        if (temporaryLobs != null) {
+            for (Value v : temporaryLobs) {
+                if (!v.isLinked()) {
+                    v.close();
+                }
+            }
+            temporaryLobs.clear();
+        }
+        if (temporaryResultLobs != null && temporaryResultLobs.size() > 0) {
+            long keepYoungerThan = System.currentTimeMillis() -
+                    database.getSettings().lobTimeout;
+            while (temporaryResultLobs.size() > 0) {
+                TimeoutValue tv = temporaryResultLobs.getFirst();
+                if (onTimeout && tv.created >= keepYoungerThan) {
+                    break;
+                }
+                Value v = temporaryResultLobs.removeFirst().value;
+                if (!v.isLinked()) {
+                    v.close();
+                }
+            }
+        }
+    }
+
     private void checkCommitRollback() {
         if (commitOrRollbackDisabled && locks.size() > 0) {
             throw DbException.get(ErrorCode.COMMIT_ROLLBACK_NOT_ALLOWED);
@@ -545,8 +578,8 @@ private void checkCommitRollback() {
     private void endTransaction() {
         if (unlinkLobMap != null && unlinkLobMap.size() > 0) {
             if (database.getMvStore() == null) {
-                // need to flush the transaction log, because we can't unlink lobs
-                // if the commit record is not written
+                // need to flush the transaction log, because we can't unlink
+                // lobs if the commit record is not written
                 database.flush();
             }
             for (Value v : unlinkLobMap.values()) {
@@ -673,6 +706,7 @@ public void close() {
         if (!closed) {
             try {
                 database.checkPowerOff();
+                removeTemporaryLobs(false);
                 cleanTempTables(true);
                 undoLog.clear();
                 database.removeSession(this);
@@ -1447,10 +1481,17 @@ public void endStatement() {
 
     @Override
     public void addTemporaryLob(Value v) {
-        if (temporaryLobs == null) {
-            temporaryLobs = new ArrayList<Value>();
+        if (v.getTableId() == LobStorageFrontend.TABLE_RESULT) {
+            if (temporaryResultLobs == null) {
+                temporaryResultLobs = new LinkedList<TimeoutValue>();
+            }
+            temporaryResultLobs.add(new TimeoutValue(v));
+        } else {
+            if (temporaryLobs == null) {
+                temporaryLobs = new ArrayList<Value>();
+            }
+            temporaryLobs.add(v);
         }
-        temporaryLobs.add(v);
     }
 
     /**
@@ -1470,4 +1511,25 @@ public static class Savepoint {
         long transactionSavepoint;
     }
 
+    /**
+     * An object with a timeout.
+     */
+    public static class TimeoutValue {
+
+        /**
+         * The time when this object was created.
+         */
+        final long created = System.currentTimeMillis();
+
+        /**
+         * The value.
+         */
+        final Value value;
+
+        TimeoutValue(Value v) {
+            this.value = v;
+        }
+
+    }
+
 }

h2/src/main/org/h2/result/LocalResult.java

@@ -42,6 +42,7 @@ public class LocalResult implements ResultInterface, ResultTarget {
     private boolean distinct;
     private boolean randomAccess;
     private boolean closed;
+    private boolean containsLobs;
 
     /**
      * Construct a local result object.
@@ -114,12 +115,15 @@ public static LocalResult read(Session session, ResultSet rs, int maxrows) {
      * (if there is any) is not copied.
      *
      * @param targetSession the session of the copy
-     * @return the copy
+     * @return the copy if possible, or null if copying is not possible
      */
     public LocalResult createShallowCopy(Session targetSession) {
         if (external == null && (rows == null || rows.size() < rowCount)) {
             return null;
         }
+        if (containsLobs) {
+            return null;
+        }
         ResultExternal e2 = null;
         if (external != null) {
             e2 = external.createShallowCopy();
@@ -260,6 +264,7 @@ private void cloneLobs(Value[] values) {
             Value v = values[i];
             Value v2 = v.copyToResult();
             if (v2 != v) {
+                containsLobs = true;
                 session.addTemporaryLob(v2);
                 values[i] = v2;
             }

h2/src/main/org/h2/store/LobStorageMap.java

@@ -268,7 +268,14 @@ public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount)
         init();
         Object[] value = lobMap.get(lob.getLobId());
         if (value == null) {
-            throw DbException.throwInternalError("Lob not found: " + lob.getLobId());
+            if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT ||
+                    lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
+                throw DbException.get(
+                        ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" +
+                                lob.getLobId() + "/" + lob.getTableId());
+            }
+            throw DbException.throwInternalError("Lob not found: " +
+                    lob.getLobId() + "/" + lob.getTableId());
         }
         byte[] streamStoreId = (byte[]) value[0];
         return streamStore.get(streamStoreId);
@@ -348,7 +355,7 @@ private void removeLob(int tableId, long lobId) {
     }
 
     private static void trace(String op) {
-        System.out.println(Thread.currentThread().getName() + " LOB " + op);
+        System.out.println("[" + Thread.currentThread().getName() + "] LOB " + op);
     }
 
 }

h2/src/main/org/h2/value/ValueLobDb.java

@@ -204,6 +204,7 @@ public Value convertTo(int t) {
     @Override
     public boolean isLinked() {
         return tableId != LobStorageFrontend.TABLE_ID_SESSION_VARIABLE &&
+                tableId != LobStorageFrontend.TABLE_RESULT &&
                 small == null;
     }
 

h2/src/test/org/h2/test/db/TestLob.java

@@ -59,6 +59,7 @@ public static void main(String... a) throws Exception {
 
     @Override
     public void test() throws Exception {
+        testRemovedAfterTimeout();
         testConcurrentRemoveRead();
         testCloseLobTwice();
         testCleaningUpLobsOnRollback();
@@ -112,6 +113,42 @@ public void test() throws Exception {
         FileUtils.deleteRecursive(TEMP_DIR, true);
     }
 
+    private void testRemovedAfterTimeout() throws Exception {
+        deleteDb("lob");
+        final String url = getURL("lob;lob_timeout=50", true);
+        Connection conn = getConnection(url);
+        Statement stat = conn.createStatement();
+        stat.execute("create table test(id int primary key, data clob)");
+        PreparedStatement prep = conn.prepareStatement("insert into test values(?, ?)");
+        prep.setInt(1, 1);
+        prep.setString(2, "aaa" + new String(new char[1024 * 16]).replace((char) 0, 'x'));
+        prep.execute();
+        prep.setInt(1, 2);
+        prep.setString(2, "bbb" + new String(new char[1024 * 16]).replace((char) 0, 'x'));
+        prep.execute();
+        ResultSet rs = stat.executeQuery("select * from test order by id");
+        rs.next();
+        Clob c1 = rs.getClob(2);
+        assertEquals("aaa", c1.getSubString(1, 3));
+        rs.next();
+        assertEquals("aaa", c1.getSubString(1, 3));
+        rs.close();
+        assertEquals("aaa", c1.getSubString(1, 3));
+        stat.execute("delete from test");
+        c1.getSubString(1, 3);
+        // wait until it times out
+        Thread.sleep(100);
+        // start a new transaction, to be sure
+        stat.execute("delete from test");
+        try {
+            c1.getSubString(1, 3);
+            fail();
+        } catch (SQLException e) {
+            // expected
+        }
+        conn.close();
+    }
+
     private void testConcurrentRemoveRead() throws Exception {
         deleteDb("lob");
         final String url = getURL("lob", true);
@@ -1255,7 +1292,6 @@ private void testUpdateLob() throws SQLException {
                         "CREATE TABLE IF NOT EXISTS p( id int primary key, rawbyte BLOB ); ");
         prep.execute();
         prep.close();
-
         prep = conn.prepareStatement("INSERT INTO p(id) VALUES(?);");
         for (int i = 0; i < 10; i++) {
             prep.setInt(1, i);