WT-1930 Add configuration option to disable sweep of old handles

Merge pull request #2044 from wiredtiger/WT-1930
(cherry picked from commit 263c5b7)

dist/api_data.py

@@ -374,7 +374,8 @@ def __cmp__(self, other):
         type='category', subconfig=[
         Config('close_idle_time', '30', r'''
             amount of time in seconds a file handle needs to be idle
-            before attempting to close it''', min=1, max=100000),
+            before attempting to close it. A setting of 0 means that idle
+            handles are not closed''', min=0, max=100000),
         Config('close_handle_minimum', '250', r'''
             number of handles open before the file manager will look for handles
             to close'''),

src/config/config_def.c

@@ -61,7 +61,7 @@ static const WT_CONFIG_CHECK confchk_eviction_subconfigs[] = {
 
 static const WT_CONFIG_CHECK confchk_file_manager_subconfigs[] = {
 	{ "close_handle_minimum", "string", NULL, NULL, NULL },
-	{ "close_idle_time", "int", NULL, "min=1,max=100000", NULL },
+	{ "close_idle_time", "int", NULL, "min=0,max=100000", NULL },
 	{ "close_scan_interval", "int",
 	    NULL, "min=1,max=100000",
 	    NULL },

src/conn/conn_sweep.c

@@ -35,7 +35,8 @@ __sweep_mark(WT_SESSION_IMPL *session, int *dead_handlesp)
 			continue;
 		}
 		if (dhandle->session_inuse != 0 ||
-		    now <= dhandle->timeofdeath + conn->sweep_idle_time)
+		    now <= dhandle->timeofdeath + conn->sweep_idle_time ||
+		    conn->sweep_idle_time == 0)
 			continue;
 		if (dhandle->timeofdeath == 0) {
 			dhandle->timeofdeath = now;
@@ -121,6 +122,10 @@ __sweep_expire(WT_SESSION_IMPL *session)
 
 	conn = S2C(session);
 
+	/* If sweep_idle_time is 0, then we won't expire any cursors */
+	if (conn->sweep_idle_time == 0)
+		return (0);
+
 	/* Don't discard handles that have been open recently. */
 	WT_RET(__wt_seconds(session, &now));
 
@@ -265,8 +270,14 @@ __sweep_server(void *arg)
 		 */
 		WT_ERR(__sweep_mark(session, &dead_handles));
 
+		/*
+		 * We only want to flush and expire if there are no dead handles
+		 * and if either the sweep_idle_time is not 0, or if we have
+		 * reached the configured limit of handles.
+		 */
 		if (dead_handles == 0 &&
-		    conn->open_file_count < conn->sweep_handles_min)
+		    (conn->open_file_count < conn->sweep_handles_min ||
+		    conn->sweep_idle_time != 0))
 			continue;
 
 		/* Close handles if we have reached the configured limit */

src/include/wiredtiger.in

@@ -1619,8 +1619,9 @@ struct __wt_connection {
 	 * handles open before the file manager will look for handles to close.,
 	 * a string; default \c 250.}
 	 * @config{    close_idle_time, amount of time in
-	 * seconds a file handle needs to be idle before attempting to close
-	 * it., an integer between 1 and 100000; default \c 30.}
+	 * seconds a file handle needs to be idle before attempting to close it.
+	 * A setting of 0 means that idle handles are not closed., an integer
+	 * between 0 and 100000; default \c 30.}
 	 * @config{    close_scan_interval, interval in
 	 * seconds at which to check for files that are inactive and close
 	 * them., an integer between 1 and 100000; default \c 10.}
@@ -1993,8 +1994,9 @@ struct __wt_connection {
  * before the file manager will look for handles to close., a string; default \c
  * 250.}
  * @config{    close_idle_time, amount of time in
- * seconds a file handle needs to be idle before attempting to close it., an
- * integer between 1 and 100000; default \c 30.}
+ * seconds a file handle needs to be idle before attempting to close it.  A
+ * setting of 0 means that idle handles are not closed., an integer between 0
+ * and 100000; default \c 30.}
  * @config{    close_scan_interval, interval in seconds at
  * which to check for files that are inactive and close them., an integer
  * between 1 and 100000; default \c 10.}

test/suite/test_sweep03.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2015 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# test_sweep03.py
+# Test to confirm if setting close_idle_time to 0 does not sweep old handles
+#
+
+import fnmatch, os, shutil, run, time
+from suite_subprocess import suite_subprocess
+from wiredtiger import wiredtiger_open, stat
+from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios
+import wttest
+
+class test_sweep03(wttest.WiredTigerTestCase, suite_subprocess):
+    tablebase = 'test_sweep03'
+    uri = 'table:' + tablebase
+    numfiles = 400 # Make this more than the default close_handle_minimum
+    numkv = 100
+
+    types = [
+        ('row', dict(tabletype='row',
+                    create_params = 'key_format=i,value_format=i')),
+        ('var', dict(tabletype='var',
+                    create_params = 'key_format=r,value_format=i')),
+        ('fix', dict(tabletype='fix',
+                    create_params = 'key_format=r,value_format=8t')),
+    ]
+
+    scenarios = types
+
+    # Overrides WiredTigerTestCase
+    def setUpConnectionOpen(self, dir):
+        self.home = dir
+
+        conn_params = \
+                ',create,error_prefix="%s: ",' % self.shortid() + \
+                'file_manager=(close_handle_minimum=10,' + \
+                'close_idle_time=0,close_scan_interval=1),' + \
+                'statistics=(fast),'
+        # print "Creating conn at '%s' with config '%s'" % (dir, conn_params)
+        try:
+            conn = wiredtiger_open(dir, conn_params)
+        except wiredtiger.WiredTigerError as e:
+            print "Failed conn at '%s' with config '%s'" % (dir, conn_params)
+        self.pr(`conn`)
+        self.session2 = conn.open_session()
+        return conn
+
+    def test_disable_idle_timeout1(self):
+        #
+        # Set up numfiles with numkv entries.  We just want some data in there
+        # we don't care what it is.
+        #
+        for f in range(self.numfiles):
+            uri = '%s.%d' % (self.uri, f)
+            # print "Creating %s with config '%s'" % (uri, self.create_params)
+            self.session.create(uri, self.create_params)
+            c = self.session.open_cursor(uri, None)
+            for k in range(self.numkv):
+                c[k+1] = 1
+            c.close()
+
+        #
+        # The idle timeout is disabled - we don't expect the sweep server to
+        # close any regular handles. Give the sweep server 5x the scan
+        # interval to close any handles.
+        #
+        time.sleep(5)
+
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        close1 = stat_cursor[stat.conn.dh_conn_handles][2]
+        sweep1 = stat_cursor[stat.conn.dh_conn_sweeps][2]
+        stat_cursor.close()
+
+        # The sweep server should have run, or the test isn't working.
+        self.assertGreater(sweep1, 0)
+        # We expect nothing to have been closed, so dh_conn_handles should be 0
+        self.assertEqual(close1, 0)
+
+    def test_disable_idle_timeout_drop_force(self):
+        # Create a table to drop. A drop should close its associated handle
+        drop_uri = '%s.%s' % (self.uri, "force_drop_test")
+
+        self.session.create(drop_uri, self.create_params)
+
+        c = self.session.open_cursor(drop_uri, None)
+        for k in range(self.numkv):
+            c[k+1] = 1
+        c.close()
+
+        # We just filled the table, now check what the stats are
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        cache1 = stat_cursor[stat.conn.cache_bytes_inuse][2]
+        sweep1 = stat_cursor[stat.conn.dh_conn_sweeps][2]
+        stat_cursor.close()
+
+        # We force the drop in this case to confirm that the handle is closed
+        self.session.drop(drop_uri, "force=true")
+
+        time.sleep(5)
+
+        # Grab the stats post table drop to see things have decremented
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        cache2 = stat_cursor[stat.conn.cache_bytes_inuse][2]
+        close2 = stat_cursor[stat.conn.dh_conn_handles][2]
+        sweep2 = stat_cursor[stat.conn.dh_conn_sweeps][2]
+        stat_cursor.close()
+
+        # Make sure the sweep server is still working.
+        self.assertGreater(sweep2, sweep1)
+        # Ensure that the handle has been closed after the drop.
+        self.assertEqual(close2, 1)
+        # Ensure that any space was reclaimed from cache.
+        self.assertLess(cache2, cache1)
+
+    def test_disable_idle_timeout_drop(self):
+        # Create a table to drop. A drop should close its associated handles
+        drop_uri = '%s.%s' % (self.uri, "drop_test")
+        self.session.create(drop_uri, self.create_params)
+
+        c = self.session.open_cursor(drop_uri, None)
+        for k in range(self.numkv):
+            c[k+1] = 1
+        c.close()
+
+        # We just filled the table, now check what the stats are
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        cache1 = stat_cursor[stat.conn.cache_bytes_inuse][2]
+        close1 = stat_cursor[stat.conn.dh_conn_handles][2]
+        sweep1 = stat_cursor[stat.conn.dh_conn_sweeps][2]
+        stat_cursor.close()
+
+        self.session.drop(drop_uri, None)
+
+        time.sleep(5)
+
+        # Grab the stats post table drop to see things have decremented
+        stat_cursor = self.session.open_cursor('statistics:', None, None)
+        cache2 = stat_cursor[stat.conn.cache_bytes_inuse][2]
+        close2 = stat_cursor[stat.conn.dh_conn_handles][2]
+        sweep2 = stat_cursor[stat.conn.dh_conn_sweeps][2]
+        stat_cursor.close()
+
+        self.assertGreater(sweep2, sweep1)
+        # The sweep server should not be involved in regular drop cleanup
+        self.assertEqual(close2, close1)
+        # Ensure that any space was reclaimed from cache.
+        self.assertLess(cache2, cache1)
+
+if __name__ == '__main__':
+    wttest.run()