Add --min-objects option

To ensure that the underlying database has at least a set number of
objects in place. This lets us test scaling issues and be more
repeatable.

.travis.yml

@@ -27,8 +27,8 @@ script:
 # to ~30s for CPython
   - python -m zodbshootout --help
   - coverage run setup.py test
-  - PYTHONPATH=".travis" coverage run .travis/cover.py -c 1 -n 100 .travis/$ENV.conf --btrees -r 2 --test-reps 1
-  - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then PYTHONPATH=".travis" coverage run .travis/cover.py -c 3 -n 100 .travis/$ENV.conf --threads --gevent -r 2 --test-reps 2 --leaks --dump-json; fi
+  - PYTHONPATH=".travis" coverage run .travis/cover.py -c 1 -n 100 .travis/$ENV.conf --btrees -r 2 --test-reps 1 --min-objects 100
+  - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then PYTHONPATH=".travis" coverage run .travis/cover.py -c 3 -n 100 .travis/$ENV.conf --min-objects 200 --threads --gevent -r 2 --test-reps 2 --leaks --dump-json; fi
   - if [[ $TRAVIS_PYTHON_VERSION == pypy* ]]; then PYTHONPATH=".travis" python .travis/cover.py -c 3 -n 100 .travis/$ENV.conf --threads -r 2 --test-reps 2 --leaks --dump-json; fi
 after_success:
   - coverage combine

CHANGES.rst

@@ -2,7 +2,7 @@
  Changes
 =========
 
-0.6.1 (unreleased)
+0.7.0 (unreleased)
 ==================
 
 - Multi-threaded runs handle exceptions and signals more reliably.
@@ -13,7 +13,12 @@
   non-deterministic. See :issue:`28`.
 - When using gevent, use its Event and Queue implementations for
   better cooperation with the event loop.
-
+- Add ``--min-objects`` option to ensure that the underlying database
+  has at least a set number of objects in place. This lets us test
+  scaling issues and be more repeatable. This is tested with
+  FileStorage, ZEO, and RelStorage (RelStorage 2.1a2 or later is
+  needed for accurate results; earlier versions will add new objects
+  each time, resulting in database growth).
 
 
 0.6.0 (2016-12-13)

doc/zodbshootout.rst

@@ -66,7 +66,7 @@ configuration file as the table column names.
 
 An example of a configuration file testing the built-in ZODB file
 storage, a few variations of ZEO, and `RelStorage <http://relstorage.readthedocs.io/en/latest/configure-application.html#configuring-repoze-zodbconn>`_
-FileStorage would look like this:
+would look like this:
 
 .. literalinclude:: ../samples/fs-sample.conf
    :language: nginx
@@ -121,6 +121,15 @@ These options control the objects put in the database.
 
   .. versionadded:: 0.6
 
+* ``--min-objects`` ensures that at least the specified number of
+  objects exist in the database independently of the objects being
+  tested. If the database packs away objects or if ``--zap`` is used,
+  this option will add back the necessary number of objects. If there
+  are more objects, nothing will be done. This option is helpful for
+  testing for scalability issues.
+
+  .. versionadded:: 0.7
+
 Concurrency
 -----------
 

src/zodbshootout/_runner.py

@@ -306,6 +306,7 @@ def run_with_options(options):
                     options.profile_dir,
                     mp_strategy=(options.threads or 'mp'),
                     test_reps=options.test_reps)
+                speedtest.min_object_count = options.min_object_count
                 if options.btrees:
                     import BTrees
                     if options.btrees == 'IO':
@@ -316,9 +317,9 @@ def run_with_options(options):
                 for contender_name, db in contenders:
                     print((
                         'Testing %s with objects_per_txn=%d, object_size=%d, '
-                        'mappingtype=%s and concurrency=%d (threads? %s)'
+                        'mappingtype=%s, min_objects=%d and concurrency=%d (threads? %s)'
                         % (contender_name, objects_per_txn, object_size,
-                           speedtest.MappingType,
+                           speedtest.MappingType, options.min_object_count,
                            concurrency, options.threads)), file=sys.stderr)
 
                     all_times = _run_one_contender(options, speedtest, contender_name, db)

src/zodbshootout/main.py

@@ -55,6 +55,10 @@ def main(argv=None):
         "--zap", action='store_true', default=False,
         help="Zap the entire RelStorage before running tests. This will destroy all data. "
     )
+    obj_group.add_argument(
+        "--min-objects", dest="min_object_count",
+        type=int, default=0, action="store",
+        help="Ensure the database has at least this many objects before running tests.")
 
     # Repetitions
     rep_group.add_argument(

src/zodbshootout/speedtest.py

@@ -20,24 +20,26 @@
 from functools import partial
 from itertools import chain
 from pstats import Stats
-from threading import Event
 
 import cProfile
 
 import os
 import random
 import statistics
-import sys
+
 import time
 import transaction
 
 from persistent.mapping import PersistentMapping
+from persistent.list import PersistentList
 
 from .fork import distribute
 from .fork import run_in_child
 from ._pobject import pobject_base_size
 from ._pobject import PObject
 
+logger = __import__('logging').getLogger(__name__)
+
 def itervalues(d):
     try:
         iv = d.itervalues
@@ -84,9 +86,10 @@ def fdata():
 class SpeedTest(object):
 
     MappingType = PersistentMapping
-    debug = False
+
 
     individual_test_reps = 20
+    min_object_count = 0
 
     def __init__(self, concurrency, objects_per_txn, object_size,
                  profile_dir=None,
@@ -137,18 +140,45 @@ def populate(self, db_factory):
 
         # clear the database
         root['speedtest'] = None
+        # We explicitly leave the `speedtest_min` value around
+        # so that it can survive packs.
         transaction.commit()
         db.pack()
 
+        # Make sure the minimum objects are present
+        if self.min_object_count:
+            # not all storages support __len__ to return the size of the database.
+            # FileStorage, RelStorage and ClientStorage do.
+            db_count = max(len(db.storage), len(conn._storage))
+            needed = max(self.min_object_count - db_count, 0)
+            if needed:
+                logger.debug("Adding %d objects to a DB of size %d",
+                             needed, db_count)
+                # We append to a list the new objects. This makes sure that we
+                # don't *erase* some objects we were counting on.
+                l = root.get('speedtest_min')
+                if l is None:
+                    l = root['speedtest_min'] = PersistentList()
+
+                # If `needed` is large, this could result in a single
+                # very large transaction. Do we need to think about splitting it up?
+                m = PersistentMapping()
+                m.update(dict((n, PObject('Minimum object size')) for n in range(needed)))
+                l.append(m)
+                transaction.commit()
+                logger.debug("Added %d objects to a DB of size %d",
+                             len(m), db_count)
+            else:
+                logger.debug("Database is already of size %s", db_count)
+
         # put a tree in the database
         root['speedtest'] = t = self.MappingType()
         for i in range(self.concurrency):
             t[i] = self.MappingType()
         transaction.commit()
         conn.close()
         db.close()
-        if self.debug:
-            print('Populated storage.', file=sys.stderr)
+        logger.debug('Populated storage.')
 
 
     def _clear_all_caches(self, db):
@@ -172,10 +202,10 @@ def _times_of_runs(self, func, times, args=()):
         return run_times
 
     def _close_conn(self, conn):
-        if self.debug:
-            loads, stores = conn.getTransferCounts(True)
-            db_name = conn.db().database_name
-            print("DB", db_name, "conn", conn, "loads", loads, "stores", stores, file=sys.stderr)
+        loads, stores = conn.getTransferCounts(True)
+        db_name = conn.db().database_name
+        logger.debug("DB %s conn %s loads %s stores %s",
+                     db_name, conn, loads, stores)
         conn.close()
 
     # We should always include conn.open() inside our times,