From 5eabd272b2b144847355b9216b1b37dce0e9de4b Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Fri, 15 Nov 2019 10:34:49 -0600 Subject: [PATCH 1/2] Add basic documentation. Fixes #13. Also make some options more configurable, and fix storage ordering in ZEO. --- CHANGES.rst | 4 + README.rst | 647 +++++++++++++++++++++- setup.py | 4 + src/nti/recipes/zodb/__init__.py | 38 +- src/nti/recipes/zodb/relstorage.py | 48 +- src/nti/recipes/zodb/tests/test_readme.py | 55 ++ src/nti/recipes/zodb/tests/test_zeo.py | 4 +- src/nti/recipes/zodb/zeo.py | 11 +- 8 files changed, 746 insertions(+), 65 deletions(-) create mode 100644 src/nti/recipes/zodb/tests/test_readme.py diff --git a/CHANGES.rst b/CHANGES.rst index 8621e78..88f5c10 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -33,6 +33,10 @@ specified. Previously it was complex to specify a DSN using configurable values. +- RelStorage: Make ``shared_blob_dir``, ``cache_local_mb`` and + ``pack_gc`` be configurable at the storage level instead of just the + recipe part level. + 1.0.0a1 (2019-11-14) ==================== diff --git a/README.rst b/README.rst index 3da687c..8bf725f 100644 --- a/README.rst +++ b/README.rst @@ -2,10 +2,655 @@ nti.recipes.zodb ================== -Recipes for creating RelStorage and ZEO configurations. +Opinionated recipes for creating RelStorage and ZEO configurations, especially +tailored for multi-databases. .. image:: https://travis-ci.org/NextThought/nti.recipes.zodb.svg?branch=master :target: https://travis-ci.org/NextThought/nti.recipes.zodb .. image:: https://coveralls.io/repos/github/NextThought/nti.recipes.zodb/badge.svg?branch=master :target: https://coveralls.io/github/NextThought/nti.recipes.zodb?branch=master + +Limitations +=========== + +A single buildout can use *either* one RelStorage recipe *or* one ZEO +recipe. It can never have both, or more than one of each. This is +because both recipes write to the same configuration files + +Dependencies +============ + +The recipes defined here use `collective.recipe.template`_ to output +configuration files, and `z3c.recipe.mkdir`_ to create implicitly +defined directories. `zc.zodbrecipes`_ is used to create the ZEO +server. You shouldn't need to install these manually as buildout will +take care of making them available when needed. + +.. _collective.recipe.template: https://pypi.org/project/collective.recipe.template/ +.. _z3c.recipe.mkdir: https://pypi.org/project/z3c.recipe.mkdir/ +.. _zc.zodbrecipes: https://pypi.org/project/zc.zodbrecipes/ + +Directories and Files +===================== + +The recipes defined here use the directory structure and variables +defined by `zc.recipe.deployment`_. There should be a buildout part +called ``deployment`` that uses this recipe. Alternatively (and +especially useful when composing a buildout from multiple +configurations), you can define a ``deployment`` part that lists these +directories manually. You'll also need to include the username that +should own created files and directories: + +.. code:: ini + + [deployment] + bin-directory = ${buildout:bin-directory} + cache-directory = ${:run-directory}/caches + crontab-directory = ${:root-directory}/etc/cron.d + data-directory = ${:root-directory}/data + etc-directory = ${:root-directory}/etc + log-directory = ${:root-directory}/var/log + logrotate-directory = ${:root-directory}/etc/logrotate.d + rc-directory = ${:root-directory}/bin/rc + root-directory = ${buildout:root-directory} + run-directory = ${:root-directory}/var + +The deployment recipe takes care of creating the needed directories, +but here we'll just do so manually. We'll define a common +configuration snippet that we'll include in future examples:: + + >>> write(sample_buildout, 'deployment.cfg', + ... """ + ... [deployment] + ... root-directory = ${buildout:directory} + ... data-directory = ${:root-directory}/data + ... etc-directory = ${:root-directory}/etc + ... log-directory = ${:root-directory}/var/log + ... run-directory = ${:root-directory}/var + ... rc-directory = ${:root-directory}/bin/rc + ... cache-directory = ${:run-directory}/caches + ... logrotate-directory = ${:root-directory}/etc/logrotate.d + ... crontab-directory = ${:root-directory}/etc/cron.d + ... user = user + ... + ... [directories] + ... recipe = z3c.recipe.mkdir + ... create-intermediate = true + ... mode = 0700 + ... paths = + ... ${deployment:etc-directory} + ... ${deployment:run-directory} + ... ${deployment:cache-directory} + ... ${deployment:data-directory} + ... ${deployment:log-directory} + ... ${deployment:rc-directory} + ... ${deployment:logrotate-directory} + ... ${deployment:crontab-directory} + ... """) + +Both recipes create two files in the ``etc-directory``. + +zodb_conf.xml + This file is meant to be read with + ``ZODB.config.databaseFromFile`` or ``databaseFromURL``. If you + specify more than one storage, they will be listed in the order + provided, creating a multi-database, with the first listed storage + as the "root" database. + +zeo_uris.ini + This file provides the same database configuration as + ``zodb_conf.xml`` (indeed, it references that file), but in a form + of a single URL string that can be read using zodburi_. This can + be convient for passing in the form of a string. + +.. _zc.recipe.deployment: https://pypi.org/project/zc.recipe.deployment/_ +.. _zodburi: https://pypi.org/project/zodburi/ + + +Recipe Options +============== + +Both recipes defined here accept some common options. + +storages + Required. A whitespace delimited list of storage names. Each of these will be + added to the generated configuration files for a client to use (and + for ZEO, for the server to serve). + + This can only be defined directly in the recipe part. +compress + If "decompress" (the default) each storage will be wrapped in a + `zc.zlibstorage`_ that only compress existing records. If set to + "compress" new records will also be compressed. + + Set to "none" to disable the wrapper entirely. + + This can be set in the recipe part. If it's not defined there, a + value defined in the ``environment`` part will be used before + falling back to the default. + +.. _zc.zlibstorage: https://pypi.org/project/zc.zlibstorage/ + +Storage and Database Options +============================ + +Some options are available to configure the ZODB database. These are +used by both recipes and may be defined at a per-database level (see +each recipe for an explanation of how). The defaults are built-in, but +setting a value in the recipe part will provide a new default for all +storages. Additionally, for backwards compatibility and composing +buildout configurations, if there is a part named ``_opts_``, +where ```` is the name of the recipe part, options defined there +will override options defined in the recipe part, but will bee +overridden by options defined for an individual storage. + +These two configurations both set the same ZODB cache size. + +.. code:: ini + + [zeo] + recipe = nti.recipes.zodb:zeo + storages = users + cache_size = 50 + +.. code:: ini + + [zeo] + recipe = nti.recipes.zodb:zeo + storages = users + + [zeo_opts] + cache_size = 50 + + +cache_size + Controls the ZODB per-connection object cache. Setting this to a large-enough + value to contain your application's working set can be very important, especially + in read-heavy workloads. Setting it too large can waste memory. +pool_size + Controls the number of ZODB connections kept in the ZODB pool. It + is very important to set this large enough to accomodate the + number of concurrent activities (requests) your application will + handle. Each connection in the pool holds resources like its cache + and in the case of RelStorage RDBMS sockets and possibly memcache + sockets. Setting it too large can waste memory and file-descriptors. + + Normally, opening a DB and closing the connection will create a + connection (if needed), then return it to the pool (if the pool is + not full). However, in the case of multi-databases, when an object + from a secondary database needs to be loaded, the active + connection will request a connection to that database, and when + the active connection is closed, that secondary connection is also + closed *but not returned to the pool*. Instead, the active + (primary) connection keeps a reference to it that it will use in + the future. This has the effect of driving all secondary pools + based on the efficiency of the primary pool. Thus, the pool-size + for everything except the primary database is essentially + meaningless (if the application always begins by opening that + primary database), but that pool size controls everything. + + Calling DB.connectionDebugInfo() can show improperly sized pools: + connections in the pool have 'opened' of None, while those in use + have a timestamp and the length of time it's been open. + +RelStorage +========== + +The ``relstorage`` recipe creates configurations to connect to a +MySQL, PostgreSQL, or SQLite3 RelStorage database *in history-free +mode*. (Oracle is not supported.) + +There are a number of different ways to configure storages. If you +have multiple storages residing on a common database server, and you +also have other databases on that server (SQLAlchemy, etc), you might +be interested in using a shared ``environment`` section to contain +the server location and account credentials. + +.. note:: Do not store plain passwords in buildout configuration + files. Use something like `nti.recipes.passwords + `_ to store + them encrypted instead. + +By default, the name of the database is the same as the storage name. + + >>> write(sample_buildout, 'buildout.cfg', + ... """ + ... [buildout] + ... extends = deployment.cfg + ... parts = directories relstorage + ... + ... [environment] + ... sql_user = the_user + ... sql_host = the_server + ... sql_passwd = the_passwd + ... + ... [relstorage] + ... recipe = nti.recipes.zodb:relstorage + ... storages = users sessions + ... compress = none + ... """) + + >>> print_(system(buildout), end='') + Installing... + Installing relstorage. + >>> ls(sample_buildout, 'etc') + d cron.d + d logrotate.d + - zeo_uris.ini + - zodb_conf.xml + >>> cat(sample_buildout, 'etc', 'zodb_conf.xml') + %import relstorage + + cache-size 100000 + database-name users + pool-size 60 + + + # This comment preserves whitespace + db users + host the_server + passwd the_passwd + user the_user + + blob-dir /sample-buildout/data/users.blobs + cache-local-dir /sample-buildout/var/caches/data_cache/users.cache + cache-local-mb 300 + cache-prefix users + commit-lock-timeout 60 + keep-history false + name users + pack-gc false + shared-blob-dir false + + + + cache-size 100000 + database-name sessions + pool-size 60 + + + # This comment preserves whitespace + db sessions + host the_server + passwd the_passwd + user the_user + + blob-dir /sample-buildout/data/sessions.blobs + cache-local-dir /sample-buildout/var/caches/data_cache/sessions.cache + cache-local-mb 300 + cache-prefix sessions + commit-lock-timeout 60 + keep-history false + name sessions + pack-gc false + shared-blob-dir false + + + +Much can be configured at both the recipe level and in a section named +for the storage (prefixed with the name of the recipe, unlike in the +zeo recipe, and suffixed with '_storage_opts'), but a few settings can +only be configured on the recipe or environment part. + +enable-persistent-cache + Defaults to true. +cache-servers + Deprecated. A list of memcache servers to use. Can be configured at the recipe + or environment level. +blob-cache-size + Defaults to no size cap. + + >>> write(sample_buildout, 'buildout.cfg', + ... """ + ... [buildout] + ... extends = deployment.cfg + ... parts = directories relstorage + ... + ... [environment] + ... sql_user = the_user + ... sql_host = the_server + ... sql_passwd = the_passwd + ... + ... [relstorage] + ... recipe = nti.recipes.zodb:relstorage + ... storages = users sessions + ... compress = none + ... pack_gc = true + ... commit_lock_timeout = 10 + ... + ... [relstorage_users_storage_opts] + ... cache_size = 42 + ... cache_local_mb = 2 + ... sql_user = custom_user + ... """) + + >>> print_(system(buildout), end='') + Uninstalling relstorage... + Installing relstorage. + >>> ls(sample_buildout, 'etc') + d cron.d + d logrotate.d + - zeo_uris.ini + - zodb_conf.xml + >>> cat(sample_buildout, 'etc', 'zodb_conf.xml') + %import relstorage + + cache-size 42 + database-name users + pool-size 60 + + + # This comment preserves whitespace + db users + host the_server + passwd the_passwd + user custom_user + + blob-dir /sample-buildout/data/users.blobs + cache-local-dir /sample-buildout/var/caches/data_cache/users.cache + cache-local-mb 2 + cache-prefix users + commit-lock-timeout 10 + keep-history false + name users + pack-gc true + shared-blob-dir false + + + + cache-size 100000 + database-name sessions + pool-size 60 + + + # This comment preserves whitespace + db sessions + host the_server + passwd the_passwd + user the_user + + blob-dir /sample-buildout/data/sessions.blobs + cache-local-dir /sample-buildout/var/caches/data_cache/sessions.cache + cache-local-mb 300 + cache-prefix sessions + commit-lock-timeout 10 + keep-history false + name sessions + pack-gc true + shared-blob-dir false + + + +Configuring The Adapter +----------------------- + +By default, a MySQL adapter is assumed. Use the ``sql_adapter`` +setting (at the recipe or storage level) to change this. + +The ``sql_adapter_extra_args`` may be used to add additional +configuration to the ```` section. This is frequently used to +select a driver. + +If you change it to ``postgresql`` a DSN will be constructed based on +the ``sql_*`` settings. You can set ``sql_adapter_args`` to completely +specify the contents of the ```` section (this disables +``sql_adapter_extra_args``). + + >>> write(sample_buildout, 'buildout.cfg', + ... """ + ... [buildout] + ... extends = deployment.cfg + ... parts = directories relstorage + ... + ... [environment] + ... sql_user = the_user + ... sql_host = the_server + ... sql_passwd = the_passwd + ... + ... [relstorage] + ... recipe = nti.recipes.zodb:relstorage + ... storages = users sessions + ... compress = none + ... sql_adapter = postgresql + ... sql_adapter_extra_args = + ... driver gevent psycopg2 + ... + ... [relstorage_users_storage_opts] + ... sql_adapter = sqlite3 + ... sql_adapter_extra_args = + ... driver gevent sqlite3 + ... """) + + >>> print_(system(buildout), end='') + Uninstalling relstorage... + Installing relstorage. + >>> ls(sample_buildout, 'etc') + d cron.d + d logrotate.d + - zeo_uris.ini + - zodb_conf.xml + >>> cat(sample_buildout, 'etc', 'zodb_conf.xml') + %import relstorage + + cache-size 100000 + database-name users + pool-size 60 + + + # This comment preserves whitespace + data-dir /sample-buildout/data/relstorage_users_storage + driver gevent sqlite3 + + blob-dir /sample-buildout/data/users.blobs + cache-local-dir /sample-buildout/var/caches/data_cache/users.cache + cache-local-mb 300 + cache-prefix users + commit-lock-timeout 60 + keep-history false + name users + pack-gc false + shared-blob-dir true + + + + cache-size 100000 + database-name sessions + pool-size 60 + + + # This comment preserves whitespace + driver gevent psycopg2 + dsn dbname='sessions' user='the_user' password='the_passwd' host='the_server' + + blob-dir /sample-buildout/data/sessions.blobs + cache-local-dir /sample-buildout/var/caches/data_cache/sessions.cache + cache-local-mb 300 + cache-prefix sessions + commit-lock-timeout 60 + keep-history false + name sessions + pack-gc false + shared-blob-dir false + + + +Other Files +----------- + +If the recipe was ``write-zodbconvert`` set to ``true``, then a set of +configuration files for converting to and from RelStorage and +FileStorage using ``zodbconvert`` will be generated. + + >>> write(sample_buildout, 'buildout.cfg', + ... """ + ... [buildout] + ... extends = deployment.cfg + ... parts = directories relstorage + ... + ... [environment] + ... sql_user = the_user + ... sql_host = the_server + ... sql_passwd = the_passwd + ... + ... [relstorage] + ... recipe = nti.recipes.zodb:relstorage + ... storages = users + ... compress = none + ... write-zodbconvert = true + ... """) + + >>> print_(system(buildout), end='') + Uninstalling relstorage... + Installing relstorage. + >>> ls(sample_buildout, 'etc') + d cron.d + d logrotate.d + d relstorage + - zeo_uris.ini + - zodb_conf.xml + >>> ls(sample_buildout, 'etc', 'relstorage') + - users_from_relstorage_conf.xml + - users_to_relstorage_conf.xml + +ZEO +=== + +The ``zeo`` recipe can be used to create configurations for a ZEO +client and ZEO server. It is only intended for personal or test +environments. + +.. rubric:: Options + +Just like the ``relstorage`` recipe, it requires one or more storages. +Options can be set in the ``zeo`` part to apply to all storages, in a +part named for the storage to configure the server, or in a part named +for the client to configure the client. Note that the client also +inherits configuration options from the server. + +pack-gc + Defaults to false. This can only be set on the recipe part. + + + >>> write(sample_buildout, 'buildout.cfg', + ... """ + ... [buildout] + ... extends = deployment.cfg + ... parts = directories zeo + ... + ... [zeo] + ... recipe = nti.recipes.zodb:zeo + ... storages = users sessions + ... compress = none + ... pack-gc = true + ... + ... [users_storage_opts] + ... pack-gc = true + ... pool_size = 25 + ... + ... [sessions_client_opts] + ... cache-size = 42 + ... """) + +And it creates several configuration files:: + + >>> print_(system(buildout), end='') + Uninstalling relstorage... + Installing zeo. + >>> ls(sample_buildout, 'etc') + d cron.d + d logrotate.d + - zeo-zdaemon.conf + - zeo-zeo.conf + - zeo_uris.ini + - zodb_conf.xml + - zodb_file_uris.ini + +.. rubric:: Standard Files + +The ``zodb_conf.xml`` and ``zeo_uris.ini`` are created as for RelStorage (the ``zconfig://`` +prefixes are missing from the URIs because of a quirk in testing): + + >>> cat(sample_buildout, 'etc', 'zodb_conf.xml') + + cache-size 100000 + database-name users_client + pool-size 25 + + blob-dir /sample-buildout/data/users_client.blobs + name users_client + server /sample-buildout/var/zeosocket + shared-blob-dir true + storage 1 + + + + cache-size 42 + database-name sessions_client + pool-size 60 + + blob-dir /sample-buildout/data/sessions_client.blobs + name sessions_client + server /sample-buildout/var/zeosocket + shared-blob-dir true + storage 2 + + + >>> cat(sample_buildout, 'etc', 'zeo_uris.ini') + [ZODB] + uris = /sample-buildout/etc/zodb_conf.xml#users /sample-buildout/etc/zodb_conf.xml#sessions + +.. rubric:: zdaemon.conf + +This file, prefixed with the name of the buildout part, is the +configuration to use with the ``zdaemon`` command's ``-C`` option in +order to run the ZEO server. + + >>> cat(sample_buildout, 'etc', 'zeo-zdaemon.conf') + + daemon on + directory /sample-buildout/var + program /sample-buildout/bin/runzeo -C /sample-buildout/etc/zeo-zeo.conf + socket-name /sample-buildout/var/zeo-zdaemon.sock + transcript /sample-buildout/var/log/zeo-zeo.log + user user + + + + + path /sample-buildout/var/log/zeo-zeo.log + + + +.. rubric:: zeo-conf.conf + +This is the actual ZEO server configuration, again prefixed with the part name. + + >>> cat(sample_buildout, 'etc', 'zeo-zeo.conf') + + address /sample-buildout/var/zeosocket + + + + blob-dir /sample-buildout/data/users.blobs + pack-gc true + path /sample-buildout/data/users.fs + + + + blob-dir /sample-buildout/data/sessions.blobs + pack-gc true + path /sample-buildout/data/sessions.fs + + + + + format %(asctime)s %(message)s + level DEBUG + path /sample-buildout/var/log/zeo.log + + + +.. rubric:: zodb_file_uris.ini + +Intentionally undocumented. Expert use only. diff --git a/setup.py b/setup.py index af6ab5b..f3dd12a 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,11 @@ TESTS_REQUIRE = [ 'PyHamcrest', + 'collective.recipe.template', + 'z3c.recipe.mkdir', + 'zope.testing', 'zope.testrunner', + 'zdaemon', ] def read_file(*path): diff --git a/src/nti/recipes/zodb/__init__.py b/src/nti/recipes/zodb/__init__.py index 34cc03c..b681d36 100644 --- a/src/nti/recipes/zodb/__init__.py +++ b/src/nti/recipes/zodb/__init__.py @@ -14,7 +14,6 @@ from ._model import ChoiceRef from ._model import Part from ._model import Default -from ._model import hyphenated class MetaRecipe(object): # Contains the base methods that are required of a recipe, @@ -47,33 +46,6 @@ class serverzlibstorage(zlibstorage): class zodb(ZConfigSection): - # Also crucial is the pool-size. Each connection has resources - # like a memcache connection, a MySQL connection, and its - # in-memory cache. Normally, opening a DB and closing the - # connection will create a connection (if needed), then return - # it to the pool. However, in the case of multi-databases, - # when an object from a secondary database needs to be loaded, - # the active connection will request a connection to that - # database, and when the active connection is closed, that - # secondary connection is also closed: BUT NOT RETURNED TO THE - # POOL. Instead, the active (primary) connection keeps a - # reference to it that it will use in the future. This has the - # effect of driving all secondary pools based on the - # efficiency of the primary pool. Thus, the pool-size for - # everything except the primary database is essentially - # meaningless (if the application always begins by opening - # that primary database), but that pool size controls everything. - - # If we're doing XHR-polling, it's not unusual for one gunicorn/gevent - # worker to have 30 or 40 polling requests active on it an any - # one time. Each of those consumes a connection; if our pool size - # is smaller than the number of active requests, we can wind - # up thrashing on connections, rapidly opening and closing them. - # (Because closing the main connection causes it to be repushed on the pool, - # which triggers the pool to shrink in size if need be). Calling - # DB.connectionDebugInfo() can show this: connections in the pool - # have 'opened' of None, while those in use have a timestamp and the length - # of time it's been open. pool_size = Default(60).hyphenate() database_name = Ref('name').hyphenate() cache_size = Ref('cache-size').hyphenate() @@ -121,8 +93,8 @@ def __init__(self, buildout, my_name, my_options): # that uses the z3c.recipe.mkdir recipe to create them. self._dirs_to_create_refs = set() # Likewise, but referring to settings that define a - # element as a string. - self._zodb_refs = set() + # element as a string. Order matters. + self._zodb_refs = [] buildout[self.my_name + '_opts_base'] = { k: v @@ -135,7 +107,7 @@ def create_directory(self, part, setting): def add_database(self, part, setting): # Adds the ZCML at part:setting to zodb_conf.xml - self._zodb_refs.add(Ref(part, setting)) + self._zodb_refs.append(Ref(part, setting)) def ref(self, part, setting=None): """ @@ -179,6 +151,8 @@ def buildout_add_mkdirs(self, name=None): paths=self.__refs_to_lines(self._dirs_to_create_refs)) self._parse(part) + import_relstorage = '%import relstorage' + def buildout_add_zodb_conf(self): zcml_names = self.__refs_to_lines(self._zodb_refs) part = Part( @@ -188,7 +162,7 @@ def buildout_add_zodb_conf(self): input=[ 'inline:', self.zlibstorage_import(), - '%import relstorage', + self.import_relstorage, ] + zcml_names ) self._parse(part) diff --git a/src/nti/recipes/zodb/relstorage.py b/src/nti/recipes/zodb/relstorage.py index 786c649..2ff3d6e 100644 --- a/src/nti/recipes/zodb/relstorage.py +++ b/src/nti/recipes/zodb/relstorage.py @@ -57,7 +57,7 @@ def __init__(self, memcache_config): ) class BaseStoragePart(ZodbClientPart): - blob_cache_size = hyphenated(None) + blob_cache_size = Default(None).hyphenate() blob_dir = LocalSubstVar('data_dir') / LocalSubstVar('name') + '.blobs' blob_dump_dir = ( LocalSubstVar('data_dir') @@ -66,7 +66,7 @@ class BaseStoragePart(ZodbClientPart): / 'blobs' ) cache_local_dir = hyphenated(None) - cache_local_mb = hyphenated(None) + cache_local_mb = Default(300).hyphenate() commit_lock_timeout = Default(60) data_dir = SubstVar('deployment', 'data-directory') @@ -74,9 +74,14 @@ class BaseStoragePart(ZodbClientPart): dump_name = LocalSubstVar('name') filestorage_name = 'NONE' name = 'BASE' - pack_gc = hyphenated(False) + pack_gc = Default(False).hyphenate() relstorage_name_prefix = hyphenated(None) - shared_blob_dir = hyphenated(False) + # Prior to RelStorage 3, by default, relstorage assumes a shared blob + # directory. However, our most common use case here + # is not to share. While using either wrong setting + # in an environment is dangerous and can lead to data loss, + # it's slightly worse to assume shared when its not + shared_blob_dir = Default(False).hyphenate() sql_db = LocalSubstVar('name') sql_adapter_args = ZConfigSnippet( @@ -115,14 +120,6 @@ def __init__(self, buildout, name, options): sql_host = options.get('sql_host') or environment.get('sql_host') sql_adapter = options.get('sql_adapter') or 'mysql' - # by default, relstorage assumes a shared blob - # directory. However, our most common use case here - # is not to share. While using either wrong setting - # in an environment is dangerous and can lead to data loss, - # it's slightly worse to assume shared when its not - if 'shared-blob-dir' not in options: - options['shared-blob-dir'] = 'false' - shared_blob_dir = options['shared-blob-dir'] cache_local_dir = '' if _option_true(options.get('enable-persistent-cache', 'true')): @@ -132,10 +129,6 @@ def __init__(self, buildout, name, options): # avoid taking any user-defined values since it might be # confusing to have one (count limited) directory for all storages. cache_local_dir = '${deployment:cache-directory}/data_cache/${:name}.cache' - cache_local_mb = options.get('cache-local-mb', '300') - - blob_cache_size = options.get('blob-cache-size', '') - pack_gc = options.get('pack-gc', 'false') # Utilizing the built in memcache capabilites is not # beneficial in all cases. In fact it rarely is. It's @@ -160,7 +153,7 @@ def __init__(self, buildout, name, options): relstorage_zcml = self.zlibstorage_wrapper(relstorage(remote_cache_config)) filestorage_zcml = self.zlibstorage_wrapper(filestorage(self.ref('filestorage_name'))) - + blob_cache_size = options.get('blob-cache-size', '') # Order matters base_storage_name = name + '_base_storage' @@ -173,15 +166,13 @@ def __init__(self, buildout, name, options): storage_zcml=relstorage_zcml, client_zcml=zodb(self.ref('name'), self.ref('storage_zcml')), filestorage_zcml=filestorage_zcml, - shared_blob_dir=shared_blob_dir, relstorage_name_prefix=relstorage_name_prefix, cache_local_dir=cache_local_dir, - cache_local_mb=cache_local_mb, blob_cache_size=blob_cache_size, - pack_gc=pack_gc, **extra_base_kwargs ) + # TODO: Let this be configured for each storage. if not blob_cache_size: del base_storage_part['blob-cache-size'] zcml = base_storage_part['storage_zcml'] @@ -211,6 +202,7 @@ def __init__(self, buildout, name, options): name=storage, ) + part = part.with_settings(**self.__adapter_settings(part)) self._parse(part) @@ -225,6 +217,15 @@ def __init__(self, buildout, name, options): self.buildout_add_zodb_conf() self.buildout_add_zeo_uris() + def _resolve(self, part, obj): + if isinstance(obj, SubstVar): + if not obj.part: # Relative. + return self._resolve(part, part.get(obj.setting)) + # buildout values are already fully resolved + return self.buildout[obj.part][obj.setting] # pragma: no cover + return obj + + def __clear_top_level_inherited_adapter_settings(self, sql_adapter_args): for k in BaseStoragePart.sql_adapter_args.keys(): sql_adapter_args.pop(k, None) @@ -251,12 +252,7 @@ def _adapter_settings_for_postgresql(self, part, sql_adapter_args): # Hoist everything present by default into the dsn. Resolve them now so that we don't # put in empty fields and can use defaults. def resolve(obj): - if isinstance(obj, SubstVar): - if not obj.part: # Relative. - return resolve(part.get(obj.setting)) - # buildout values are already fully resolved - return self.buildout[obj.part][obj.setting] # pragma: no cover - return obj + return self._resolve(part, obj) dsn = ' ' for dsn_key, setting_key in ( ('dbname', 'db'), diff --git a/src/nti/recipes/zodb/tests/test_readme.py b/src/nti/recipes/zodb/tests/test_readme.py new file mode 100644 index 0000000..02789a3 --- /dev/null +++ b/src/nti/recipes/zodb/tests/test_readme.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import doctest +import os +import unittest + +from zope.testing import renormalizing +import zc.buildout.testing + +def test_suite(): + # If these get passed to the child buildout, + # they result in unwanted warnings which mess up + # doctests. They've already been read, no harm in + # removing them. + os.environ.pop('PYTHONWARNINGS', None) + os.environ.pop('PYTHONDEVMODE', None) + root = this_dir = os.path.dirname(os.path.abspath(__file__)) + while not os.path.exists(os.path.join(root, 'setup.py')): + prev, root = root, os.path.dirname(root) + if root == prev: + # Let's avoid infinite loops at root + raise AssertionError('could not find my setup.py') + + optionflags = ( + doctest.NORMALIZE_WHITESPACE + | doctest.ELLIPSIS + | doctest.IGNORE_EXCEPTION_DETAIL + ) + + index_rst = os.path.join(root, 'README.rst') + # Can't pass absolute paths to DocFileSuite, needs to be + # module relative + index_rst = os.path.relpath(index_rst, this_dir) + + return unittest.TestSuite(( + unittest.defaultTestLoader.loadTestsFromName(__name__), + doctest.DocFileSuite( + index_rst, + setUp=zc.buildout.testing.buildoutSetUp, + tearDown=zc.buildout.testing.buildoutTearDown, + optionflags=optionflags, + checker=renormalizing.RENormalizing([ + zc.buildout.testing.normalize_path, + zc.buildout.testing.normalize_endings, + zc.buildout.testing.normalize_script, + zc.buildout.testing.normalize_egg_py, + zc.buildout.testing.not_found, + ]), + ) + )) diff --git a/src/nti/recipes/zodb/tests/test_zeo.py b/src/nti/recipes/zodb/tests/test_zeo.py index 6c3c9de..093fb38 100644 --- a/src/nti/recipes/zodb/tests/test_zeo.py +++ b/src/nti/recipes/zodb/tests/test_zeo.py @@ -36,7 +36,7 @@ def test_parse(self): name users_1_client server /var/zeosocket shared-blob-dir true - storage 1 + storage 2 compress false @@ -105,7 +105,7 @@ def test_parse_no_compress(self): name users_1_client server /var/zeosocket shared-blob-dir true - storage 1 + storage 2 """ diff --git a/src/nti/recipes/zodb/zeo.py b/src/nti/recipes/zodb/zeo.py index 6246b82..12ef59a 100644 --- a/src/nti/recipes/zodb/zeo.py +++ b/src/nti/recipes/zodb/zeo.py @@ -57,7 +57,7 @@ def __init__(self): 'logfile', None, path=Ref('logFile'), format="%(asctime)s %(message)s", - level='DEBUG', + level="DEBUG", ) ZConfigSection.__init__( self, @@ -73,11 +73,12 @@ class BaseZeoPart(Part): deployment = 'deployment' class Databases(MultiStorageRecipe): + import_relstorage = '' def __init__(self, buildout, name, options): MultiStorageRecipe.__init__(self, buildout, name, options) storages = options['storages'].split() - zeo_name = options.get('name', 'Dataserver') + zeo_name = options.get('name', name) # Order matters base_storage_part = BaseStoragePart( @@ -97,6 +98,7 @@ def __init__(self, buildout, name, options): base_client_part = BaseClientPart( self._derive_related_part_name('base_client'), extends=(base_storage_part,), + storage_num=1, client_zcml=zodb( Ref('name'), self.zlibstorage_wrapper( @@ -104,7 +106,7 @@ def __init__(self, buildout, name, options): server=BaseZeoPart.clientPipe, shared_blob_dir=hyphenated(True), blob_dir=hyphenated(self.ref('blob_dir')), - storage=1, + storage=self.ref('storage_num'), name=self.ref('name'), ) ) @@ -158,7 +160,8 @@ def __init__(self, buildout, name, options): client_part = Part( client_part_name, extends=client_part_extends, - name=client_part_name + name=client_part_name, + storage_num=i, ) client_parts.append(client_part) From e8ac5f256cd8568db4d4d1d5ce0c86d0b4aa1502 Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Fri, 15 Nov 2019 10:41:43 -0600 Subject: [PATCH 2/2] Back to 100% coverage. --- src/nti/recipes/zodb/tests/test_readme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nti/recipes/zodb/tests/test_readme.py b/src/nti/recipes/zodb/tests/test_readme.py index 02789a3..817fcb5 100644 --- a/src/nti/recipes/zodb/tests/test_readme.py +++ b/src/nti/recipes/zodb/tests/test_readme.py @@ -22,7 +22,7 @@ def test_suite(): root = this_dir = os.path.dirname(os.path.abspath(__file__)) while not os.path.exists(os.path.join(root, 'setup.py')): prev, root = root, os.path.dirname(root) - if root == prev: + if root == prev: # pragma: no cover # Let's avoid infinite loops at root raise AssertionError('could not find my setup.py')