Merge pull request #3 from plone/zlib-compressed-storage-support

Added support for compressed file storage
plone · Oct 27, 2011 · 982947e · 982947e
2 parents 8ba04b4 + 81036c4
commit 982947e
Show file tree

Hide file tree

Showing 3 changed files with 161 additions and 3 deletions.
diff --git a/README.txt b/README.txt
@@ -372,6 +372,15 @@ webdav-address
 webdav-force-connection-close
   Valid options are off and on. Defaults to off
 
+zlib-storage
+  Adds support for file compression on a file storage database. The
+  option accepts the values 'active' (compress new records) or
+  'passive' (do not compress new records). Both options support
+  already compressed records.
+
+  You can use the 'passive' setting while you prepare a number of
+  connected clients for compressed records.
+
 zodb-cache-size-bytes
   Set the ZODB cache sizes in bytes. This feature is still experimental.
 

diff --git a/src/plone/recipe/zope2instance/__init__.py b/src/plone/recipe/zope2instance/__init__.py
@@ -295,6 +295,8 @@ def build_zope_conf(self):
         demo_storage = options.get('demo-storage', 'off') \
                        not in ('off', 'disable', 'false')
 
+        zlib = options.get('zlib-storage')
+
         default_blob = os.path.join(var_dir, 'blobstorage')
         default_file = os.path.sep.join(('filestorage', 'Data.fs',))
 
@@ -352,7 +354,7 @@ def is_rs_option(name):
             file_storage_snippet = rel_storage_template % opts
         else:
             file_storage_snippet = self.render_file_storage(
-                file_storage, blob_storage, base_dir, var_dir)
+                file_storage, blob_storage, base_dir, var_dir, zlib)
 
         zserver_threads = options.get('zserver-threads', '2')
         if zserver_threads:
@@ -455,7 +457,7 @@ def is_rs_option(name):
             if demo_file_storage or demo_blob_storage:
                 base = storage_snippet.replace('>', ' base>', 1)
                 changes = self.render_file_storage(
-                    demo_file_storage, demo_blob_storage, base_dir, var_dir).\
+                    demo_file_storage, demo_blob_storage, base_dir, var_dir, zlib).\
                     replace('>', ' changes>', 1)
 
                 storage_snippet = demo_storage2_template % (base, changes)
@@ -656,13 +658,28 @@ def build_package_includes(self):
                     % (package, filename)
                     )
 
-    def render_file_storage(self, file_storage, blob_storage, base_dir, var_dir):
+    def render_file_storage(self, file_storage, blob_storage,
+                            base_dir, var_dir, zlib):
         if file_storage:
             file_storage = os.path.join(var_dir, file_storage)
             file_storage_dir = os.path.dirname(file_storage)
             if not os.path.exists(file_storage_dir):
                 os.makedirs(file_storage_dir)
             storage = file_storage_template % file_storage
+            if zlib is not None:
+                if zlib == 'active':
+                    compress = 'true'
+                elif zlib == 'passive':
+                    compress = 'false'
+                else:
+                    raise ValueError(
+                        "Valid options for ``zlib-storage`` are "\
+                        "('compress', 'uncompress'). Got: %s." % zlib
+                        )
+
+                storage = zlib_storage_template % (
+                    compress, indent(storage, 2)
+                    )
         else:
             storage = "    <demostorage />"
 
@@ -686,6 +703,15 @@ def render_file_storage(self, file_storage, blob_storage, base_dir, var_dir):
     </filestorage>
 """
 
+zlib_storage_template="""
+    %%import zc.zlibstorage
+    # ZlibStorage wrapper
+    <zlibstorage>
+      compress %s
+%s
+    </zlibstorage>
+"""
+
 demo_storage_template="""
     # DemoStorage
     <demostorage>

diff --git a/src/plone/recipe/zope2instance/tests/zope2instance.txt b/src/plone/recipe/zope2instance/tests/zope2instance.txt
@@ -301,6 +301,9 @@ Let's run it::
     Uninstalling instance.
     Installing instance.
 
+
+
+
 We should have a zope instance, with a basic zope.conf without demostorage::
 
     >>> instance = os.path.join(sample_buildout, 'parts', 'instance')
@@ -492,6 +495,126 @@ We should have a zope instance, with a basic zope.conf::
     </zodb_db>
     ...
 
+
+ZlibStorage
+===========
+
+To have a ZlibStorage configuration, you can use zlib-storage::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = instance
+    ... find-links = %(sample_buildout)s/eggs
+    ...
+    ... [instance]
+    ... recipe = plone.recipe.zope2instance
+    ... eggs =
+    ... user = me:me
+    ... file-storage = newfs/Data.fs
+    ... zlib-storage = active
+    ...
+    ... ''' % options)
+
+Let's run it::
+
+    >>> print system(join('bin', 'buildout')),
+    Uninstalling instance.
+    Installing instance.
+
+We should have a zope instance, with a basic zope.conf::
+
+    >>> instance = os.path.join(sample_buildout, 'parts', 'instance')
+    >>> zope_conf = open(os.path.join(instance, 'etc', 'zope.conf')).read()
+    >>> zope_conf = zope_conf.replace('\\', '/')
+    >>> print zope_conf
+    %define INSTANCEHOME .../sample-buildout/parts/instance
+    ...
+    <zodb_db main>
+        # Main database
+        cache-size 10000
+    <BLANKLINE>
+    # Blob-enabled FileStorage database
+        <blobstorage>
+          blob-dir .../sample-buildout/var/blobstorage
+    <BLANKLINE>
+          %import zc.zlibstorage
+          # ZlibStorage wrapper
+          <zlibstorage>
+            compress true
+    <BLANKLINE>
+            # FileStorage database
+            <filestorage>
+              path .../sample-buildout/var/newfs/Data.fs
+            </filestorage>
+    <BLANKLINE>
+          </zlibstorage>
+    <BLANKLINE>
+        </blobstorage>
+        mount-point /
+    </zodb_db>
+    ...
+    <BLANKLINE>
+
+To have a ZlibStorage configuration with no active compression, you
+can set the ``zlib-storage`` option to 'passive'::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = instance
+    ... find-links = %(sample_buildout)s/eggs
+    ...
+    ... [instance]
+    ... recipe = plone.recipe.zope2instance
+    ... eggs =
+    ... user = me:me
+    ... file-storage = newfs/Data.fs
+    ... zlib-storage = passive
+    ...
+    ... ''' % options)
+
+Let's run it::
+
+    >>> print system(join('bin', 'buildout')),
+    Uninstalling instance.
+    Installing instance.
+
+We should have a zope instance, with a basic zope.conf::
+
+    >>> instance = os.path.join(sample_buildout, 'parts', 'instance')
+    >>> zope_conf = open(os.path.join(instance, 'etc', 'zope.conf')).read()
+    >>> zope_conf = zope_conf.replace('\\', '/')
+    >>> print zope_conf
+    %define INSTANCEHOME .../sample-buildout/parts/instance
+    ...
+    <zodb_db main>
+        # Main database
+        cache-size 10000
+    <BLANKLINE>
+    # Blob-enabled FileStorage database
+        <blobstorage>
+          blob-dir .../sample-buildout/var/blobstorage
+    <BLANKLINE>
+          %import zc.zlibstorage
+          # ZlibStorage wrapper
+          <zlibstorage>
+            compress false
+    <BLANKLINE>
+            # FileStorage database
+            <filestorage>
+              path .../sample-buildout/var/newfs/Data.fs
+            </filestorage>
+    <BLANKLINE>
+          </zlibstorage>
+    <BLANKLINE>
+        </blobstorage>
+        mount-point /
+    </zodb_db>
+    ...
+    <BLANKLINE>
+
+
 BeforeStorage
 =============