Merge pull request #26 from newtdb/aux-tables

Aux tables

CHANGES.rst

@@ -1,10 +1,13 @@
 Changes
 =======
 
-0.8.1 (unreleased)
+0.9.0 (unreleased)
 ------------------
 
-- Nothing changed yet.
+- Added an experimental feature to update auxiliary tables to support
+  indexing data that comes from multiple newt records. (More on this
+  later if initial experiment confirm the usefulness of this
+  feature.)
 
 
 0.8.0 (2017-06-20)

src/newt/db/_adapter.py

@@ -28,6 +28,8 @@ def __init__(self, *args, **kw):
 
         self.mover.jsonifier = Jsonifier(
             transform=getattr(self.options, 'transform', None))
+        self.mover.auxiliary_tables = getattr(self.options,
+                                              'auxiliary_tables', ())
 
 class Mover(relstorage.adapters.postgresql.mover.PostgreSQLObjectMover):
 
@@ -73,9 +75,16 @@ def store_temp(self, cursor, batcher, oid, prev_tid, data):
       state = EXCLUDED.state;
     """
 
+    _update_aux_sql = """
+    DELETE FROM %(name)s WHERE zoid IN (SELECT zoid FROM temp_store);
+    INSERT INTO %(name)s (zoid)         SELECT zoid FROM temp_store ;
+    """
+
     def move_from_temp(self, cursor, tid, txn_has_blobs):
         r = super(Mover, self).move_from_temp(cursor, tid, txn_has_blobs)
         cursor.execute(self._move_json_sql)
+        for name in self.auxiliary_tables:
+            cursor.execute(self._update_aux_sql % dict(name=name))
         return r
 
     def restore(self, cursor, batcher, oid, tid, data):

src/newt/db/_db.py

@@ -109,7 +109,7 @@ def _split_options(
         large_record_size=large_record_size,
         ), storage_options
 
-def storage(dsn, keep_history=False, transform=None, **kw):
+def storage(dsn, keep_history=False, transform=None, auxiliary_tables=(), **kw):
     """Create a RelStorage storage using the newt PostgresQL adapter.
 
     Keyword options can be used to provide either `ZODB.DB
@@ -120,6 +120,7 @@ def storage(dsn, keep_history=False, transform=None, **kw):
     """
     options = relstorage.options.Options(keep_history=keep_history, **kw)
     options.transform = transform
+    options.auxiliary_tables = auxiliary_tables
     return relstorage.storage.RelStorage(Adapter(dsn, options), options=options)
 
 def DB(dsn, **kw):

src/newt/db/component.py

@@ -9,6 +9,7 @@ def global_by_name(name):
 class Adapter:
 
     def __init__(self, config):
+        self.auxiliary_tables = config.auxiliary_tables
         self.transform = config.transform
         self.config = config.adapter.config
 
@@ -17,6 +18,7 @@ def create(self, options):
         transform = self.transform
         if transform is not None:
             options.transform = global_by_name(transform)
+        options.auxiliary_tables = self.auxiliary_tables or ()
 
         return Adapter(dsn=self.config.dsn, options=options)
 

src/newt/db/component.xml

@@ -27,6 +27,18 @@
       </description>
     </key>
 
+    <key name="auxiliary-tables" datatype="string-list">
+      <description>
+        One or more names of auxiliary data tables.
+
+        These tables must have zoid as a primary key and it should be
+        the only required field.  Records will be added or replaced in
+        these tables after the newt table has been updated and they
+        will generally use newt data to compute additional columns for
+        indexing.
+      </description>
+    </key>
+
   </sectiontype>
 
   <sectiontype

src/newt/db/tests/testadapter.py

@@ -1,3 +1,4 @@
+import contextlib
 import persistent.mapping
 import os
 import pickle
@@ -55,6 +56,37 @@ def test_basic(self):
 
         conn.close()
 
+    def test_basic_auxiliary_tables(self):
+        import newt.db
+
+        conn = newt.db.pg_connection(self.dsn)
+        with conn:
+            with conn.cursor() as cursor:
+                for i in range(1, 3):
+                    cursor.execute(
+                        "create table x%s (zoid bigint primary key, x int)"
+                        % i)
+        conn.close()
+
+        conn = newt.db.connection(
+            self.dsn,
+            keep_history=self.keep_history,
+            auxiliary_tables=('x1', 'x2'),
+            )
+
+        # Add an object:
+        conn.root.x = o = Object(a=1)
+        conn.commit()
+
+        self.__assertBasicData(conn, o)
+        for i in range(1, 3):
+            self.assertEqual(
+                [[0], [1]],
+                [list(map(int, r))
+                 for r in conn.query_data('select zoid from x%s' % i)])
+
+        conn.close()
+
     def test_restore(self):
         source_db = ZODB.DB(None)
         source_conn = source_db.open()

src/newt/db/tests/testdbconfig.py

@@ -68,3 +68,40 @@ def test_newt(self):
 
         db.close()
 
+    def test_aux(self):
+        from newt.db import pg_connection
+        conn = pg_connection(self.dsn)
+        with conn:
+            with conn.cursor() as cursor:
+                cursor.execute("create table extra (zoid bigint)")
+                cursor.execute("create table extra2 (zoid bigint)")
+        conn.close()
+
+        db = databaseFromString("""\
+            %%import newt.db
+
+            <newtdb foo>
+              <zodb>
+                <relstorage>
+                  <newt>
+                    auxiliary-tables extra extra2
+                    <postgresql>
+                      dsn dbname=%s
+                    </postgresql>
+                  </newt>
+                </relstorage>
+              </zodb>
+            </newtdb>
+            """ % self.dbname)
+
+        from .._db import NewtDB
+        self.assertEqual(db.__class__, NewtDB)
+        from relstorage.storage import RelStorage
+        self.assertEqual(db.storage.__class__, RelStorage)
+        from .._adapter import Adapter
+        self.assertEqual(db.storage._adapter.__class__, Adapter)
+        self.assertEqual(db.storage._adapter.mover.auxiliary_tables,
+                         ['extra', 'extra2'])
+
+        db.close()
+