Merge 84b2c6e into 8c4d476

CHANGES.rst

@@ -4,7 +4,8 @@ Changes
 1.2 (unreleased)
 ----------------
 
-- Nothing changed yet.
+- Fixed skipping of blob records so that oids in references to blobs
+  are still converted.
 
 
 1.1 (2018-10-05)

src/zodbupdate/serialize.py

@@ -19,22 +19,20 @@
 import six
 import zodbpickle
 
+from ZODB.blob import Blob
 from ZODB.broken import find_global, Broken, rebuild
 from zodbupdate import utils
 
 logger = logging.getLogger('zodbupdate.serialize')
 known_broken_modules = {}
 
-# types to skip when renaming/migrating databases
-SKIP_SYMBS = [('ZODB.blob', 'Blob')]
-
 
 def create_broken_module_for(symb):
     """If your pickle refer a broken class (not an instance of it, a
        reference to the class symbol itself) you have no choice than
        having this module available in the same symbol and with the
        same name, otherwise repickling doesn't work (as both pickle
-       and cPikle __import__ the module, and verify the class symbol
+       and cPickle __import__ the module, and verify the class symbol
        is the same than the one provided).
     """
     parts = symb.__module__.split('.')
@@ -158,10 +156,7 @@ def __update_symb(self, symb_info):
         loaded and its location is checked to see if it have moved as
         well.
         """
-        if symb_info in SKIP_SYMBS:
-            self.__skipped = True
-            return symb_info
-        elif symb_info in self.__renames:
+        if symb_info in self.__renames:
             self.__changed = True
             return self.__renames[symb_info]
         else:
@@ -298,18 +293,17 @@ def rename(self, input_file):
         return None otherwise.
         """
         self.__changed = False
-        self.__skipped = False
 
         unpickler = self.__unpickler(input_file)
         class_meta = unpickler.load()
         data = unpickler.load()
 
-        class_meta = self.__update_class_meta(class_meta)
-
-        if self.__skipped:
+        if class_meta is Blob:
             # do not do renames/conversions on blob records
             return None
 
+        class_meta = self.__update_class_meta(class_meta)
+
         self.__decode_data(class_meta, data)
 
         if not (self.__changed or self.__repickle_all):
@@ -322,7 +316,7 @@ def rename(self, input_file):
             pickler.dump(data)
         except utils.PicklingError as error:
             logger.error(
-                'Error: cannot pickling modified record: {}'.format(error))
+                'Error: cannot pickle modified record: {}'.format(error))
             # Could not pickle that record, skip it.
             return None
 

src/zodbupdate/tests.py

@@ -14,10 +14,12 @@
 ##############################################################################
 
 import ZODB
+import ZODB.blob
 import ZODB.broken
 import ZODB.FileStorage
 import os
 import persistent
+import shutil
 import sys
 import tempfile
 import transaction
@@ -89,25 +91,33 @@ class OtherFactory(persistent.Persistent):
         IOtherFactory.__module__ = 'module2.interfaces'
 
         self.tmphnd, self.dbfile = tempfile.mkstemp()
+        self.tmpblob = tempfile.mkdtemp()
 
-        self.storage = ZODB.FileStorage.FileStorage(self.dbfile)
+        self.storage = ZODB.blob.BlobStorage(
+            self.tmpblob,
+            ZODB.FileStorage.FileStorage(self.dbfile),
+        )
         self.db = ZODB.DB(self.storage)
         self.conn = self.db.open()
         self.root = self.conn.root()
 
-        self._skipped_symbs = zodbupdate.serialize.SKIP_SYMBS
-
     def update(self, **args):
         self.conn.close()
         self.db.close()
         self.storage.close()
 
-        self.storage = ZODB.FileStorage.FileStorage(self.dbfile)
+        self.storage = ZODB.blob.BlobStorage(
+            self.tmpblob,
+            ZODB.FileStorage.FileStorage(self.dbfile),
+        )
         updater = zodbupdate.main.create_updater(self.storage, **args)
         updater()
         self.storage.close()
 
-        self.storage = ZODB.FileStorage.FileStorage(self.dbfile)
+        self.storage = ZODB.blob.BlobStorage(
+            self.tmpblob,
+            ZODB.FileStorage.FileStorage(self.dbfile),
+        )
         self.db = ZODB.DB(self.storage)
         self.conn = self.db.open()
         self.root = self.conn.root()
@@ -134,8 +144,7 @@ def tearDown(self):
         os.unlink(self.dbfile + '.index')
         os.unlink(self.dbfile + '.tmp')
         os.unlink(self.dbfile + '.lock')
-
-        zodbupdate.serialize.SKIP_SYMBS = self._skipped_symbs
+        shutil.rmtree(self.tmpblob)
 
     def test_no_transaction_if_no_changes(self):
         # If an update run doesn't produce any changes it won't commit the
@@ -180,45 +189,6 @@ def getName(self):
         renames = updater.processor.get_rules(implicit=True)
         self.assertEqual({}, renames)
 
-    def test_skipped_types_are_left_untouched(self):
-        skipped = sys.modules['module1'].Factory()
-        self.root['skipped'] = skipped
-        transaction.commit()
-
-        self.assertIn(('ZODB.blob', 'Blob'), zodbupdate.serialize.SKIP_SYMBS)
-        zodbupdate.serialize.SKIP_SYMBS += [('module1', 'Factory')]
-
-        oid = self.root['skipped']._p_oid
-        old_pickle, old_serial = self.storage.load(oid)
-
-        self.update(
-            default_renames={
-                ('module1', 'Factory'): ('module2', 'OtherFactory')})
-
-        pickle, serial = self.storage.load(oid)
-        self.assertEqual(old_pickle, pickle)
-        self.assertEqual(old_serial, serial)
-
-    def test_not_skipped_types_are_touched(self):
-        skipped = sys.modules['module1'].Factory()
-        self.root['skipped'] = skipped
-        transaction.commit()
-
-        self.assertNotIn(
-            ('module1', 'Factory'),
-            zodbupdate.serialize.SKIP_SYMBS)
-
-        oid = self.root['skipped']._p_oid
-        old_pickle, old_serial = self.storage.load(oid)
-
-        self.update(
-            default_renames={
-                ('module1', 'Factory'): ('module2', 'OtherFactory')})
-
-        pickle, serial = self.storage.load(oid)
-        self.assertNotEqual(old_pickle, pickle)
-        self.assertNotEqual(old_serial, serial)
-
 
 class Python2Tests(Tests):
 
@@ -671,6 +641,28 @@ def test_decode_attribute_leaves_none_untouched(self):
         self.assertEquals(result, False)
         self.assertEquals(mock['foo'], None)
 
+    def test_blobs_are_left_untouched(self):
+        from zodbupdate.convert import encode_binary
+
+        blob = ZODB.blob.Blob()
+        self.root['blob'] = blob
+        transaction.commit()
+
+        oid = self.root['blob']._p_oid
+        old_pickle, old_serial = self.storage.load(oid)
+
+        self.update(convert_py3=True)
+
+        pickle, serial = self.storage.load(oid)
+        self.assertEqual(old_pickle, pickle)
+        self.assertEqual(old_serial, serial)
+
+        # make sure the reference to the blob was converted
+        self.assertIn(
+            b'C\x08\x00\x00\x00\x00\x00\x00\x00\x01cZODB.blob',
+            self.storage.load(self.root._p_oid, '')[0]
+        )
+
 
 class Python3Tests(Tests):