Merge pull request #304 from qutech/issues/250_serialization_conversi…

…on_script

Serialization conversion script

qctoolkit/serialization.py

@@ -10,7 +10,7 @@
 """
 
 from abc import ABCMeta, abstractmethod
-from typing import Dict, Any, Optional, NamedTuple, Union, Mapping, MutableMapping
+from typing import Dict, Any, Optional, NamedTuple, Union, Mapping, MutableMapping, Set
 import os
 import zipfile
 import tempfile
@@ -23,7 +23,8 @@
 from qctoolkit.utils.types import DocStringABCMeta
 
 __all__ = ["StorageBackend", "FilesystemBackend", "ZipFileBackend", "CachingBackend", "Serializable", "Serializer",
-           "AnonymousSerializable", "DictBackend", "JSONSerializableEncoder", "JSONSerializableDecoder", "PulseStorage"]
+           "AnonymousSerializable", "DictBackend", "JSONSerializableEncoder", "JSONSerializableDecoder", "PulseStorage",
+           "convert_pulses_in_storage", "convert_stored_pulse_in_storage"]
 
 
 class StorageBackend(metaclass=ABCMeta):
@@ -91,6 +92,14 @@ def delete(self, identifier: str) -> None:
     def __delitem__(self, identifier: str) -> None:
         self.delete(identifier)
 
+    @abstractmethod
+    def list_contents(self) -> Set[str]:
+        """Return a listing of all available identifiers.
+
+        Returns:
+            List of all available identifiers.
+        """
+
 
 class FilesystemBackend(StorageBackend):
     """A StorageBackend implementation based on a regular filesystem.
@@ -141,6 +150,21 @@ def delete(self, identifier):
         except FileNotFoundError as fnf:
             raise KeyError(identifier) from fnf
 
+    def list_contents(self) -> Set[str]:
+        contents = set()
+        for dirpath, dirs, files in os.walk(self._root):
+            contents = contents | {filename
+                                   for filename, ext in (os.path.splitext(file) for file in files)
+                                   if ext == '.json'}
+            break # abort after first iteration; FileSystemBackend doesn't allow for subdirectories anyway right now, so this is a safeguard
+
+            # pref = os.path.commonprefix((dirpath, self._root))
+            # dir_rel_path = dirpath[len(pref):]
+            # contents = contents | {os.path.join(dir_rel_path, filename)
+            #                        for filename, ext in (os.path.splitext(file) for file in files)
+            #                        if ext == '.json'}
+        return contents
+
 
 class ZipFileBackend(StorageBackend):
     """A StorageBackend implementation based on a single zip file.
@@ -225,6 +249,12 @@ def _update(self, filename: str, data: Optional[str]) -> None:
             with zipfile.ZipFile(self._root, mode='a', compression=zipfile.ZIP_DEFLATED) as zf:
                 zf.writestr(filename, data)
 
+    def list_contents(self) -> Set[str]:
+        with zipfile.ZipFile(self._root, 'r') as myzip:
+            return set(filename
+                       for filename, ext in (os.path.splitext(file) for file in myzip.namelist())
+                       if ext == '.json')
+
 
 class CachingBackend(StorageBackend):
     """Adds naive memory caching functionality to another StorageBackend.
@@ -265,6 +295,9 @@ def delete(self, identifier: str) -> None:
         if identifier in self._cache:
             del self._cache[identifier]
 
+    def list_contents(self) -> Set[str]:
+        return self._backend.list_contents()
+
 
 class DictBackend(StorageBackend):
     """DictBackend uses a dictionary to store the data for convenience serialization."""
@@ -289,6 +322,9 @@ def storage(self) -> Dict[str, str]:
     def delete(self, identifier: str) -> None:
         del self._cache[identifier]
 
+    def list_contents(self) -> Set[str]:
+        return set(self._cache.keys())
+
 
 def get_type_identifier(obj: Any) -> str:
     """Return a unique type identifier for any object.
@@ -622,7 +658,7 @@ def deserialize(self, representation: Union[str, Dict[str, Any]]) -> Serializabl
         if isinstance(representation, str):
             if representation in self.__subpulses:
                 return self.__subpulses[representation].serializable
-        
+
         if isinstance(representation, str):
             repr_ = json.loads(self.__storage_backend.get(representation))
             repr_['identifier'] = representation
@@ -632,12 +668,12 @@ def deserialize(self, representation: Union[str, Dict[str, Any]]) -> Serializabl
         module_name, class_name = repr_['type'].rsplit('.', 1)
         module = __import__(module_name, fromlist=[class_name])
         class_ = getattr(module, class_name)
-        
+
         repr_to_store = repr_.copy()
         repr_.pop('type')
-        
+
         serializable = class_.deserialize(self, **repr_)
-        
+
         if 'identifier' in repr_:
             identifier = repr_['identifier']
             self.__subpulses[identifier] = self.__FileEntry(repr_, serializable)
@@ -824,3 +860,49 @@ def default(self, o: Any) -> Any:
             return list(o)
         else:
             return super().default(o)
+
+
+def convert_stored_pulse_in_storage(identifier: str, source_storage: StorageBackend, dest_storage: StorageBackend) -> None:
+    """Converts a pulse from the old to the new serialization format.
+
+    The pulse with the given identifier is completely (including subpulses) converted from the old serialization format
+    read from a given source storage to the new serialization format and written to a given destination storage.
+
+    Args:
+        identifier (str): The identifier of the pulse to convert.
+        source_storage (StorageBackend): A StorageBackend containing the pulse identified by the identifier argument in the old serialization format.
+        dest_storage (StorageBackend): A StorageBackend the converted pulse will be written to in the new serialization format.
+    Raises:
+        ValueError: if the dest_storage StorageBackend contains identifiers also assigned in source_storage.
+    """
+    if dest_storage.list_contents().intersection(source_storage.list_contents()):
+        raise ValueError("dest_storage already contains pulses with the same ids. Aborting to prevent inconsistencies for duplicate keys.")
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', DeprecationWarning)
+        serializer = Serializer(source_storage)
+        pulse_storage = PulseStorage(dest_storage)
+        serializable = serializer.deserialize(identifier)
+        pulse_storage.overwrite(identifier, serializable)
+
+
+def convert_pulses_in_storage(source_storage: StorageBackend, dest_storage: StorageBackend) -> None:
+    """Converts all pulses from the old to the new serialization format.
+
+        All pulses in a given source storage are completely (including subpulses) converted from the old serialization format
+        to the new serialization format and written to a given destination storage.
+
+        Args:
+            source_storage (StorageBackend): A StorageBackend containing pulses in the old serialization format.
+            dest_storage (StorageBackend): A StorageBackend the converted pulses will be written to in the new serialization format.
+        Raises:
+            ValueError: if the dest_storage StorageBackend contains identifiers also assigned in source_storage.
+        """
+    if dest_storage.list_contents().intersection(source_storage.list_contents()):
+        raise ValueError("dest_storage already contains pulses with the same ids. Aborting to prevent inconsistencies for duplicate keys.")
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', DeprecationWarning)
+        serializer = Serializer(source_storage)
+        pulse_storage = PulseStorage(dest_storage)
+        for identifier in source_storage.list_contents():
+            serializable = serializer.deserialize(identifier)
+            pulse_storage.overwrite(identifier, serializable)

tests/pulses/sequencing_dummies.py

@@ -365,6 +365,8 @@ def requires_stop(self, parameters: Dict[str, Parameter], conditions: Dict[str,
 
     def get_serialization_data(self, serializer: Optional['Serializer']=None) -> Dict[str, Any]:
         data = super().get_serialization_data(serializer=serializer)
+        if serializer: # compatibility with old serialization routines
+            data = dict()
         data['requires_stop'] = self.requires_stop_
         data['is_interruptable'] = self.is_interruptable
         data['parameter_names'] = self.parameter_names

tests/serialization_dummies.py

@@ -1,4 +1,4 @@
-from typing import Union, Dict, Any, Callable
+from typing import Union, Dict, Any, Callable, Set
 
 from qctoolkit.serialization import Serializer, Serializable, StorageBackend
 
@@ -27,9 +27,12 @@ def exists(self, identifier: str) -> bool:
         self.times_exists_called += 1
         return identifier in self.stored_items
 
-    def delete(self, identifier: str):
+    def delete(self, identifier: str) -> None:
         del self.stored_items[identifier]
 
+    def list_contents(self) -> Set[str]:
+        return set(self.stored_items.keys())
+
 
 class DummySerializer(Serializer):