Store serializers

docs/releases/dev.rst

@@ -21,6 +21,8 @@ Details of changes
 - :djadmin:`refresh_scores` now recalculates user scores and accepts
   multiple usernames.
 - :djadmin:`contributors` has two new options
+- :djadmin:`list_serializers` has been added to view serializers and
+  deserializers installed on your system.
   :option:`--since <contributors --since>` and
   :option:`--only-emails <contributors --only-emails>`.
 - :djadmin:`flush_cache` flushes ``default``, ``redis``, ``stats`` caches,

docs/server/commands.rst

@@ -316,6 +316,27 @@ directly on the file system.
    deleted from the database. Handle with care!
 
 
+.. django-admin:: list_serializers
+
+list_serializers
+^^^^^^^^^^^^^^^^
+
+  .. versionadded:: 2.8.0
+
+List the installed serializers and deserializers on your system.
+
+Available options:
+
+.. django-admin-option:: -m, --model
+
+List serializers for specified model. The model should be expressed as a
+contenttype label - eg ``app_name``.``model_name``
+
+.. django-admin-option:: -d, --deserializers
+
+List available deserializers set up for our system.
+
+
 .. django-admin:: list_languages
 
 list_languages

pootle/apps/pootle_app/management/commands/list_serializers.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) Pootle contributors.
+#
+# This file is a part of the Pootle project. It is distributed under the GPL3
+# or later license. See the LICENSE file for a copy of the license and the
+# AUTHORS file for copyright and authorship information.
+
+import os
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings'
+
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import BaseCommand, CommandError
+
+from pootle.core.delegate import serializers, deserializers
+
+
+class Command(BaseCommand):
+    help = "Manage serialization for Projects."
+
+    def add_arguments(self, parser):
+        super(Command, self).add_arguments(parser)
+        parser.add_argument(
+            '-m --model',
+            dest="model",
+            action="store",
+            help='List de/serializers for given model.')
+        parser.add_argument(
+            '-d --deserializers',
+            action="store_true",
+            dest="deserializers",
+            help='List deserializers')
+
+    def model_from_content_type(self, ct_name):
+        if not ct_name:
+            return
+        if "." not in ct_name:
+            raise CommandError(
+                "Model name should be contenttype "
+                "$app_name.$label")
+        try:
+            return ContentType.objects.get_by_natural_key(
+                *ct_name.split(".")).model_class()
+        except ContentType.DoesNotExist as e:
+            raise CommandError(e)
+
+    def handle(self, **kwargs):
+        model = self.model_from_content_type(kwargs["model"])
+        if kwargs["deserializers"]:
+            return self.print_serializers_list(
+                deserializers.gather(model),
+                serializer_type="deserializers")
+        return self.print_serializers_list(
+            serializers.gather(model))
+
+    def print_serializers_list(self, serials, serializer_type="serializers"):
+        if not serials.keys():
+            self.stdout.write(
+                "There are no %s set up on your system" % serializer_type)
+        if not serials.keys():
+            return
+        heading = serializer_type.capitalize()
+        self.stdout.write("\n%s" % heading)
+        self.stdout.write("-" * len(heading))
+        for name, serializer in serials.items():
+            self.stdout.write(
+                "{: <30} {: <50} ".format(name, serializer))

pootle/apps/pootle_store/getters.py

@@ -6,8 +6,12 @@
 # or later license. See the LICENSE file for a copy of the license and the
 # AUTHORS file for copyright and authorship information.
 
+from django.core.exceptions import ValidationError
 
-from pootle.core.delegate import search_backend
+from pootle_config.delegate import (
+    config_should_not_be_set, config_should_not_be_appended)
+from pootle.core.delegate import (
+    deserializers, search_backend, serializers)
 from pootle.core.plugin import getter
 
 from .models import Unit
@@ -17,3 +21,26 @@
 @getter(search_backend, sender=Unit)
 def get_search_backend(**kwargs):
     return DBSearchBackend
+
+
+@getter([config_should_not_be_set, config_should_not_be_appended])
+def serializer_should_not_be_saved(**kwargs):
+
+    if kwargs["key"] == "pootle.core.serializers":
+        if not isinstance(kwargs["value"], list):
+            return ValidationError(
+                "pootle.core.serializers must be a list")
+        available_serializers = serializers.gather(kwargs["sender"]).keys()
+        for k in kwargs["value"]:
+            if k not in available_serializers:
+                return ValidationError(
+                    "Unrecognised pootle.core.serializers: '%s'" % k)
+    elif kwargs["key"] == "pootle.core.deserializers":
+        if not isinstance(kwargs["value"], list):
+            return ValidationError(
+                "pootle.core.deserializers must be a list")
+        available_deserializers = deserializers.gather(kwargs["sender"]).keys()
+        for k in kwargs["value"]:
+            if k not in available_deserializers:
+                return ValidationError(
+                    "Unrecognised pootle.core.deserializers: '%s'" % k)

pootle/apps/pootle_store/models.py

@@ -52,6 +52,8 @@
 from .fields import (PLURAL_PLACEHOLDER, SEPARATOR, MultiStringField,
                      TranslationStoreField)
 from .filetypes import factory_classes
+from .store.deserialize import StoreDeserialization
+from .store.serialize import StoreSerialization
 from .util import (
     FUZZY, OBSOLETE, TRANSLATED, UNTRANSLATED, get_change_str,
     vfolders_installed)
@@ -1750,27 +1752,11 @@ def increment_unsynced_unit_revision(self, update_revision):
 
         return count
 
-    def serialize(self):
-        from django.core.cache import caches
-
-        cache = caches["exports"]
-        rev = self.get_max_unit_revision()
-        path = self.pootle_path
-
-        ret = cache.get(path, version=rev)
-        if not ret:
-            storeclass = self.get_file_class()
-            store = self.convert(storeclass)
-            if hasattr(store, "updateheader"):
-                # FIXME We need those headers on import
-                # However some formats just don't support setting metadata
-                store.updateheader(add=True, X_Pootle_Path=path)
-                store.updateheader(add=True, X_Pootle_Revision=rev)
-
-            ret = str(store)
-            cache.set(path, ret, version=rev)
+    def deserialize(self, data):
+        return StoreDeserialization(self).deserialize(data)
 
-        return ret
+    def serialize(self):
+        return StoreSerialization(self).serialize()
 
     def sync(self, update_structure=False, conservative=True,
              user=None, skip_missing=False, only_newer=True):

pootle/apps/pootle_store/store/deserialize.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) Pootle contributors.
+#
+# This file is a part of the Pootle project. It is distributed under the GPL3
+# or later license. See the LICENSE file for a copy of the license and the
+# AUTHORS file for copyright and authorship information.
+
+import io
+
+from translate.storage.factory import getclass
+
+from django.utils.functional import cached_property
+
+from pootle.core.delegate import config, deserializers
+
+
+class StoreDeserialization(object):
+    """Calls configured deserializers for Store"""
+
+    def __init__(self, store):
+        self.store = store
+
+    @property
+    def project_deserializers(self):
+        project = self.store.translation_project.project
+        return (
+            config.get(
+                project.__class__,
+                instance=project,
+                key="pootle.core.deserializers")
+            or [])
+
+    @cached_property
+    def deserializers(self):
+        available_deserializers = deserializers.gather(
+            self.store.__class__, instance=self.store)
+        found_deserializers = []
+        for deserializer in self.project_deserializers:
+            found_deserializers.append(available_deserializers[deserializer])
+        return found_deserializers
+
+    def pipeline(self, data):
+        if not self.deserializers:
+            return data
+        for deserializer in self.deserializers:
+            data = deserializer(self, data).output
+        return data
+
+    def fromstring(self, data):
+        return getclass(io.BytesIO(data))(data)
+
+    def deserialize(self, data):
+        return self.fromstring(self.pipeline(data))

pootle/apps/pootle_store/store/serialize.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) Pootle contributors.
+#
+# This file is a part of the Pootle project. It is distributed under the GPL3
+# or later license. See the LICENSE file for a copy of the license and the
+# AUTHORS file for copyright and authorship information.
+
+from django.core.cache import caches
+from django.utils.functional import cached_property
+
+from pootle.core.delegate import config, serializers
+
+
+class StoreSerialization(object):
+    """Calls configured deserializers for Store"""
+
+    def __init__(self, store):
+        self.store = store
+
+    @property
+    def project_serializers(self):
+        project = self.store.translation_project.project
+        return (
+            config.get(
+                project.__class__,
+                instance=project,
+                key="pootle.core.serializers")
+            or [])
+
+    @property
+    def pootle_path(self):
+        return self.store.pootle_path
+
+    @cached_property
+    def max_unit_revision(self):
+        return self.store.get_max_unit_revision()
+
+    @cached_property
+    def serializers(self):
+        available_serializers = serializers.gather(
+            self.store.__class__, instance=self.store)
+        found_serializers = []
+        for serializer in self.project_serializers:
+            found_serializers.append(available_serializers[serializer])
+        return found_serializers
+
+    def tostring(self):
+        storeclass = self.store.get_file_class()
+        store = self.store.convert(storeclass)
+        if hasattr(store, "updateheader"):
+            # FIXME We need those headers on import
+            # However some formats just don't support setting metadata
+            store.updateheader(add=True, X_Pootle_Path=self.pootle_path)
+            store.updateheader(add=True, X_Pootle_Revision=self.max_unit_revision)
+        return str(store)
+
+    def pipeline(self, data):
+        if not self.serializers:
+            return data
+        for serializer in self.serializers:
+            data = serializer(self, data).output
+        return data
+
+    def serialize(self):
+        cache = caches["exports"]
+        ret = cache.get(
+            self.pootle_path,
+            version=self.max_unit_revision)
+        if not ret:
+            ret = self.pipeline(self.tostring())
+            cache.set(
+                self.pootle_path,
+                ret,
+                version=self.max_unit_revision)
+        return ret

pootle/core/delegate.py

@@ -7,8 +7,11 @@
 # AUTHORS file for copyright and authorship information.
 
 
-from pootle.core.plugin.delegate import Getter
+from pootle.core.plugin.delegate import Getter, Provider
 
 
 config = Getter(providing_args=["instance"])
 search_backend = Getter(providing_args=["instance"])
+
+serializers = Provider(providing_args=["instance"])
+deserializers = Provider(providing_args=["instance"])

pootle/core/serializers.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) Pootle contributors.
+#
+# This file is a part of the Pootle project. It is distributed under the GPL3
+# or later license. See the LICENSE file for a copy of the license and the
+# AUTHORS file for copyright and authorship information.
+
+
+class Serializer(object):
+
+    def __init__(self, context, data):
+        self.context = context
+        self.original_data = data
+
+    @property
+    def data(self):
+        return self.original_data
+
+
+class Deserializer(object):
+
+    def __init__(self, context, data):
+        self.context = context
+        self.original_data = data
+
+    @property
+    def data(self):
+        return self.original_data

setup.cfg

@@ -107,7 +107,8 @@ skip=.tox,
      pootle/core/views.py,
      pootle/middleware/errorpages.py,
      pootle/apps/pootle_comment/forms.py,
-     pootle/apps/pootle_config/managers.py
+     pootle/apps/pootle_config/managers.py,
+     pootle/apps/pootle_store/getters.py
 
 known_standard_library=
 known_third_party=elasticsearch,translate