From c97f5477fbb01b43eda6c4c76279e22de2d6ce45 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 25 Apr 2016 15:14:38 +0100 Subject: [PATCH 01/12] Add de/serializer providers to pootle.core --- pootle/core/delegate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pootle/core/delegate.py b/pootle/core/delegate.py index 03742e58484..89c9c665e4f 100644 --- a/pootle/core/delegate.py +++ b/pootle/core/delegate.py @@ -7,8 +7,11 @@ # AUTHORS file for copyright and authorship information. -from pootle.core.plugin.delegate import Getter +from pootle.core.plugin.delegate import Getter, Provider config = Getter(providing_args=["instance"]) search_backend = Getter(providing_args=["instance"]) + +serializers = Provider(providing_args=["instance"]) +deserializers = Provider(providing_args=["instance"]) From 236696640112276abbc0090a031d22e5ce93f1a2 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 25 Apr 2016 16:35:00 +0100 Subject: [PATCH 02/12] Add StoreSerialization class --- pootle/apps/pootle_store/store/__init__.py | 0 pootle/apps/pootle_store/store/serialize.py | 76 +++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 pootle/apps/pootle_store/store/__init__.py create mode 100644 pootle/apps/pootle_store/store/serialize.py diff --git a/pootle/apps/pootle_store/store/__init__.py b/pootle/apps/pootle_store/store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pootle/apps/pootle_store/store/serialize.py b/pootle/apps/pootle_store/store/serialize.py new file mode 100644 index 00000000000..656ca564c3b --- /dev/null +++ b/pootle/apps/pootle_store/store/serialize.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + +from django.core.cache import caches +from django.utils.functional import cached_property + +from pootle.core.delegate import config, serializers + + +class StoreSerialization(object): + """Calls configured deserializers for Store""" + + def __init__(self, store): + self.store = store + + @property + def project_serializers(self): + project = self.store.translation_project.project + return ( + config.get( + project.__class__, + instance=project, + key="pootle.core.serializers") + or []) + + @property + def pootle_path(self): + return self.store.pootle_path + + @cached_property + def max_unit_revision(self): + return self.store.get_max_unit_revision() + + @cached_property + def serializers(self): + available_serializers = serializers.gather( + self.store.__class__, instance=self.store) + found_serializers = [] + for serializer in self.project_serializers: + found_serializers.append(available_serializers[serializer]) + return found_serializers + + def tostring(self): + storeclass = self.store.get_file_class() + store = self.store.convert(storeclass) + if hasattr(store, "updateheader"): + # FIXME We need those headers on import + # However some formats just don't support setting metadata + store.updateheader(add=True, X_Pootle_Path=self.pootle_path) + store.updateheader(add=True, X_Pootle_Revision=self.max_unit_revision) + return str(store) + + def pipeline(self, data): + if not self.serializers: + return data + for serializer in self.serializers: + data = serializer(self, data).output + return data + + def serialize(self): + cache = caches["exports"] + ret = cache.get( + self.pootle_path, + version=self.max_unit_revision) + if not ret: + ret = self.pipeline(self.tostring()) + cache.set( + self.pootle_path, + ret, + version=self.max_unit_revision) + return ret From 09fa636f1055d96dccf18e3bdceffe6a9249f237 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 25 Apr 2016 18:03:49 +0100 Subject: [PATCH 03/12] Add list_serializers cli command to list available de/serializers --- .../management/commands/list_serializers.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 pootle/apps/pootle_app/management/commands/list_serializers.py diff --git a/pootle/apps/pootle_app/management/commands/list_serializers.py b/pootle/apps/pootle_app/management/commands/list_serializers.py new file mode 100644 index 00000000000..0e373bb9bb9 --- /dev/null +++ b/pootle/apps/pootle_app/management/commands/list_serializers.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + +import os + +os.environ['DJANGO_SETTINGS_MODULE'] = 'pootle.settings' + +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import BaseCommand, CommandError + +from pootle.core.delegate import serializers, deserializers + + +class Command(BaseCommand): + help = "Manage serialization for Projects." + + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser.add_argument( + '-m --model', + dest="model", + action="store", + help='List de/serializers for given model.') + parser.add_argument( + '-d --deserializers', + action="store_true", + dest="deserializers", + help='List deserializers') + + def model_from_content_type(self, ct_name): + if not ct_name: + return + if "." not in ct_name: + raise CommandError( + "Model name should be contenttype " + "$app_name.$label") + try: + return ContentType.objects.get_by_natural_key( + *ct_name.split(".")).model_class() + except ContentType.DoesNotExist as e: + raise CommandError(e) + + def handle(self, **kwargs): + model = self.model_from_content_type(kwargs["model"]) + if kwargs["deserializers"]: + return self.print_serializers_list( + deserializers.gather(model), + serializer_type="deserializers") + return self.print_serializers_list( + serializers.gather(model)) + + def print_serializers_list(self, serials, serializer_type="serializers"): + if not serials.keys(): + self.stdout.write( + "There are no %s set up on your system" % serializer_type) + if not serials.keys(): + return + heading = serializer_type.capitalize() + self.stdout.write("\n%s" % heading) + self.stdout.write("-" * len(heading)) + for name, serializer in serials.items(): + self.stdout.write( + "{: <30} {: <50} ".format(name, serializer)) From cfba944ca98b8e34b4723ba367f3575d05a16788 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 25 Apr 2016 18:39:31 +0100 Subject: [PATCH 04/12] Update Store to use de/serialization - Store.serialize uses StoreSerialization.serialize - Add deserialize method to Store --- pootle/apps/pootle_store/models.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/pootle/apps/pootle_store/models.py b/pootle/apps/pootle_store/models.py index 4d77f68e414..b434c60703a 100644 --- a/pootle/apps/pootle_store/models.py +++ b/pootle/apps/pootle_store/models.py @@ -52,6 +52,8 @@ from .fields import (PLURAL_PLACEHOLDER, SEPARATOR, MultiStringField, TranslationStoreField) from .filetypes import factory_classes +from .store.deserialize import StoreDeserialization +from .store.serialize import StoreSerialization from .util import ( FUZZY, OBSOLETE, TRANSLATED, UNTRANSLATED, get_change_str, vfolders_installed) @@ -1750,27 +1752,11 @@ def increment_unsynced_unit_revision(self, update_revision): return count - def serialize(self): - from django.core.cache import caches - - cache = caches["exports"] - rev = self.get_max_unit_revision() - path = self.pootle_path - - ret = cache.get(path, version=rev) - if not ret: - storeclass = self.get_file_class() - store = self.convert(storeclass) - if hasattr(store, "updateheader"): - # FIXME We need those headers on import - # However some formats just don't support setting metadata - store.updateheader(add=True, X_Pootle_Path=path) - store.updateheader(add=True, X_Pootle_Revision=rev) - - ret = str(store) - cache.set(path, ret, version=rev) + def deserialize(self, data): + return StoreDeserialization(self).deserialize(data) - return ret + def serialize(self): + return StoreSerialization(self).serialize() def sync(self, update_structure=False, conservative=True, user=None, skip_missing=False, only_newer=True): From c1f6d6a4804c43e8e69c8248265ac2a02bdc2fa3 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 25 Apr 2016 21:28:33 +0100 Subject: [PATCH 05/12] Add StoreDeserialization class --- pootle/apps/pootle_store/store/deserialize.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 pootle/apps/pootle_store/store/deserialize.py diff --git a/pootle/apps/pootle_store/store/deserialize.py b/pootle/apps/pootle_store/store/deserialize.py new file mode 100644 index 00000000000..963e989a2f1 --- /dev/null +++ b/pootle/apps/pootle_store/store/deserialize.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + +import io + +from translate.storage.factory import getclass + +from django.utils.functional import cached_property + +from pootle.core.delegate import config, deserializers + + +class StoreDeserialization(object): + """Calls configured deserializers for Store""" + + def __init__(self, store): + self.store = store + + @property + def project_deserializers(self): + project = self.store.translation_project.project + return ( + config.get( + project.__class__, + instance=project, + key="pootle.core.deserializers") + or []) + + @cached_property + def deserializers(self): + available_deserializers = deserializers.gather( + self.store.__class__, instance=self.store) + found_deserializers = [] + for deserializer in self.project_deserializers: + found_deserializers.append(available_deserializers[deserializer]) + return found_deserializers + + def pipeline(self, data): + if not self.deserializers: + return data + for deserializer in self.deserializers: + data = deserializer(self, data).output + return data + + def fromstring(self, data): + return getclass(io.BytesIO(data))(data) + + def deserialize(self, data): + return self.fromstring(self.pipeline(data)) From e228081dafb035916f35222aa035810a4cf12187 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 26 Apr 2016 06:36:34 +0100 Subject: [PATCH 06/12] Add tests for Store de/serialization --- tests/models/store.py | 197 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/tests/models/store.py b/tests/models/store.py index d278350b4e3..b879374328e 100644 --- a/tests/models/store.py +++ b/tests/models/store.py @@ -6,10 +6,13 @@ # or later license. See the LICENSE file for a copy of the license and the # AUTHORS file for copyright and authorship information. +import io import os import shutil import time +import six + import pytest import pytest_pootle @@ -19,7 +22,12 @@ from django.core.files.uploadedfile import SimpleUploadedFile +from pootle.core.delegate import config from pootle.core.models import Revision +from pootle.core.delegate import deserializers, serializers +from pootle.core.plugin import provider +from pootle.core.serializers import Serializer, Deserializer +from pootle_config.exceptions import ConfigurationError from pootle_statistics.models import SubmissionTypes from pootle_store.models import NEW, OBSOLETE, PARSED, POOTLE_WINS, Store from pootle_store.util import parse_pootle_revision @@ -510,3 +518,192 @@ def test_store_repr(): store = Store.objects.first() assert str(store) == str(store.convert(store.get_file_class())) assert repr(store) == u"" % store.pootle_path + + +@pytest.mark.django_db +def test_store_po_deserializer(test_fs, store_po): + + with test_fs.open("data/po/complex.po") as test_file: + test_string = test_file.read() + ttk_po = getclass(test_file)(test_string) + + store_po.update(store_po.deserialize(test_string)) + assert len(ttk_po.units) - 1 == store_po.units.count() + + +@pytest.mark.django_db +def test_store_po_serializer(test_fs, store_po): + + with test_fs.open("data/po/complex.po") as test_file: + test_string = test_file.read() + ttk_po = getclass(test_file)(test_string) + + store_po.update(store_po.deserialize(test_string)) + store_io = io.BytesIO(store_po.serialize()) + store_ttk = getclass(store_io)(store_io.read()) + assert len(store_ttk.units) == len(ttk_po.units) + + +@pytest.mark.django_db +def test_store_po_serializer_custom(test_fs, store_po): + + class SerializerCheck(object): + serializer_was_called = False + input_is_bytes = False + + checker = SerializerCheck() + + class EGSerializer(object): + + def __init__(self, store, data): + self.store = store + self.original_data = data + + @property + def output(self): + checker.serializer_was_called = True + checker.input_is_bytes = ( + not isinstance( + self.original_data, six.text_type) + and isinstance( + self.original_data, str)) + + @provider(serializers) + def provide_serializers(**kwargs): + return dict(eg_serializer=EGSerializer) + + with test_fs.open("data/po/complex.po") as test_file: + test_string = test_file.read() + # ttk_po = getclass(test_file)(test_string) + store_po.update(store_po.deserialize(test_string)) + + # add config to the project + project = store_po.translation_project.project + config.get(project.__class__, instance=project).set_config( + "pootle.core.serializers", + ["eg_serializer"]) + + store_po.serialize() + assert checker.serializer_was_called is True + assert checker.input_is_bytes is True + + +@pytest.mark.django_db +def test_store_po_deserializer_custom(test_fs, store_po): + + class DeserializerCheck(object): + deserializer_was_called = False + input_is_bytes = False + + checker = DeserializerCheck() + + class EGDeserializer(object): + + def __init__(self, store, data): + self.store = store + self.original_data = data + + @property + def output(self): + checker.deserializer_was_called = True + checker.input_is_bytes = ( + not isinstance( + self.original_data, six.text_type) + and isinstance( + self.original_data, str)) + return self.original_data + + @provider(deserializers) + def provide_deserializers(**kwargs): + return dict(eg_deserializer=EGDeserializer) + + with test_fs.open("data/po/complex.po") as test_file: + test_string = test_file.read() + + # add config to the project + project = store_po.translation_project.project + config.get().set_config( + "pootle.core.deserializers", + ["eg_deserializer"], + project) + + store_po.update(store_po.deserialize(test_string)) + + assert checker.deserializer_was_called is True + assert checker.input_is_bytes is True + + +@pytest.mark.django_db +def test_store_base_serializer(store_po): + original_data = "SOME DATA" + serializer = Serializer(store_po, original_data) + assert serializer.context == store_po + assert serializer.data == original_data + + +@pytest.mark.django_db +def test_store_base_deserializer(store_po): + original_data = "SOME DATA" + deserializer = Deserializer(store_po, original_data) + assert deserializer.context == store_po + assert deserializer.data == original_data + + +@pytest.mark.django_db +def test_store_set_bad_deserializers(store_po): + project = store_po.translation_project.project + with pytest.raises(ConfigurationError): + config.get(project.__class__, instance=project).set_config( + "pootle.core.deserializers", + ["DESERIALIZER_DOES_NOT_EXIST"]) + + class EGDeserializer(object): + pass + + @provider(deserializers) + def provide_deserializers(**kwargs): + return dict(eg_deserializer=EGDeserializer) + + # must be list + with pytest.raises(ConfigurationError): + config.get(project.__class__, instance=project).set_config( + "pootle.core.deserializers", + "eg_deserializer") + with pytest.raises(ConfigurationError): + config.get(project.__class__, instance=project).set_config( + "pootle.core.deserializers", + dict(serializer="eg_deserializer")) + + config.get(project.__class__, instance=project).set_config( + "pootle.core.deserializers", + ["eg_deserializer"]) + + +@pytest.mark.django_db +def test_store_set_bad_serializers(store_po): + project = store_po.translation_project.project + with pytest.raises(ConfigurationError): + config.get(project.__class__, instance=project).set_config( + "pootle.core.serializers", + ["SERIALIZER_DOES_NOT_EXIST"]) + + class EGSerializer(object): + pass + + @provider(serializers) + def provide_serializers(**kwargs): + return dict(eg_serializer=EGSerializer) + + # must be list + with pytest.raises(ConfigurationError): + config.get(project.__class__, instance=project).set_config( + "pootle.core.serializers", + "eg_serializer") + with pytest.raises(ConfigurationError): + config.get(project.__class__, instance=project).set_config( + "pootle.core.serializers", + dict(serializer="eg_serializer")) + + config.get(project.__class__, instance=project).set_config( + "pootle.core.serializers", + ["eg_serializer"]) From e40e19fe966b907850d0888badce954caeee1859 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sat, 30 Apr 2016 19:21:44 +0100 Subject: [PATCH 07/12] Add base de/serializers --- pootle/core/serializers.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 pootle/core/serializers.py diff --git a/pootle/core/serializers.py b/pootle/core/serializers.py new file mode 100644 index 00000000000..91d56506af8 --- /dev/null +++ b/pootle/core/serializers.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + + +class Serializer(object): + + def __init__(self, context, data): + self.context = context + self.original_data = data + + @property + def data(self): + return self.original_data + + +class Deserializer(object): + + def __init__(self, context, data): + self.context = context + self.original_data = data + + @property + def data(self): + return self.original_data From e1bb21e7476aabb29670a81fc74b2f4d3f0283f4 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 2 May 2016 21:46:20 +0100 Subject: [PATCH 08/12] Add validator getter for pootle.core.de/serializers config --- pootle/apps/pootle_store/getters.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pootle/apps/pootle_store/getters.py b/pootle/apps/pootle_store/getters.py index 12776bab1f9..29fdff3bde1 100644 --- a/pootle/apps/pootle_store/getters.py +++ b/pootle/apps/pootle_store/getters.py @@ -6,8 +6,12 @@ # or later license. See the LICENSE file for a copy of the license and the # AUTHORS file for copyright and authorship information. +from django.core.exceptions import ValidationError -from pootle.core.delegate import search_backend +from pootle_config.delegate import ( + config_should_not_be_set, config_should_not_be_appended) +from pootle.core.delegate import ( + deserializers, search_backend, serializers) from pootle.core.plugin import getter from .models import Unit @@ -17,3 +21,26 @@ @getter(search_backend, sender=Unit) def get_search_backend(**kwargs): return DBSearchBackend + + +@getter([config_should_not_be_set, config_should_not_be_appended]) +def serializer_should_not_be_saved(**kwargs): + + if kwargs["key"] == "pootle.core.serializers": + if not isinstance(kwargs["value"], list): + return ValidationError( + "pootle.core.serializers must be a list") + available_serializers = serializers.gather(kwargs["sender"]).keys() + for k in kwargs["value"]: + if k not in available_serializers: + return ValidationError( + "Unrecognised pootle.core.serializers: '%s'" % k) + elif kwargs["key"] == "pootle.core.deserializers": + if not isinstance(kwargs["value"], list): + return ValidationError( + "pootle.core.deserializers must be a list") + available_deserializers = deserializers.gather(kwargs["sender"]).keys() + for k in kwargs["value"]: + if k not in available_deserializers: + return ValidationError( + "Unrecognised pootle.core.deserializers: '%s'" % k) From 60e1f9c6f86d53f94f36da0d7021af22bc0b865a Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 2 May 2016 22:04:33 +0100 Subject: [PATCH 09/12] Add cli tests for list_serializers --- tests/commands/list_serializers.py | 281 +++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 tests/commands/list_serializers.py diff --git a/tests/commands/list_serializers.py b/tests/commands/list_serializers.py new file mode 100644 index 00000000000..3f46f964437 --- /dev/null +++ b/tests/commands/list_serializers.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) Pootle contributors. +# +# This file is a part of the Pootle project. It is distributed under the GPL3 +# or later license. See the LICENSE file for a copy of the license and the +# AUTHORS file for copyright and authorship information. + +from collections import OrderedDict + +import pytest + +from django.core.management import call_command +from django.core.management.base import CommandError + +from pootle.core.delegate import serializers, deserializers +from pootle.core.plugin import provider +from pootle_project.models import Project + + +class EGSerializer1(object): + pass + + +class EGSerializer2(object): + pass + + +class EGDeserializer1(object): + pass + + +class EGDeserializer2(object): + pass + + +def serializer_provider_factory(sender=None): + + @provider(serializers, sender=sender) + def serializer_provider(**kwargs): + return OrderedDict( + (("serializer1", EGSerializer1), + ("serializer2", EGSerializer2))) + return serializer_provider + + +def deserializer_provider_factory(sender=None): + + @provider(deserializers, sender=sender) + def deserializer_provider(**kwargs): + return OrderedDict( + (("deserializer1", EGDeserializer1), + ("deserializer2", EGDeserializer2))) + return deserializer_provider + + +def _test_serializer_list(out, err, model=None): + NO_SERIALS = "There are no serializers set up on your system" + + serials = serializers.gather(model) + + expected = [] + + if not serials.keys(): + expected.append(NO_SERIALS) + + if serials.keys(): + heading = "Serializers" + expected.append("\n%s" % heading) + expected.append("-" * len(heading)) + for name, klass in serials.items(): + expected.append("{: <30} {: <50} ".format(name, klass)) + assert out == "%s\n" % ("\n".join(expected)) + + +def _test_deserializer_list(out, err, model=None): + NO_DESERIALS = "There are no deserializers set up on your system" + + deserials = deserializers.gather(model) + + expected = [] + + if not deserials.keys(): + expected.append(NO_DESERIALS) + + if deserials.keys(): + heading = "Deserializers" + expected.append("\n%s" % heading) + expected.append("-" * len(heading)) + for name, klass in deserials.items(): + expected.append("{: <30} {: <50} ".format(name, klass)) + assert out == "%s\n" % ("\n".join(expected)) + + +@pytest.mark.cmd +@pytest.mark.django_db +def test_cmd_list_serializers(capsys): + + # tests with no de/serializers set up + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + # set up some serializers + serial_provider = serializer_provider_factory() + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + serializers.disconnect(serial_provider) + + # set up some deserializers + deserial_provider = deserializer_provider_factory() + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + # empty again + deserializers.disconnect(deserial_provider) + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + # with both set up + deserial_provider = deserializer_provider_factory() + serial_provider = serializer_provider_factory() + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + serializers.disconnect(serial_provider) + deserializers.disconnect(deserial_provider) + + +@pytest.mark.cmd +@pytest.mark.django_db +def test_cmd_list_serializers_for_model(capsys): + + # tests with no de/serializers set up + call_command("list_serializers", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_serializer_list(out, err, Project) + + # set up some serializers + serial_provider = serializer_provider_factory(Project) + call_command("list_serializers", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_serializer_list(out, err, Project) + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + serializers.disconnect(serial_provider) + + # set up some deserializers + deserial_provider = deserializer_provider_factory(Project) + call_command("list_serializers", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_serializer_list(out, err, Project) + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + # empty again + deserializers.disconnect(deserial_provider) + call_command("list_serializers", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_serializer_list(out, err, Project) + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + + # with both set up + deserial_provider = deserializer_provider_factory(Project) + serial_provider = serializer_provider_factory(Project) + call_command("list_serializers", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_serializer_list(out, err, Project) + call_command("list_serializers") + out, err = capsys.readouterr() + _test_serializer_list(out, err) + serializers.disconnect(serial_provider) + deserializers.disconnect(deserial_provider) + + +@pytest.mark.cmd +@pytest.mark.django_db +def test_cmd_list_deserializers(capsys): + + # tests with no de/deserializers set up + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + # set up some deserializers + deserial_provider = deserializer_provider_factory() + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + deserializers.disconnect(deserial_provider) + + # set up some deserializers + deserial_provider = deserializer_provider_factory() + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + # empty again + deserializers.disconnect(deserial_provider) + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + # with both set up + deserial_provider = deserializer_provider_factory() + deserial_provider = deserializer_provider_factory() + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + deserializers.disconnect(deserial_provider) + deserializers.disconnect(deserial_provider) + + +@pytest.mark.cmd +@pytest.mark.django_db +def test_cmd_list_deserializers_for_model(capsys): + + # tests with no de/deserializers set up + call_command("list_serializers", "-d", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_deserializer_list(out, err, Project) + + # set up some deserializers + deserial_provider = deserializer_provider_factory(Project) + call_command("list_serializers", "-d", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_deserializer_list(out, err, Project) + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + deserializers.disconnect(deserial_provider) + + # set up some deserializers + deserial_provider = deserializer_provider_factory(Project) + call_command("list_serializers", "-d", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_deserializer_list(out, err, Project) + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + # empty again + deserializers.disconnect(deserial_provider) + call_command("list_serializers", "-d", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_deserializer_list(out, err, Project) + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + + # with both set up + deserial_provider = deserializer_provider_factory(Project) + deserial_provider = deserializer_provider_factory(Project) + call_command("list_serializers", "-d", "-m", "pootle_project.project") + out, err = capsys.readouterr() + _test_deserializer_list(out, err, Project) + call_command("list_serializers", "-d") + out, err = capsys.readouterr() + _test_deserializer_list(out, err) + deserializers.disconnect(deserial_provider) + deserializers.disconnect(deserial_provider) + + +@pytest.mark.cmd +@pytest.mark.django_db +def test_cmd_list_serializers_bad_models(capsys): + with pytest.raises(CommandError): + call_command("list_serializers", "-m", "foo") + with pytest.raises(CommandError): + call_command("list_serializers", "-m", "foo.bar") From 27c41a7c204a8113faf69b4d0442fdff524b7711 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 2 May 2016 22:22:15 +0100 Subject: [PATCH 10/12] Add pootle_store.getters to isortof --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 50fc298a8c1..6880d2115bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -107,7 +107,8 @@ skip=.tox, pootle/core/views.py, pootle/middleware/errorpages.py, pootle/apps/pootle_comment/forms.py, - pootle/apps/pootle_config/managers.py + pootle/apps/pootle_config/managers.py, + pootle/apps/pootle_store/getters.py known_standard_library= known_third_party=elasticsearch,translate From bc994e8898835777d2009d3fb425885800b16bc1 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Wed, 4 May 2016 18:58:46 +0100 Subject: [PATCH 11/12] Add command doc for list_serializers command --- docs/server/commands.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/server/commands.rst b/docs/server/commands.rst index cae845729c5..e9d5b69d65c 100644 --- a/docs/server/commands.rst +++ b/docs/server/commands.rst @@ -316,6 +316,27 @@ directly on the file system. deleted from the database. Handle with care! +.. django-admin:: list_serializers + +list_serializers +^^^^^^^^^^^^^^^^ + + .. versionadded:: 2.8.0 + +List the installed serializers and deserializers on your system. + +Available options: + +.. django-admin-option:: -m, --model + +List serializers for specified model. The model should be expressed as a +contenttype label - eg ``app_name``.``model_name`` + +.. django-admin-option:: -d, --deserializers + +List available deserializers set up for our system. + + .. django-admin:: list_languages list_languages From 3e3ea960b95bec5440b24615069bb6935d215dac Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 6 May 2016 11:32:54 +0100 Subject: [PATCH 12/12] Add list_serializers to release notes --- docs/releases/dev.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/releases/dev.rst b/docs/releases/dev.rst index a470c7d20ce..3ee34999a8d 100644 --- a/docs/releases/dev.rst +++ b/docs/releases/dev.rst @@ -21,6 +21,8 @@ Details of changes - :djadmin:`refresh_scores` now recalculates user scores and accepts multiple usernames. - :djadmin:`contributors` has two new options +- :djadmin:`list_serializers` has been added to view serializers and + deserializers installed on your system. :option:`--since ` and :option:`--only-emails `. - :djadmin:`flush_cache` flushes ``default``, ``redis``, ``stats`` caches,