diff --git a/wagtail_localize/management/__init__.py b/wagtail_localize/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wagtail_localize/management/commands/__init__.py b/wagtail_localize/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wagtail_localize/management/commands/sync_locale_trees.py b/wagtail_localize/management/commands/sync_locale_trees.py new file mode 100644 index 00000000..cbb95262 --- /dev/null +++ b/wagtail_localize/management/commands/sync_locale_trees.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from wagtail_localize.models import LocaleSynchronization +from wagtail_localize.synctree import PageIndex + + +class Command(BaseCommand): + help = "Synchronises the structure of all locale page trees so they contain the same pages. Creates alias pages where necessary." + + def handle(self, **options): + page_index = PageIndex.from_database().sort_by_tree_position() + for locale_sync in LocaleSynchronization.objects.all(): + locale_sync.sync_trees(page_index=page_index) diff --git a/wagtail_localize/migrations/0012_localesynchronization.py b/wagtail_localize/migrations/0012_localesynchronization.py new file mode 100644 index 00000000..388e9bfd --- /dev/null +++ b/wagtail_localize/migrations/0012_localesynchronization.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.3 on 2020-11-04 20:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('wagtailcore', '0059_apply_collection_ordering'), + ('wagtail_localize', '0011_segmentoverride'), + ] + + operations = [ + migrations.CreateModel( + name='LocaleSynchronization', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('locale', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='wagtailcore.locale')), + ('sync_from', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='wagtailcore.locale')), + ], + ), + ] diff --git a/wagtail_localize/models.py b/wagtail_localize/models.py index fa3fb40f..ec679968 100644 --- a/wagtail_localize/models.py +++ b/wagtail_localize/models.py @@ -20,7 +20,8 @@ OuterRef, Q ) -from django.db.models.signals import post_delete +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver from django.utils import timezone from django.utils.encoding import force_text from django.utils.text import capfirst, slugify @@ -34,6 +35,7 @@ from wagtail.core.utils import find_available_slug from .fields import copy_synchronised_fields +from .locales.components import register_locale_component from .segments import StringSegmentValue, TemplateSegmentValue, RelatedObjectSegmentValue, OverridableSegmentValue from .segments.extract import extract_segments from .segments.ingest import ingest_segments @@ -500,6 +502,10 @@ def create_or_update_translation(self, locale, user=None, publish=True, copy_par ingest_segments(original, translation, self.locale, locale, segments) if isinstance(translation, Page): + # Convert the page into a regular page + # TODO: Audit logging, etc + translation.alias_of_id = None + # Make sure the slug is valid translation.slug = find_available_slug(translation.get_parent(), slugify(translation.slug), ignore_page_id=translation.id) translation.save() @@ -1345,3 +1351,18 @@ def disable_translation_on_delete(instance, **kwargs): def register_post_delete_signal_handlers(): for model in get_translatable_models(): post_delete.connect(disable_translation_on_delete, sender=model) + + +@register_locale_component +class LocaleSynchronization(models.Model): + locale = models.OneToOneField('wagtailcore.Locale', on_delete=models.CASCADE, related_name='+') + sync_from = models.ForeignKey('wagtailcore.Locale', on_delete=models.CASCADE, related_name='+') + + def sync_trees(self, *, page_index=None): + from .synctree import synchronize_tree + synchronize_tree(self.sync_from, self.locale, page_index=page_index) + + +@receiver(post_save, sender=LocaleSynchronization) +def sync_trees_on_locale_sync_save(instance, **kwargs): + instance.sync_trees() diff --git a/wagtail_localize/synctree.py b/wagtail_localize/synctree.py new file mode 100644 index 00000000..8d3ad7c0 --- /dev/null +++ b/wagtail_localize/synctree.py @@ -0,0 +1,214 @@ +from collections import defaultdict + +from django.db.models.signals import post_save +from django.dispatch import receiver +from django.utils.functional import cached_property +from wagtail.core.models import Page, Locale + + +class PageIndex: + """ + An in-memory index of pages to remove the need to query the database. + + Each entry in the index is a unique page by translation key, so a page + that has been translated into different languages appears only once. + """ + + # Note: This has been designed to be as memory-efficient as possible, but it + # hasn't been tested on a very large site yet. + + class Entry: + """ + Represents a page in the index. + """ + + __slots__ = [ + "content_type", + "translation_key", + "source_locale", + "parent_translation_key", + "locales", + "aliased_locales", + ] + + def __init__( + self, + content_type, + translation_key, + source_locale, + parent_translation_key, + locales, + aliased_locales, + ): + self.content_type = content_type + self.translation_key = translation_key + self.source_locale = source_locale + self.parent_translation_key = parent_translation_key + self.locales = locales + self.aliased_locales = aliased_locales + + REQUIRED_PAGE_FIELDS = [ + "content_type", + "translation_key", + "locale", + "path", + "depth", + "last_published_at", + "latest_revision_created_at", + "live", + ] + + @classmethod + def from_page_instance(cls, page): + """ + Initialises an Entry from the given page instance. + """ + # Get parent, but only if the parent is not the root page. We consider the + # homepage of each langauge tree to be the roots + if page.depth > 2: + parent_page = page.get_parent() + else: + parent_page = None + + return cls( + page.content_type, + page.translation_key, + page.locale, + parent_page.translation_key if parent_page else None, + list( + Page.objects.filter( + translation_key=page.translation_key, + alias_of__isnull=True, + ).values_list("locale", flat=True) + ), + list( + Page.objects.filter( + translation_key=page.translation_key, + alias_of__isnull=False, + ).values_list("locale", flat=True) + ), + ) + + def __init__(self, pages): + self.pages = pages + + @cached_property + def by_translation_key(self): + return {page.translation_key: page for page in self.pages} + + @cached_property + def by_parent_translation_key(self): + by_parent_translation_key = defaultdict(list) + for page in self.pages: + by_parent_translation_key[page.parent_translation_key].append(page) + + return dict(by_parent_translation_key.items()) + + def sort_by_tree_position(self): + """ + Returns a new index with the pages sorted in depth-first-search order + using their parent in their respective source locale. + """ + remaining_pages = set(page.translation_key for page in self.pages) + + new_pages = [] + + def _walk(translation_key): + for page in self.by_parent_translation_key.get(translation_key, []): + if page.translation_key not in remaining_pages: + continue + + remaining_pages.remove(page.translation_key) + new_pages.append(page) + _walk(page.translation_key) + + _walk(None) + + if remaining_pages: + print("Warning: {} orphaned pages!".format(len(remaining_pages))) + + return PageIndex(new_pages) + + def not_translated_into(self, locale): + """ + Returns an index of pages that are not translated into the specified locale. + This includes pages that have and don't have a placeholder + """ + pages = [page for page in self.pages if locale.id not in page.locales] + + return PageIndex(pages) + + def __iter__(self): + return iter(self.pages) + + @classmethod + def from_database(cls): + """ + Populates the index from the database. + """ + pages = [] + + for page in Page.objects.filter(alias_of__isnull=True, depth__gt=1).only( + *PageIndex.Entry.REQUIRED_PAGE_FIELDS + ): + pages.append(PageIndex.Entry.from_page_instance(page)) + + return PageIndex(pages) + + +def synchronize_tree(source_locale, target_locale, *, page_index=None): + """ + Synchronises a locale tree with the other locales. + + This creates any placeholders that don't exist yet, updates placeholders where their + source has been changed and moves pages to match the structure of other trees + """ + # Build a page index + if not page_index: + page_index = PageIndex.from_database().sort_by_tree_position() + + # Find pages that are not translated for this locale + # This includes locales that have a placeholder, it only excludes locales that have an actual translation + pages_not_in_locale = page_index.not_translated_into(target_locale) + + for page in pages_not_in_locale: + # Skip pages that do not exist in the source + if source_locale.id not in page.locales and source_locale.id not in page.aliased_locales: + continue + + # Fetch source from database + model = page.content_type.model_class() + source_page = model.objects.get( + translation_key=page.translation_key, locale=source_locale + ) + + if target_locale.id not in page.aliased_locales: + source_page.copy_for_translation( + target_locale, copy_parents=True, alias=True + ) + + +@receiver(post_save) +def on_page_saved(sender, instance, **kwargs): + if not issubclass(sender, Page): + return + + # We only care about creations + if not kwargs['created']: + return + + # Check if the source tree needs to be synchronised into any other trees + from .models import LocaleSynchronization + locales_to_sync_to = Locale.objects.filter( + id__in=( + LocaleSynchronization.objects + .filter(sync_from_id=instance.locale_id) + .values_list("locale_id", flat=True) + ) + ) + + # Create aliases in all those locales + for locale in locales_to_sync_to: + instance.copy_for_translation( + locale, copy_parents=True, alias=True + ) diff --git a/wagtail_localize/templates/wagtail_localize/admin/edit_translatable_alias.html b/wagtail_localize/templates/wagtail_localize/admin/edit_translatable_alias.html new file mode 100644 index 00000000..5d0fe639 --- /dev/null +++ b/wagtail_localize/templates/wagtail_localize/admin/edit_translatable_alias.html @@ -0,0 +1,17 @@ +{% extends "wagtailadmin/pages/edit.html" %} +{% load i18n %} + +{% block form %} +
+ {% url 'wagtailadmin_pages:edit' page.alias_of_id as edit_original_page_url %} +

+ {% blocktrans with edit_original_page_url=edit_original_page_url original_locale=page.alias_of.locale.get_display_name %} + This page hasn't been translated yet. It is mirroring the {{ original_locale }} page.. + {% endblocktrans %} +

+ +

+ {% trans "Translate this page" %} +

+
+{% endblock %} diff --git a/wagtail_localize/test/settings.py b/wagtail_localize/test/settings.py index c05882ea..df88bd04 100644 --- a/wagtail_localize/test/settings.py +++ b/wagtail_localize/test/settings.py @@ -164,7 +164,7 @@ WAGTAIL_I18N_ENABLED = True -LANGUAGES = WAGTAIL_CONTENT_LANGUAGES = [("en", "English"), ("fr", "French"), ("es", "Spanish")] +LANGUAGES = WAGTAIL_CONTENT_LANGUAGES = [("en", "English"), ("fr", "French"), ("fr-CA", "French (Canada)"), ("es", "Spanish")] WAGTAILLOCALIZE_MACHINE_TRANSLATOR = { 'CLASS': 'wagtail_localize.machine_translators.dummy.DummyTranslator', diff --git a/wagtail_localize/tests/test_synctree.py b/wagtail_localize/tests/test_synctree.py new file mode 100644 index 00000000..88889801 --- /dev/null +++ b/wagtail_localize/tests/test_synctree.py @@ -0,0 +1,177 @@ +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase +from wagtail.core.models import Locale, Page + +from wagtail_localize.models import LocaleSynchronization +from wagtail_localize.synctree import PageIndex +from wagtail_localize.test.models import TestPage, TestHomePage + + +class TestPageIndex(TestCase): + def setUp(self): + self.en_locale = Locale.objects.get(language_code="en") + self.fr_locale = Locale.objects.create(language_code="fr") + self.fr_ca_locale = Locale.objects.create(language_code="fr-CA") + self.es_locale = Locale.objects.create(language_code="es") + + root_page = Page.objects.get(id=1) + root_page.get_children().delete() + root_page.refresh_from_db() + self.en_homepage = root_page.add_child(instance=TestHomePage(title="Home")) + + def test_from_database(self): + fr_homepage = self.en_homepage.copy_for_translation(self.fr_locale) + fr_ca_homepage = fr_homepage.copy_for_translation(self.fr_ca_locale) + + en_aboutpage = self.en_homepage.add_child( + instance=TestPage(title="About", slug="about",) + ) + + fr_aboutpage = en_aboutpage.copy_for_translation(self.fr_locale) + fr_aboutpage.copy_for_translation( + self.fr_ca_locale, alias=True + ) + + fr_ca_homepage.refresh_from_db() + fr_ca_canadaonlypage = fr_ca_homepage.add_child( + instance=TestPage( + title="Only Canada", slug="only-canada", locale=self.fr_ca_locale, + ) + ) + + # Create an index and sort it by tree position + page_index = PageIndex.from_database().sort_by_tree_position() + + # Homepage should be first + homepage_entry = page_index.pages[0] + self.assertEqual( + homepage_entry.content_type, ContentType.objects.get_for_model(TestHomePage) + ) + self.assertEqual(homepage_entry.translation_key, self.en_homepage.translation_key) + self.assertEqual(homepage_entry.source_locale, self.en_locale) + self.assertIsNone(homepage_entry.parent_translation_key) + self.assertEqual( + homepage_entry.locales, + [self.en_locale.id, self.fr_locale.id, self.fr_ca_locale.id], + ) + self.assertEqual(homepage_entry.aliased_locales, []) + + aboutpage_entry = page_index.pages[1] + self.assertEqual( + aboutpage_entry.content_type, ContentType.objects.get_for_model(TestPage) + ) + self.assertEqual(aboutpage_entry.translation_key, en_aboutpage.translation_key) + self.assertEqual(aboutpage_entry.source_locale, self.en_locale) + self.assertEqual( + aboutpage_entry.parent_translation_key, homepage_entry.translation_key + ) + self.assertEqual( + aboutpage_entry.locales, [self.en_locale.id, self.fr_locale.id] + ) + self.assertEqual( + aboutpage_entry.aliased_locales, [self.fr_ca_locale.id] + ) + + canadaonlypage_entry = page_index.pages[2] + self.assertEqual( + canadaonlypage_entry.content_type, + ContentType.objects.get_for_model(TestPage), + ) + self.assertEqual( + canadaonlypage_entry.translation_key, fr_ca_canadaonlypage.translation_key + ) + self.assertEqual(canadaonlypage_entry.source_locale, self.fr_ca_locale) + self.assertEqual( + canadaonlypage_entry.parent_translation_key, homepage_entry.translation_key + ) + self.assertEqual(canadaonlypage_entry.locales, [self.fr_ca_locale.id]) + self.assertEqual(canadaonlypage_entry.aliased_locales, []) + + +class TestSignals(TestCase): + def setUp(self): + self.en_locale = Locale.objects.get(language_code="en") + self.fr_locale = Locale.objects.create(language_code="fr") + self.fr_ca_locale = Locale.objects.create(language_code="fr-CA") + self.es_locale = Locale.objects.create(language_code="es") + + root_page = Page.objects.get(id=1) + root_page.get_children().delete() + root_page.refresh_from_db() + + self.en_homepage = root_page.add_child(instance=TestHomePage(title="Home")) + self.fr_homepage = self.en_homepage.copy_for_translation(self.fr_locale) + self.fr_ca_homepage = self.fr_homepage.copy_for_translation(self.fr_ca_locale) + + self.en_aboutpage = self.en_homepage.add_child( + instance=TestPage(title="About", slug="about",) + ) + + self.fr_aboutpage = self.en_aboutpage.copy_for_translation(self.fr_locale) + + self.fr_ca_homepage.refresh_from_db() + self.fr_ca_canadaonlypage = self.fr_ca_homepage.add_child( + instance=TestPage( + title="Only Canada", slug="only-canada", locale=self.fr_ca_locale, + ) + ) + + LocaleSynchronization.objects.create( + locale=self.fr_locale, + sync_from=self.en_locale, + ) + + LocaleSynchronization.objects.create( + locale=self.fr_ca_locale, + sync_from=self.fr_locale, + ) + + def test_create_new_page(self): + LocaleSynchronization.objects.create( + locale=self.es_locale, + sync_from=self.en_locale, + ) + + new_page = self.en_homepage.add_child( + instance=TestPage(title="Foo", slug="foo") + ) + + # Check it created aliases for the other locales + fr_new_page = TestPage.objects.get( + translation_key=new_page.translation_key, + locale=self.fr_locale, + alias_of=new_page, + ) + self.assertFalse(fr_new_page.revisions.exists()) + + fr_ca_new_page = TestPage.objects.get( + translation_key=new_page.translation_key, + locale=self.fr_ca_locale, + alias_of=fr_new_page, + ) + self.assertFalse(fr_ca_new_page.revisions.exists()) + + # Should've also created parent for this one + es_new_page = TestPage.objects.get( + translation_key=new_page.translation_key, + locale=self.es_locale, + alias_of=new_page, + ) + self.assertFalse(es_new_page.revisions.exists()) + + def test_create_new_locale_synchronisation(self): + new_page = self.en_homepage.add_child( + instance=TestPage(title="Foo", slug="foo") + ) + + # Spanish version shouldn't exist yet + self.assertFalse(TestPage.objects.filter(translation_key=new_page.translation_key, locale=self.es_locale).exists()) + + # Creating the locale synchronisation should create the page in Spanish + LocaleSynchronization.objects.create( + locale=self.es_locale, + sync_from=self.en_locale, + ) + + es_new_page = TestPage.objects.get(translation_key=new_page.translation_key, locale=self.es_locale) + self.assertEqual(es_new_page.alias_of, new_page.page_ptr) diff --git a/wagtail_localize/views/edit_translation.py b/wagtail_localize/views/edit_translation.py index 8086230e..178a31af 100644 --- a/wagtail_localize/views/edit_translation.py +++ b/wagtail_localize/views/edit_translation.py @@ -781,3 +781,20 @@ def machine_translate(request, translation_id): next_url = reverse('wagtailadmin_home') return redirect(next_url) + + +def edit_translatable_alias_page(request, page): + return render(request, 'wagtail_localize/admin/edit_translatable_alias.html', { + 'page': page, + 'page_for_status': page, + 'content_type': page.cached_content_type, + 'next': get_valid_next_url_from_request(request), + 'locale': page.locale, + 'translations': [ + { + 'locale': translation.locale, + 'url': reverse('wagtailadmin_pages:edit', args=[translation.id]), + } + for translation in page.get_translations().only('id', 'locale').select_related('locale') + ], + }) diff --git a/wagtail_localize/views/submit_translations.py b/wagtail_localize/views/submit_translations.py index 6f860bab..6578e310 100644 --- a/wagtail_localize/views/submit_translations.py +++ b/wagtail_localize/views/submit_translations.py @@ -41,8 +41,14 @@ def __init__(self, instance, *args, **kwargs): if hide_include_subtree: self.fields["include_subtree"].widget = forms.HiddenInput() + existing_translations = instance.get_translations(inclusive=True) + + # Don't count page aliases as existing translations. We can convert aliases into properly translated pages + if isinstance(instance, Page): + existing_translations = existing_translations.exclude(alias_of__isnull=False) + self.fields["locales"].queryset = Locale.objects.exclude( - id__in=instance.get_translations(inclusive=True).values_list('locale_id', flat=True) + id__in=existing_translations.values_list('locale_id', flat=True) ) # Using len() instead of count() here as we're going to evaluate this queryset @@ -119,7 +125,13 @@ def get_form(self): if self.request.method == 'POST': return SubmitTranslationForm(self.object, self.request.POST) else: - return SubmitTranslationForm(self.object) + initial = None + if self.request.GET.get('select_locale', None): + select_locale = Locale.objects.filter(language_code=self.request.GET['select_locale']).first() + if select_locale: + initial = {'locales': [select_locale]} + + return SubmitTranslationForm(self.object, initial=initial) def get_success_url(self): return get_valid_next_url_from_request(self.request) diff --git a/wagtail_localize/wagtail_hooks.py b/wagtail_localize/wagtail_hooks.py index 4af50128..b08d4a9f 100644 --- a/wagtail_localize/wagtail_hooks.py +++ b/wagtail_localize/wagtail_hooks.py @@ -24,6 +24,9 @@ from .models import Translation, TranslationSource from .views import edit_translation, submit_translations, update_translations +# Import synctree so it can register its signal handler +from . import synctree # noqa + @hooks.register("register_admin_urls") def register_admin_urls(): @@ -63,7 +66,7 @@ def page_listing_more_buttons(page, page_perms, is_parent=False, next_url=None): if page_perms.user.has_perm('wagtail_localize.submit_translation') and not page.is_root(): # If there's at least one locale that we haven't translated into yet, show "Translate this page" button has_locale_to_translate_to = Locale.objects.exclude( - id__in=page.get_translations(inclusive=True).values_list('locale_id', flat=True) + id__in=page.get_translations(inclusive=True).exclude(alias_of__isnull=False).values_list('locale_id', flat=True) ).exists() if has_locale_to_translate_to: @@ -122,6 +125,10 @@ def register_snippet_listing_buttons(snippet, user, next_url=None): @hooks.register("before_edit_page") def before_edit_page(request, page): + # If the page is an alias of a page in another locale, override the edit page so that we can show a "Translate this page" option + if page.alias_of and page.alias_of.locale is not page.locale: + return edit_translation.edit_translatable_alias_page(request, page) + # Check if the user has clicked the "Restart Translation" menu item if request.method == 'POST' and 'localize-restart-translation' in request.POST: try: