Fix bug 1552802: Add support for pretranslation in admin panel. (#1487)

Run pretranslation for each new locale, file or string added.

pontoon/administration/forms.py

@@ -48,7 +48,9 @@ class Meta:
             'contact',
             'disabled',
             'sync_disabled',
-            'tags_enabled')
+            'tags_enabled',
+            'pretranslation_enabled',
+        )
 
     def __init__(self, *args, **kwargs):
         super(ProjectForm, self).__init__(*args, **kwargs)

pontoon/administration/static/css/admin_project.css

@@ -150,10 +150,16 @@ form .button {
 
 .controls .button.sync,
 .controls .button.sync:hover {
-  background: #F36;
-  color: #272A2F;
+  background: #333941;
+  color: #aaa;
   float: left;
-  margin-right: 10px;
+}
+
+.controls .button.pretranslate,
+.controls .button.pretranslate:hover {
+  background: #333941;
+  color: #aaa;
+  float: right;
 }
 
 form a:link, form a:visited {
@@ -464,7 +470,7 @@ textarea.strings-source {
 .locales .errorlist {
   margin-top: 10px;
   text-align: center;
-  width: 100%;  
+  width: 100%;
 }
 
 .notification {

pontoon/administration/static/js/admin_project.js

@@ -72,6 +72,32 @@ $(function() {
     });
   });
 
+  // Manually Pretranslate project
+  $('.pretranslate').click(function(e) {
+    e.preventDefault();
+
+    var button = $(this),
+        title = button.html();
+
+    if (button.is('.in-progress')) {
+      return;
+    }
+
+    button.addClass('in-progress').html('Pretranslating...');
+
+    $.ajax({
+      url: '/admin/projects/' + $('#id_slug').val() + '/pretranslate/'
+    }).success(function() {
+      button.html('Pretranslation started');
+    }).error(function() {
+      button.html('Whoops!');
+    }).complete(function() {
+      setTimeout(function() {
+        button.removeClass('in-progress').html(title);
+      }, 2000);
+    });
+  });
+
   // Suggest slugified name for new projects
   $('#id_name').blur(function() {
     if ($('input[name=pk]').length > 0 || !$('#id_name').val()) {

pontoon/administration/templates/admin_project.html

@@ -294,10 +294,21 @@ <h3>
       {% endif %}
   </section>
 
+  <section class="controls clearfix">
+    <h3>
+      <span>Pretranslation</span>
+    </h3>
+    <div class="clearfix">
+      <button class="pretranslate button clearfix">Pretranslate</button>
+      <div class="checkbox clearfix">
+        <label for="id_pretranslation_enabled">
+            {{ form.pretranslation_enabled }}{{ form.pretranslation_enabled.label }}
+        </label>
+      </div>
+    </div>
+  </section>
+
   <div class="controls clearfix">
-      {% if settings.MANUAL_SYNC %}
-      <button class="sync button">Sync</button>
-      {% endif %}
       <div class="checkbox clearfix">
           <label for="id_disabled">
               {{ form.disabled }}{{ form.disabled.label }}
@@ -308,6 +319,9 @@ <h3>
               {{ form.sync_disabled }}{{ form.sync_disabled.label }}
           </label>
       </div>
+      {% if settings.MANUAL_SYNC %}
+      <button class="sync button">Sync</button>
+      {% endif %}
       <button class="button active save">Save project</button>
       <a href="{{ url('pontoon.admin') }}" class="cancel">Cancel</a>
   </div>

pontoon/administration/urls.py

@@ -41,6 +41,13 @@
         name='pontoon.admin.project.strings'
     ),
 
+    # Pretranslate project
+    url(
+        r'^projects/(?P<slug>[\w-]+)/pretranslate/$',
+        views.manually_pretranslate_project,
+        name='pontoon.project.sync'
+    ),
+
     # Edit project
     url(
         r'^projects/(?P<slug>.+)/$',

pontoon/administration/views.py

@@ -36,6 +36,7 @@
 )
 from pontoon.sync.models import SyncLog
 from pontoon.sync.tasks import sync_project
+from pontoon.pretranslation.tasks import pretranslate
 
 
 log = logging.getLogger(__name__)
@@ -529,3 +530,17 @@ def manually_sync_project(request, slug):
     sync_project.delay(project.pk, sync_log.pk)
 
     return HttpResponse('ok')
+
+
+@login_required(redirect_field_name='', login_url='/403')
+@require_AJAX
+def manually_pretranslate_project(request, slug):
+    if not request.user.has_perm('base.can_manage_project'):
+        return HttpResponseForbidden(
+            "Forbidden: You don't have permission for pretranslating projects"
+        )
+
+    project = Project.objects.get(slug=slug)
+    pretranslate(project)
+
+    return HttpResponse('ok')

pontoon/pretranslation/tasks.py

@@ -25,10 +25,17 @@ def pretranslate(project, locales=None, entities=None):
 
     log.info('Fetching pretranslations for project {} started'.format(project.name))
 
-    if not locales:
-        locales = project.locales.filter(
-            project_locale__readonly=False,
-        ).prefetch_project_locale(project)
+    if locales:
+        locales = project.locales.filter(pk__in=locales)
+    else:
+        locales = project.locales
+
+    locales = (
+        locales
+        .filter(project_locale__readonly=False)
+        .distinct()
+        .prefetch_project_locale(project)
+    )
 
     if not entities:
         entities = Entity.objects.filter(

pontoon/sync/changeset.py

@@ -54,6 +54,7 @@ def __init__(self, db_project, vcs_project, now, locale=None):
         self.entities_to_update = []
         self.translations_to_update = {}
         self.translations_to_create = []
+        self.new_entities = []
         self.commit_authors_per_locale = defaultdict(list)
         self.locales_to_commit = set()
 
@@ -235,6 +236,7 @@ def execute_create_db(self):
                     ))
 
         self.send_notifications(new_entities)
+        self.new_entities = new_entities
 
     def update_entity_translations_from_vcs(
             self, db_entity, locale_code, vcs_translation,

pontoon/sync/core.py

@@ -34,7 +34,7 @@ def update_originals(db_project, now, full_scan=False):
         update_entities(db_project, vcs_project, changeset)
         changeset.execute()
 
-    return added_paths, removed_paths, changed_paths
+    return added_paths, removed_paths, changed_paths, changeset.new_entities
 
 
 def serial_task(timeout, lock_key="", on_error=None, **celery_args):
@@ -172,15 +172,18 @@ def update_translations(db_project, vcs_project, locale, changeset):
 
 
 def update_translated_resources(db_project, vcs_project, locale):
-    """Update the TranslatedResource entries in the database."""
+    """
+    Update the TranslatedResource entries in the database.
+    Returns true if a new TranslatedResource is added to the locale.
+    """
     if vcs_project.configuration:
-        update_translated_resources_with_config(
+        return update_translated_resources_with_config(
             db_project,
             vcs_project,
             locale,
         )
     else:
-        update_translated_resources_without_config(
+        return update_translated_resources_without_config(
             db_project,
             vcs_project,
             locale,
@@ -192,29 +195,43 @@ def update_translated_resources_with_config(db_project, vcs_project, locale):
     Create/update the TranslatedResource objects for each Resource instance
     that is enabled for the given locale through project configuration.
     """
+    tr_created = False
+
     for resource in vcs_project.configuration.locale_resources(locale):
-        translatedresource, _ = (
+        translatedresource, created = (
             TranslatedResource.objects.get_or_create(resource=resource, locale=locale)
         )
+
+        if created:
+            tr_created = True
         translatedresource.calculate_stats()
 
+    return tr_created
+
 
 def update_translated_resources_without_config(db_project, vcs_project, locale):
     """
     We only want to create/update the TranslatedResource object if the
     resource exists in the current locale, UNLESS the file is asymmetric.
     """
+    tr_created = False
+
     for resource in db_project.resources.all():
         vcs_resource = vcs_project.resources.get(resource.path, None)
 
         if vcs_resource is not None:
             resource_exists = vcs_resource.files.get(locale) is not None
             if resource_exists or resource.is_asymmetric:
-                translatedresource, _ = (
+                translatedresource, created = (
                     TranslatedResource.objects.get_or_create(resource=resource, locale=locale)
                 )
+
+                if created:
+                    tr_created = True
                 translatedresource.calculate_stats()
 
+    return tr_created
+
 
 def update_translated_resources_no_files(db_project, locale, changed_resources):
     """

pontoon/sync/tasks.py

@@ -27,6 +27,7 @@
 from pontoon.sync.models import ProjectSyncLog, RepositorySyncLog, SyncLog
 from pontoon.sync.vcs.repositories import CommitToRepositoryException
 from pontoon.sync.vcs.models import VCSProject, MissingSourceDirectoryError
+from pontoon.pretranslation.tasks import pretranslate
 
 
 log = logging.getLogger(__name__)
@@ -121,10 +122,11 @@ def sync_project(
         source_changes.get('added_paths'),
         source_changes.get('removed_paths'),
         source_changes.get('changed_paths'),
+        source_changes.get('new_entities'),
         locale=locale,
         no_pull=no_pull,
         no_commit=no_commit,
-        full_scan=force
+        full_scan=force,
     )
 
 
@@ -148,7 +150,7 @@ def sync_sources(db_project, now, force, no_pull):
 
     if force or source_repo_changed:
         try:
-            added_paths, removed_paths, changed_paths = update_originals(
+            added_paths, removed_paths, changed_paths, new_entities = update_originals(
                 db_project, now, full_scan=force
             )
         except MissingSourceDirectoryError as e:
@@ -160,7 +162,7 @@ def sync_sources(db_project, now, force, no_pull):
         log.info('Synced sources for project {0}.'.format(db_project.slug))
 
     else:
-        added_paths, removed_paths, changed_paths = None, None, None
+        added_paths, removed_paths, changed_paths, new_entities = None, None, None, None
         log.info(
             'Skipping syncing sources for project {0}, no changes detected.'.format(
                 db_project.slug
@@ -171,6 +173,7 @@ def sync_sources(db_project, now, force, no_pull):
         'added_paths': added_paths,
         'removed_paths': removed_paths,
         'changed_paths': changed_paths,
+        'new_entities': new_entities,
     }
 
 
@@ -182,7 +185,8 @@ def sync_sources(db_project, now, force, no_pull):
 )
 def sync_translations(
     self, project_pk, project_sync_log_pk, now, added_paths=None, removed_paths=None,
-    changed_paths=None, locale=None, no_pull=False, no_commit=False, full_scan=False,
+    changed_paths=None, new_entities=None, locale=None, no_pull=False, no_commit=False,
+    full_scan=False
 ):
     db_project = get_or_fail(
         Project,
@@ -279,6 +283,9 @@ def sync_translations(
     synced_locales = set()
     failed_locales = set()
 
+    # Store newly added locales and locales with newly added resources
+    new_locales = []
+
     for locale in locales:
         try:
             with transaction.atomic():
@@ -297,7 +304,10 @@ def sync_translations(
                 changeset = ChangeSet(db_project, vcs_project, now, locale)
                 update_translations(db_project, vcs_project, locale, changeset)
                 changeset.execute()
-                update_translated_resources(db_project, vcs_project, locale)
+
+                created = update_translated_resources(db_project, vcs_project, locale)
+                if created:
+                    new_locales.append(locale.pk)
                 update_locale_project_locale_stats(locale, db_project)
 
                 # Clear out the "has_changed" markers now that we've finished
@@ -348,7 +358,9 @@ def sync_translations(
 
             # We have files: update all translated resources.
             if locale in locales:
-                update_translated_resources(db_project, vcs_project, locale)
+                created = update_translated_resources(db_project, vcs_project, locale)
+                if created:
+                    new_locales.append[locale.pk]
 
             # We don't have files: we can still update asymmetric translated resources.
             else:
@@ -389,3 +401,16 @@ def sync_translations(
             locales=repo_locales[r.pk].exclude(code__in=failed_locales)
         )
     repo_sync_log.end()
+
+    if db_project.pretranslation_enabled:
+        # Pretranslate all entities for newly added locales
+        # and locales with newly added resources
+        if len(new_locales):
+            pretranslate(db_project, locales=new_locales)
+
+        locales = db_project.locales.exclude(pk__in=new_locales).values_list('pk', flat=True)
+
+        # Pretranslate newly added entities for all locales
+        if new_entities and locales:
+            new_entities = list(set(new_entities))
+            pretranslate(db_project, locales=locales, entities=new_entities)