Bug 1441353 - Support several locales in `task.payload.upstreamArtifa…

beetmoverscript/templates/devedition_candidates_repacks.yml

@@ -7,6 +7,7 @@ metadata:
 s3_bucket_path: pub/devedition/candidates/{{ version }}-candidates/build{{ build_number }}/
 
 mapping:
+{% for locale in locales %}
   {{ locale }}:
 
     target.complete.mar:
@@ -79,3 +80,4 @@ mapping:
       update_balrog_manifest: true
       from_buildid: {{ partials[partial].buildid }}
   {% endfor %}
+{% endfor %} # end for locales

beetmoverscript/templates/fennec_candidates_repacks.yml

@@ -7,6 +7,7 @@ metadata:
 s3_bucket_path: pub/mobile/candidates/{{ version }}-candidates/build{{ build_number }}/
 
 mapping:
+{% for locale in locales %}
   {{ locale }}:
     target.apk:
       s3_key: fennec-{{ version }}.{{ locale }}.android-arm.apk
@@ -23,3 +24,4 @@ mapping:
       destinations:
         - {{ platform }}/{{ locale }}/fennec-{{ version }}.{{ locale }}.android-arm.checksums.asc
         - beetmover-checksums/{{ platform }}/{{ locale }}/fennec-{{ version }}.android-arm.checksums.asc
+{% endfor %} # end for locales

beetmoverscript/templates/fennec_nightly_repacks.yml

@@ -7,6 +7,7 @@ metadata:
 s3_bucket_path: pub/mobile/nightly/
 
 mapping:
+{% for locale in locales %}
   {{ locale }}:
     target.apk:
       s3_key: fennec-{{ version }}.{{ locale }}.android-arm.apk
@@ -24,3 +25,4 @@ mapping:
       destinations:
         - {{ upload_date }}-{{ branch }}-{{ platform }}-l10n/fennec-{{ version }}.{{ locale }}.android-arm.checksums.asc
         - latest-{{ branch }}-{{ platform }}-l10n/fennec-{{ version }}.{{ locale }}.android-arm.checksums.asc
+{% endfor %} # end for locales

beetmoverscript/templates/firefox_candidates_repacks.yml

@@ -8,8 +8,8 @@ metadata:
 s3_bucket_path: pub/firefox/candidates/{{ version }}-candidates/build{{ build_number }}/
 
 mapping:
+{% for locale in locales %}
   {{ locale }}:
-
     target.complete.mar:
       s3_key: update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
       destinations:
@@ -80,3 +80,4 @@ mapping:
       update_balrog_manifest: true
       from_buildid: {{ partials[partial].buildid }}
   {% endfor %}
+{% endfor %} # end for locales

beetmoverscript/templates/firefox_nightly_repacks.yml

@@ -7,8 +7,8 @@ metadata:
 s3_bucket_path: pub/firefox/nightly/
 
 mapping:
+{% for locale in locales %}
   {{ locale }}:
-
     target.langpack.xpi:
       s3_key: firefox-{{ version }}.{{ locale }}.langpack.xpi
       destinations:
@@ -85,3 +85,4 @@ mapping:
       update_balrog_manifest: true
       from_buildid: {{ partials[partial].buildid }}
   {% endfor %}
+{% endfor %} # end for locales

beetmoverscript/test/test_utils.py

@@ -2,14 +2,17 @@
 import pytest
 import tempfile
 
+from scriptworker.exceptions import TaskVerificationError
+
 from beetmoverscript.test import (context, get_fake_valid_task,
                                   get_fake_balrog_props, get_fake_checksums_manifest)
 import beetmoverscript.utils as butils
 from beetmoverscript.utils import (generate_beetmover_manifest, get_hash,
                                    write_json, generate_beetmover_template_args,
                                    write_file, is_release_action, is_promotion_action,
                                    get_partials_props, matches_exclude, get_candidates_prefix,
-                                   get_releases_prefix, get_product_name)
+                                   get_releases_prefix, get_product_name,
+                                   _check_locale_consistency)
 from beetmoverscript.constants import HASH_BLOCK_SIZE
 
 assert context  # silence pyflakes
@@ -104,15 +107,75 @@ def test_beetmover_template_args_generation(context, taskjson, partials):
         'version': '99.0a1',
         'buildid': '20990205110000',
         'partials': partials,
+        'locales': ['en-US'],
     }
 
     template_args = generate_beetmover_template_args(context)
     assert template_args == expected_template_args
 
     context.task['payload']['locale'] = 'ro'
+    context.task['payload']['upstreamArtifacts'][0]['locale'] = 'ro'
+    expected_template_args['template_key'] = 'fake_nightly_repacks'
+    expected_template_args['locales'] = ['ro']
     template_args = generate_beetmover_template_args(context)
+    assert template_args == expected_template_args
+
+
+@pytest.mark.parametrize('payload, expected_locales', ((
+    {'upstreamArtifacts': [{'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build'}]},
+    None
+), (
+    {
+        'upstreamArtifacts': [
+            {'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build', 'locale': 'en-US'},
+        ],
+    },
+    ['en-US']
+), (
+    {
+        'locale': 'en-US',
+        'upstreamArtifacts': [
+            {'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build', 'locale': 'en-US'},
+        ],
+    },
+    ['en-US']
+), (
+    {
+        'locale': 'ro',
+        'upstreamArtifacts': [
+            {'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build', 'locale': 'ro'},
+        ],
+    },
+    ['ro']
+), (
+    {
+        'upstreamArtifacts': [
+            {'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build', 'locale': 'ro'},
+            {'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build', 'locale': 'sk'},
+        ],
+    },
+    ['ro', 'sk']
+), (
+    {
+        'locale': 'ro',
+        'upstreamArtifacts': [
+            {'path': 'some/path', 'taskId': 'someTaskId', 'type': 'build'},
+        ],
+    },
+    ['ro']
+)))
+def test_beetmover_template_args_locales(context, payload, expected_locales):
+    context.task = get_fake_valid_task('task_partials.json')
+    context.task['payload'] = payload
+    context.task['payload']['upload_date'] = '2018/04/2018-04-09-15-30-00'
 
-    assert template_args['template_key'] == 'fake_nightly_repacks'
+    template_args = generate_beetmover_template_args(context)
+    if expected_locales:
+        assert 'locale' not in template_args    # locale used to be the old way of filling locale
+        assert template_args['locales'] == expected_locales
+    else:
+        assert 'locale' not in template_args
+        assert 'locales' not in template_args
 
 
 def test_beetmover_template_args_generation_release(context):
@@ -132,12 +195,32 @@ def test_beetmover_template_args_generation_release(context):
         'buildid': '20990205110000',
         'partials': {},
         'build_number': 3,
+        'locales': ['en-US'],
     }
 
     template_args = generate_beetmover_template_args(context)
     assert template_args == expected_template_args
 
 
+@pytest.mark.parametrize('locale_in_payload, locales_in_upstream_artifacts, raises', ((
+    'en-US', [], False,
+), (
+    'en-US', ['en-US'], False,
+), (
+    'ro', ['ro'], False,
+), (
+    'en-US', ['ro'], True,
+), (
+    'en-US', ['en-US', 'ro'], True,
+)))
+def test_check_locale_consistency(locale_in_payload, locales_in_upstream_artifacts, raises):
+    if raises:
+        with pytest.raises(TaskVerificationError):
+            _check_locale_consistency(locale_in_payload, locales_in_upstream_artifacts)
+    else:
+        _check_locale_consistency(locale_in_payload, locales_in_upstream_artifacts)
+
+
 # is_release_action is_promotion_action {{{1
 @pytest.mark.parametrize("action,release,promotion", ((
     'push-to-nightly', False, False,

beetmoverscript/utils.py

@@ -9,6 +9,8 @@
 import re
 import yaml
 
+from scriptworker.exceptions import TaskVerificationError
+
 from beetmoverscript.constants import (
     HASH_BLOCK_SIZE, TEMPLATE_KEY_PLATFORMS,
     RELEASE_ACTIONS, PROMOTION_ACTIONS, PRODUCT_TO_PATH
@@ -104,8 +106,21 @@ def generate_beetmover_template_args(context):
     # e.g. action = 'push-to-candidates' or 'push-to-nightly'
     tmpl_bucket = context.action.split('-')[-1]
 
-    if 'locale' in task["payload"]:
-        tmpl_args["locale"] = task["payload"]["locale"]
+    locales_in_upstream_artifacts = [
+        upstream_artifact['locale']
+        for upstream_artifact in task['payload']['upstreamArtifacts']
+        if 'locale' in upstream_artifact
+    ]
+
+    if 'locale' in task['payload'] and locales_in_upstream_artifacts:
+        _check_locale_consistency(task['payload']['locale'], locales_in_upstream_artifacts)
+        tmpl_args['locales'] = locales_in_upstream_artifacts
+    elif locales_in_upstream_artifacts:
+        tmpl_args['locales'] = locales_in_upstream_artifacts
+    elif 'locale' in task['payload']:
+        tmpl_args['locales'] = [task['payload']['locale']]
+
+    if tmpl_args.get('locales') and tmpl_args.get('locales') != ['en-US']:
         product_name = get_product_name(release_props["appName"].lower(), tmpl_key_platform)
         tmpl_args["template_key"] = "%s_%s_repacks" % (product_name, tmpl_bucket)
     else:
@@ -114,6 +129,21 @@ def generate_beetmover_template_args(context):
     return tmpl_args
 
 
+def _check_locale_consistency(locale_in_payload, locales_in_upstream_artifacts):
+    if len(locales_in_upstream_artifacts) > 1:
+        raise TaskVerificationError(
+            '`task.payload.locale` is defined ("{}") but too many locales set in \
+`task.payload.upstreamArtifacts` ({})'.format(locale_in_payload, locales_in_upstream_artifacts)
+        )
+    elif len(locales_in_upstream_artifacts) == 1:
+        locale_in_upstream_artifacts = locales_in_upstream_artifacts[0]
+        if locale_in_payload != locale_in_upstream_artifacts:
+            raise TaskVerificationError(
+                '`task.payload.locale` ("{}") does not match the one set in \
+`task.payload.upstreamArtifacts` ("{}")'.format(locale_in_payload, locale_in_upstream_artifacts)
+            )
+
+
 def generate_beetmover_manifest(context):
     """
     generates and outputs a manifest that maps expected Taskcluster artifact names