wire artifacts_to_beetmove to move_beets

mozilla-releng · Jul 23, 2018 · ef5fe70 · ef5fe70
1 parent e227115
commit ef5fe70
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 119 deletions.
diff --git a/beetmoverscript/script.py b/beetmoverscript/script.py
@@ -166,10 +166,12 @@ async def push_to_maven(context):
 
     expected_files = list(mapping_manifest['mapping'].keys())
 
-    context.artifacts_to_beetmove = check_and_extract_zip_archives(
-        context.artifacts_to_beetmove, expected_files,
-        context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO)
-    )
+    context.artifacts_to_beetmove = {
+        'en-US': check_and_extract_zip_archives(
+            context.artifacts_to_beetmove, expected_files,
+            context.config.get('zip_extract_max_file_size_in_mb', ZIP_MAX_COMPRESSION_RATIO)
+        )
+    }
 
     await move_beets(context, context.artifacts_to_beetmove, mapping_manifest)
 

diff --git a/beetmoverscript/templates/maven_geckoview.yml b/beetmoverscript/templates/maven_geckoview.yml
@@ -7,53 +7,56 @@ metadata:
 s3_bucket_path: maven2/org/mozilla/{{ artifact_id }}/{{ version }}/  # Maven groupId is org.mozilla
 
 mapping:
+{% for locale in ['en-US'] %}
+  "{{ locale }}":   # Locale is not needed for geckoview, it's used by move_beets, though
 {% for product in ['geckoview'] %}
-  "{{ product }}.aar":
-    s3_key: {{ artifact_id }}-{{ version }}.aar
-    destinations:
-      - {{ artifact_id }}-{{ version }}.aar
-  "{{ product }}.aar.md5":
-    s3_key: {{ artifact_id }}-{{ version }}.aar.md5
-    destinations:
-      - {{ artifact_id }}-{{ version }}.aar.md5
-  "{{ product }}.aar.sha1":
-    s3_key: {{ artifact_id }}-{{ version }}.aar.sha1
-    destinations:
-      - {{ artifact_id }}-{{ version }}.aar.sha1
-  "{{ product }}.pom":
-    s3_key: {{ artifact_id }}-{{ version }}.pom
-    destinations:
-      - {{ artifact_id }}-{{ version }}.pom
-  "{{ product }}.pom.md5":
-    s3_key: {{ artifact_id }}-{{ version }}.pom.md5
-    destinations:
-      - {{ artifact_id }}-{{ version }}.pom.md5
-  "{{ product }}.pom.sha1":
-    s3_key: {{ artifact_id }}-{{ version }}.pom.sha1
-    destinations:
-      - {{ artifact_id }}-{{ version }}.pom.sha1
-  "{{ product }}-javadoc.jar":
-    s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar
-    destinations:
-      - {{ artifact_id }}-{{ version }}-javadoc.jar
-  "{{ product }}-javadoc.jar.md5":
-    s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5
-    destinations:
-      - {{ artifact_id }}-{{ version }}-javadoc.jar.md5
-  "{{ product }}-javadoc.jar.sha1":
-    s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
-    destinations:
-      - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
-  "{{ product }}-sources.jar":
-    s3_key: {{ artifact_id }}-{{ version }}-sources.jar
-    destinations:
-      - {{ artifact_id }}-{{ version }}-sources.jar
-  "{{ product }}-sources.jar.md5":
-    s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5
-    destinations:
-      - {{ artifact_id }}-{{ version }}-sources.jar.md5
-  "{{ product }}-sources.jar.sha1":
-    s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1
-    destinations:
-      - {{ artifact_id }}-{{ version }}-sources.jar.sha1
+    "{{ product }}.aar":
+      s3_key: {{ artifact_id }}-{{ version }}.aar
+      destinations:
+        - {{ artifact_id }}-{{ version }}.aar
+    "{{ product }}.aar.md5":
+      s3_key: {{ artifact_id }}-{{ version }}.aar.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}.aar.md5
+    "{{ product }}.aar.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}.aar.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}.aar.sha1
+    "{{ product }}.pom":
+      s3_key: {{ artifact_id }}-{{ version }}.pom
+      destinations:
+        - {{ artifact_id }}-{{ version }}.pom
+    "{{ product }}.pom.md5":
+      s3_key: {{ artifact_id }}-{{ version }}.pom.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}.pom.md5
+    "{{ product }}.pom.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}.pom.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}.pom.sha1
+    "{{ product }}-javadoc.jar":
+      s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar
+      destinations:
+        - {{ artifact_id }}-{{ version }}-javadoc.jar
+    "{{ product }}-javadoc.jar.md5":
+      s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}-javadoc.jar.md5
+    "{{ product }}-javadoc.jar.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}-javadoc.jar.sha1
+    "{{ product }}-sources.jar":
+      s3_key: {{ artifact_id }}-{{ version }}-sources.jar
+      destinations:
+        - {{ artifact_id }}-{{ version }}-sources.jar
+    "{{ product }}-sources.jar.md5":
+      s3_key: {{ artifact_id }}-{{ version }}-sources.jar.md5
+      destinations:
+        - {{ artifact_id }}-{{ version }}-sources.jar.md5
+    "{{ product }}-sources.jar.sha1":
+      s3_key: {{ artifact_id }}-{{ version }}-sources.jar.sha1
+      destinations:
+        - {{ artifact_id }}-{{ version }}-sources.jar.sha1
+{% endfor %}
 {% endfor %}
diff --git a/beetmoverscript/test/test_utils.py b/beetmoverscript/test/test_utils.py
@@ -97,53 +97,55 @@ def test_generate_manifest_maven(context, mocker):
 
     assert generate_beetmover_manifest(context) == {
         'mapping': {
-            'geckoview-javadoc.jar': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar',
-            },
-            'geckoview-javadoc.jar.md5': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5',
-            },
-            'geckoview-javadoc.jar.sha1': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1',
-            },
-            'geckoview-sources.jar': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar',
-            },
-            'geckoview-sources.jar.md5': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5',
-            },
-            'geckoview-sources.jar.sha1': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1',
-            },
-            'geckoview.aar': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar',
-            },
-            'geckoview.aar.md5': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5',
-            },
-            'geckoview.aar.sha1': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1',
-            },
-            'geckoview.pom': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom',
-            },
-            'geckoview.pom.md5': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5',
-            },
-            'geckoview.pom.sha1': {
-                'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'],
-                's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1'
+            'en-US': {
+                'geckoview-javadoc.jar': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar',
+                },
+                'geckoview-javadoc.jar.md5': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.md5',
+                },
+                'geckoview-javadoc.jar.sha1': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000-javadoc.jar.sha1',
+                },
+                'geckoview-sources.jar': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar',
+                },
+                'geckoview-sources.jar.md5': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.md5',
+                },
+                'geckoview-sources.jar.sha1': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000-sources.jar.sha1',
+                },
+                'geckoview.aar': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar',
+                },
+                'geckoview.aar.md5': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.md5'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.md5',
+                },
+                'geckoview.aar.sha1': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000.aar.sha1'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000.aar.sha1',
+                },
+                'geckoview.pom': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom',
+                },
+                'geckoview.pom.md5': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.md5'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.md5',
+                },
+                'geckoview.pom.sha1': {
+                    'destinations': ['geckoview-nightly-x86-63.0.20181231120000.pom.sha1'],
+                    's3_key': 'geckoview-nightly-x86-63.0.20181231120000.pom.sha1'
+                },
             },
         },
         'metadata': {

diff --git a/beetmoverscript/test/test_zip.py b/beetmoverscript/test/test_zip.py
@@ -90,21 +90,17 @@ def test_check_and_extract_zip_archives():
             artifacts_per_task_id, expected_files_per_archive_per_task_id, zip_max_size_in_mb=100
         )
 
-        assert files_once_extracted == {
-            'firstTaskId': [
-                '/a/non/archive',
-                '/another/non/archive',
-                os.path.join(d, 'firstTaskId-archive1.zip.out', 'some_file1'),
-                os.path.join(d, 'firstTaskId-archive1.zip.out', 'some', 'subfolder', 'file1'),
-                os.path.join(d, 'firstTaskId-archive2.zip.out', 'some_file2'),
-                os.path.join(d, 'firstTaskId-archive2.zip.out', 'some', 'subfolder', 'file2'),
-            ],
-            'secondTaskId': ['/just/another/regular/file'],
-            'thirdTaskId': [
-                os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some_file3'),
-                os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some', 'subfolder', 'file3'),
-            ],
-        }
+        assert sorted(files_once_extracted) == sorted([
+            '/a/non/archive',
+            '/another/non/archive',
+            os.path.join(d, 'firstTaskId-archive1.zip.out', 'some_file1'),
+            os.path.join(d, 'firstTaskId-archive1.zip.out', 'some', 'subfolder', 'file1'),
+            os.path.join(d, 'firstTaskId-archive2.zip.out', 'some_file2'),
+            os.path.join(d, 'firstTaskId-archive2.zip.out', 'some', 'subfolder', 'file2'),
+            '/just/another/regular/file',
+            os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some_file3'),
+            os.path.join(d, 'thirdTaskId-archive1.zip.out', 'some', 'subfolder', 'file3'),
+        ])
 
 
 def test_check_and_extract_zip_archives_for_given_task():

diff --git a/beetmoverscript/zip.py b/beetmoverscript/zip.py
@@ -10,26 +10,23 @@
 
 
 def check_and_extract_zip_archives(artifacts_per_task_id, expected_files_per_archive_per_task_id, zip_max_size_in_mb):
-    deflated_artifacts_per_task_id = {}
+    deflated_artifacts = []
 
     for task_id, task_artifacts_params in artifacts_per_task_id.items():
         for artifacts_param in task_artifacts_params:
             paths_for_task = artifacts_param['paths']
-            deflated_artifacts = deflated_artifacts_per_task_id.get(task_id, [])
 
             if artifacts_param['zip_extract'] is False:
                 log.debug('Skipping artifacts marked as not `zipExtract`able: {}'.format(paths_for_task))
                 deflated_artifacts.extend(paths_for_task)
-                deflated_artifacts_per_task_id[task_id] = deflated_artifacts
                 continue
 
             expected_files_per_archive = expected_files_per_archive_per_task_id[task_id]
             deflated_artifacts.extend(_check_and_extract_zip_archives_for_given_task(
                 task_id, expected_files_per_archive, zip_max_size_in_mb
             ))
-            deflated_artifacts_per_task_id[task_id] = deflated_artifacts
 
-    return deflated_artifacts_per_task_id
+    return deflated_artifacts
 
 
 def _check_and_extract_zip_archives_for_given_task(task_id, expected_files_per_archive, zip_max_size_in_mb):