Bug 1423885 - Yet another milestone beetmoverscript (#96)

* Bug 1398805 - stop transfering checksums.asc as beet files. Leave them be are. * 3.4.1-dev1 * Add temp devedition templates and fix appName hassle. * 3.4.1-dev2 * maybe fix devedition * dev3 * try to match actual balrog_props * dev4 * ah, this is the real fix? * dev5 * ARGH * dev6 * Fix l10n BM for devedition * Fix README * 3.4.1-dev7 * Fix the 403, rewrite product bucket logic. * 3.4.1-dev8 * Adjust product name in Balrog manifest for Devedition * 3.4.1-dev9 * fix devedition beetmover-cdns * dev10 * Fix platforms for Balrog manifests * 3.4.1-dev10 * 3.4.1-dev11 * 3.5.0 * 3.5.0 * Add a break statement in generate_balrog_info() to tiny improve perf. * Fix source, checksums and bump to 4.0.0 * Remove SOURCE as beetmover-checksums * 4.0.2 * 4.0.2 in Changelog for 2017-12-14
mozilla-releng · Dec 14, 2017 · 42b1ea7 · 42b1ea7
1 parent dc2f8cd
commit 42b1ea7
Show file tree

Hide file tree

Showing 17 changed files with 489 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,21 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [4.0.2] - 2017-12-14
+
+### Added
+- beetmoverscript support for Devedition releases
+- `STAGE_PLATFORM_MAP` now encompasses the devedition platforms as well
+- `NORMALIZED_BALROG_PLATFORMS` to correct platforms before writing them to balrog manifests
+- support for `.beet` files in order to enhance the BBB checksums
+- `get_product_name` function to standardize the way to refer to the product name based on platform and appName property from balrog props
+- checksums for Fennec
+- SOURCE files for Fennec
+
+### Changed
+- stop uploading checksums.asc files as `.beet`  under beetmover-checksums
+- `get_release_props` and `update_props`  functions now take context as argument
+
 ## [3.4.0] - 2017-12-05
 ### Added
 - beetmoverscript support to handle in-tree scheduled Firefox releases

diff --git a/README.md b/README.md
@@ -127,6 +127,8 @@ my_email: "scriptworker@example.com"
             "fennecx86_candidates": "/path/to/beetmoverscript/beetmoverscript/templates/fennecx86_candidates.yml",
             "firefox_candidates": "/path/to/beetmoverscript/beetmoverscript/templates/firefox_candidates.yml",
             "firefox_candidates_repacks": "/path/to/beetmoverscript/beetmoverscript/templates/firefox_candidates_repacks.yml"
+            "devedition_candidates": "/path/to/beetmoverscript/beetmoverscript/templates/devedition_candidates.yml",
+            "devedition_candidates_repacks": "/path/to/beetmoverscript/beetmoverscript/templates/devedition_candidates_repacks.yml"
         },
         "push-to-releases": {},
         "push-to-staging": {}

diff --git a/beetmoverscript/constants.py b/beetmoverscript/constants.py
@@ -11,13 +11,20 @@
     '.xpi': 'application/x-xpinstall',
     '.apk': 'application/vnd.android.package-archive',
 }
+
 STAGE_PLATFORM_MAP = {
     'linux': 'linux-i686',
+    'linux-devedition': 'linux-i686',
     'linux64': 'linux-x86_64',
+    'linux64-devedition': 'linux-x86_64',
     'macosx64': 'mac',
+    'macosx64-devedition': 'mac',
     'win32': 'win32',
+    'win32-devedition': 'win32',
     'win64': 'win64',
+    'win64-devedition': 'win64',
 }
+
 TEMPLATE_KEY_PLATFORMS = {
     "android-api-15": "fennec",
     "android-api-15-old-id": "fennec",
@@ -37,7 +44,17 @@
     "win32-devedition": "devedition",
     "win64-devedition": "devedition",
 }
+
+NORMALIZED_BALROG_PLATFORMS = {
+    "linux-devedition": "linux",
+    "linux64-devedition": "linux64",
+    "macosx64-devedition": "macosx64",
+    "win32-devedition": "win32",
+    "win64-devedition": "win64",
+}
+
 HASH_BLOCK_SIZE = 1024*1024
+
 INITIAL_RELEASE_PROPS_FILE = "balrog_props.json"
 # release buckets don't require a copy of the following artifacts
 IGNORED_UPSTREAM_ARTIFACTS = ["balrog_props.json"]
@@ -79,6 +96,7 @@
 PROMOTION_ACTIONS = (
     'push-to-candidates',
 )
+
 RELEASE_ACTIONS = (
     'push-to-releases',
 )
@@ -111,5 +129,6 @@
 PRODUCT_TO_PATH = {
     'mobile': 'pub/mobile/',
     'fennec': 'pub/mobile/',
+    'devedition': 'pub/devedition/',
     'firefox': 'pub/firefox/',
 }
diff --git a/beetmoverscript/script.py b/beetmoverscript/script.py
@@ -20,6 +20,7 @@
 
 from beetmoverscript.constants import (
     MIME_MAP, RELEASE_BRANCHES, CACHE_CONTROL_MAXAGE, RELEASE_EXCLUDE,
+    TEMPLATE_KEY_PLATFORMS, NORMALIZED_BALROG_PLATFORMS
 )
 from beetmoverscript.task import (
     validate_task_schema, add_balrog_manifest_to_artifacts,
@@ -31,7 +32,8 @@
     load_json, get_hash, get_release_props, generate_beetmover_manifest,
     get_size, alter_unpretty_contents, matches_exclude,
     get_candidates_prefix, get_releases_prefix, get_creds, get_bucket_name,
-    is_release_action, is_promotion_action, get_partials_props
+    is_release_action, is_promotion_action, get_partials_props,
+    get_product_name
 )
 
 log = logging.getLogger(__name__)
@@ -51,7 +53,7 @@ async def push_to_nightly(context):
 
     # find release properties and make a copy in the artifacts directory
     release_props_file = get_initial_release_props_file(context)
-    context.release_props = get_release_props(release_props_file)
+    context.release_props = get_release_props(context, release_props_file)
 
     # generate beetmover mapping manifest
     mapping_manifest = generate_beetmover_manifest(context)
@@ -294,11 +296,11 @@ def generate_balrog_info(context, artifact_pretty_name, locale, destinations, fr
         data["from_buildid"] = from_buildid
         if is_promotion_action(context.action):
             partials = get_partials_props(context.task)
-            # TODO: improve search by a dedicated dict with buildid as key
             for p in partials.values():
                 if p['buildid'] == str(from_buildid):
                     data['previousVersion'] = p['previousVersion']
                     data['previousBuildNumber'] = p['previousBuildNumber']
+                    break
 
     return data
 
@@ -313,14 +315,17 @@ def enrich_balrog_manifest(context, locale):
                                  'http://download.cdn.mozilla.net/pub'])
 
     enrich_dict = {
-        "appName": release_props["appName"],
+        "appName": get_product_name(release_props['appName'],
+                                    TEMPLATE_KEY_PLATFORMS[release_props['stage_platform']]),
         "appVersion": release_props["appVersion"],
         "branch": release_props["branch"],
         "buildid": release_props["buildid"],
         "extVersion": release_props["appVersion"],
         "hashType": release_props["hashType"],
         "locale": locale if not locale == 'multi' else 'en-US',
-        "platform": release_props["stage_platform"],
+        # XXX: temporary fix for devedition until we solve 1424482
+        "platform": NORMALIZED_BALROG_PLATFORMS.get(release_props["stage_platform"],
+                                                    release_props["stage_platform"]),
         "url_replacements": url_replacements
     }
 
@@ -366,9 +371,10 @@ async def put(context, url, headers, abs_filename, session=None):
 
 # upload_to_s3 {{{1
 async def upload_to_s3(context, s3_key, path):
-    app = context.release_props['appName'].lower()
+    product = get_product_name(context.release_props['appName'].lower(),
+                               TEMPLATE_KEY_PLATFORMS[context.release_props['stage_platform']])
     api_kwargs = {
-        'Bucket': get_bucket_name(context, app),
+        'Bucket': get_bucket_name(context, product),
         'Key': s3_key,
         'ContentType': mimetypes.guess_type(path)[0]
     }

diff --git a/beetmoverscript/templates/devedition_candidates.yml b/beetmoverscript/templates/devedition_candidates.yml
@@ -0,0 +1,194 @@
+---
+metadata:
+    name: "Beet Mover Manifest"
+    description: "Maps DevEdition artifacts to pretty names for en-US"
+    owner: "release@mozilla.com"
+
+s3_bucket_path: pub/devedition/candidates/{{ version }}-candidates/build{{ build_number }}/
+
+mapping:
+{% for locale in ['en-US'] %}
+  {{ locale }}:
+    target.common.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.common.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.common.tests.zip
+    target.cppunittest.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.cppunittest.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.cppunittest.tests.zip
+    target.mochitest.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.mochitest.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.mochitest.tests.zip
+    target.reftest.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.reftest.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.reftest.tests.zip
+    target.talos.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.talos.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.talos.tests.zip
+    target.awsy.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.awsy.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.awsy.tests.zip
+    target.test_packages.json:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.test_packages.json
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.test_packages.json
+    target.web-platform.tests.tar.gz:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.web-platform.tests.tar.gz
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.web-platform.tests.tar.gz
+    target.xpcshell.tests.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.xpcshell.tests.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.xpcshell.tests.zip
+    target.json:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.json
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.json
+    target.mozinfo.json:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
+    target.txt:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.txt
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.txt
+    target.jsshell.zip:
+      s3_key: jsshell/jsshell-{{ platform }}.zip
+      destinations:
+        - jsshell/jsshell-{{ platform }}.zip
+    mozharness.zip:
+      s3_key: {{ platform }}/{{ locale }}/mozharness.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/mozharness.zip
+    target.langpack.xpi:
+      s3_key: {{ platform }}/xpi/{{ locale }}.xpi
+      destinations:
+        - {{ platform }}/xpi/{{ locale }}.xpi
+    target.crashreporter-symbols.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
+    target.complete.mar:
+      s3_key: update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
+      destinations:
+        - update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
+      update_balrog_manifest: true
+    # FIXME: potentially other names here
+    target.bz2.complete.mar:
+      s3_key: update/{{ platform }}/{{ locale }}/firefox-{{ version }}.bz2.complete.mar
+      destinations:
+        - update/{{ platform }}/{{ locale }}/firefox-{{ version }}.bz2.complete.mar
+    # FIXME: potentially other names here
+    target.bz2.complete.mar.asc:
+      s3_key: update/{{ platform }}/{{ locale }}/firefox-{{ version }}.bz2.complete.mar.asc
+      destinations:
+        - update/{{ platform }}/{{ locale }}/firefox-{{ version }}.bz2.complete.mar.asc
+    target.checksums:
+      s3_key: beetmover-checksums/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.beet
+      destinations:
+        - beetmover-checksums/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.beet
+    target.checksums.asc:
+      s3_key: beetmover-checksums/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.asc
+      destinations:
+        - beetmover-checksums/{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.asc
+  {% if platform in ["linux-i686", "linux-x86_64", "mac"] %}
+    mar:
+      s3_key: mar-tools/{{ stage_platform }}/mar
+      destinations:
+        - mar-tools/{{ stage_platform }}/mar
+    mbsdiff:
+      s3_key: mar-tools/{{ stage_platform }}/mbsdiff
+      destinations:
+        - mar-tools/{{ stage_platform }}/mbsdiff
+  {% endif %}
+  {% if platform in ["linux-i686", "linux-x86_64"] %}
+    target.tar.bz2:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
+    target.tar.bz2.asc:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2.asc
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2.asc
+    SOURCE:
+      s3_key: SOURCE
+      destinations:
+        - SOURCE
+    SOURCE.asc:
+      s3_key: SOURCE.asc
+      destinations:
+        - SOURCE.asc
+  {% endif %}
+  {% if platform in ["win32", "win64"] %}
+    mar.exe:
+      s3_key: mar-tools/{{ stage_platform }}/mar.exe
+      destinations:
+        - mar-tools/{{ stage_platform }}/mar.exe
+    mbsdiff.exe:
+      s3_key: mar-tools/{{ stage_platform }}/mbsdiff.exe
+      destinations:
+        - mar-tools/{{ stage_platform }}/mbsdiff.exe
+    target.zip:
+      s3_key: {{ platform }}/{{ locale }}/firefox-{{ version }}.zip
+      destinations:
+        - {{ platform }}/{{ locale }}/firefox-{{ version }}.zip
+    target.installer.exe:
+      s3_key: {{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
+      destinations:
+        - {{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
+  {% endif %}
+  {% if platform in ["linux-i686"] %}
+    target_info.txt:
+      s3_key: linux_info.txt
+      destinations:
+        - linux_info.txt
+  {% endif %}
+  {% if platform in ["linux-x86_64"] %}
+    target_info.txt:
+      s3_key: linux64_info.txt
+      destinations:
+        - linux64_info.txt
+  {% endif %}
+  {% if platform in ["win32"] %}
+    target_info.txt:
+      s3_key: win32_info.txt
+      destinations:
+        - win32_info.txt
+    {% if "esr" not in version %}
+    target.stub-installer.exe:
+      s3_key: {{ platform }}/{{ locale }}/Firefox Installer.exe
+      destinations:
+        - {{ platform }}/{{ locale }}/Firefox Installer.exe
+    {% endif %}
+  {% endif %}
+  {% if platform in ["win64"] %}
+    target_info.txt:
+      s3_key: win64_info.txt
+      destinations:
+        - win64_info.txt
+  {% endif %}
+  {% if platform in ["mac"] %}
+    target_info.txt:
+      s3_key: macosx64_info.txt
+      destinations:
+        - macosx64_info.txt
+    target.dmg:
+      s3_key: {{ platform }}/{{ locale }}/Firefox {{ version }}.dmg
+      destinations:
+        - {{ platform }}/{{ locale }}/Firefox {{ version }}.dmg
+  {% endif %}
+  {% for partial in partials %}
+    {{ partial }}:
+      s3_key: update/{{ platform }}/{{ locale }}/firefox-{{ partials[partial].previousVersion }}-{{ version }}.partial.mar
+      destinations:
+        - update/{{ platform }}/{{ locale }}/firefox-{{ partials[partial].previousVersion }}-{{ version }}.partial.mar
+      update_balrog_manifest: true
+      from_buildid: {{ partials[partial].buildid }}
+  {% endfor %}
+
+{% endfor %}