Nightly automation (without pushapk)

.taskcluster.yml

@@ -0,0 +1,99 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+version: 1
+policy:
+  pullRequests: public
+tasks:
+  - $if: 'tasks_for == "cron"'
+    then:
+      $let:
+        decision_task_id: {$eval: as_slugid("decision_task")}
+        expires_in: {$fromNow: '1 year'}
+        repository: ${event.repository.clone_url}
+        scheduler_id: focus-nightly-sched
+        is_mozilla_mobile_repo:
+          $eval: event.repository.clone_url == 'https://github.com/mozilla-mobile/fenix'
+        track:
+          $if: event.repository.clone_url == 'https://github.com/mozilla-mobile/fenix'
+          then: 'nightly'
+          else: 'staging-nightly'
+      in:
+        taskId: ${decision_task_id}
+        taskGroupId: ${decision_task_id}  # Must be explicit because of Chain of Trust
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '2 hours'}
+        expires: ${expires_in}
+        schedulerId: ${scheduler_id}    # Must be explicit because of Chain of Trust
+        provisionerId: aws-provisioner-v1
+        workerType: gecko-focus   # This workerType has ChainOfTrust enabled
+        requires: all-completed   # Must be explicit because of Chain of Trust
+        priority: medium
+        retries: 5
+        scopes:
+          $flatten:
+            - queue:scheduler-id:${scheduler_id}
+            - queue:create-task:highest:aws-provisioner-v1/gecko-focus
+            - project:mobile:fenix:releng:signing:format:autograph_apk
+            - $if: is_mozilla_mobile_repo
+              then:
+                - queue:create-task:highest:scriptworker-prov-v1/mobile-signing-v1
+                - queue:create-task:highest:scriptworker-prov-v1/mobile-pushapk-v1
+                - project:mobile:fenix:releng:signing:cert:release-signing
+                - project:mobile:fenix:releng:googleplay:product:fenix
+                - queue:route:index.project.mobile.fenix.signed-nightly.*
+              else:
+                - queue:create-task:highest:scriptworker-prov-v1/mobile-signing-dep-v1
+                - queue:create-task:highest:scriptworker-prov-v1/mobile-pushapk-dep-v1
+                - project:mobile:fenix:releng:signing:cert:dep-signing
+                - project:mobile:fenix:releng:googleplay:product:fenix:dep
+                - queue:route:index.project.mobile.fenix.staging-signed-nightly.*
+        payload:
+          maxRunTime: 600   # Decision should remain fast enough to schedule a handful of tasks
+          image: mozillamobile/fenix:1.3
+          features:
+            taskclusterProxy: true
+            chainOfTrust: true
+          env:
+            TASK_ID: ${decision_task_id}
+            SCHEDULER_ID: ${scheduler_id}
+            MOBILE_HEAD_REPOSITORY: ${repository}
+            MOBILE_HEAD_BRANCH: ${event.release.target_commitish}
+            MOBILE_HEAD_REV: ${event.release.tag_name}
+            MOBILE_TRIGGERED_BY: ${event.sender.login}
+          command:
+            - /bin/bash
+            - --login
+            - -cx
+            - >-
+              cd ..
+              && git clone ${repository} repository
+              && cd repository
+              && python automation/taskcluster/decision_task_nightly.py \
+                --track ${track} \
+                --commit \
+                --output /opt/repository/app/build/outputs/apk \
+                --apk arm/release/app-arm-release-unsigned.apk \
+                --apk x86/release/app-x86-release-unsigned.apk \
+                --date ${now}
+          artifacts:
+            public/task-graph.json:
+              type: file
+              path: /opt/repository/task-graph.json
+              expires: ${expires_in}
+            public/actions.json:
+              type: file
+              path: /opt/repository/actions.json
+              expires: ${expires_in}
+            public/parameters.yml:
+              type: file
+              path: /opt/repository/parameters.yml
+              expires: ${expires_in}
+        extra:
+          cron: {$json: {$eval: 'cron'}}
+          tasks_for: ${tasks_for}
+        metadata:
+          name: Fenix Nightly Decision Task
+          description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
+          owner: ${event.sender.login}@users.noreply.github.com
+          source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml

app/build.gradle

@@ -43,11 +43,11 @@ android.applicationVariants.all { variant ->
         // same version code. Therefore we need to have different version codes for our ARM and x86
         // builds.
 
-        // Our generated version code now has a length of 9 (See tools/gradle/versionCode.gradle).
+        // Our generated version code now has a length of 9 (See automation/gradle/versionCode.gradle).
         // Our x86 builds need a higher version code to avoid installing ARM builds on an x86 device
         // with ARM compatibility mode.
 
-        if (variant.flavorName.contains("X86")) {
+        if (variant.flavorName.contains("x86")) {
             versionCode = versionCode + 1
         }// else variant.flavorName.contains("Arm")) use generated version code
 

automation/gradle/versionCode.gradle

@@ -37,7 +37,8 @@ ext {
     // minute.
     def time = new SimpleDateFormat("HHmm").format(today)
 
-    generatedVersionCode = (year + day + time) as int
+    // todo after first build is deployed to Google Play generatedVersionCode = (year + day + time) as int
+    generatedVersionCode = 1
 
     println("Generated versionCode: $generatedVersionCode")
     println()

automation/taskcluster/decision_task_nightly.py

@@ -0,0 +1,152 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Decision task for nightly releases.
+"""
+
+from __future__ import print_function
+
+import argparse
+import arrow
+import json
+import lib.tasks
+import os
+import taskcluster
+
+TASK_ID = os.environ.get('TASK_ID')
+SCHEDULER_ID = os.environ.get('SCHEDULER_ID')
+GITHUB_HTTP_REPOSITORY = os.environ.get('MOBILE_HEAD_REPOSITORY')
+HEAD_REV = os.environ.get('MOBILE_HEAD_REV')
+
+BUILDER = lib.tasks.TaskBuilder(
+    task_id=TASK_ID,
+    owner="android-components-team@mozilla.com",
+    source='{}/raw/{}/.taskcluster.yml'.format(GITHUB_HTTP_REPOSITORY, HEAD_REV),
+    scheduler_id=SCHEDULER_ID
+)
+
+
+def generate_build_task(apks):
+    artifacts = {'public/{}'.format(os.path.basename(apk)): {
+        "type": 'file',
+        "path": apk,
+        "expires": taskcluster.stringDate(taskcluster.fromNow('1 year')),
+    } for apk in apks}
+
+    checkout = 'git clone {} repository && cd repository'.format(GITHUB_HTTP_REPOSITORY)
+
+    return taskcluster.slugId(), BUILDER.build_task(
+        name="(Fenix) Build task",
+        description="Build Fenix from source code.",
+        command='cd .. && {} && ./gradlew --no-daemon clean test assembleRelease'.format(checkout),
+        features={
+            "chainOfTrust": True,
+            "taskClusterProxy": True
+        },
+        artifacts=artifacts
+    )
+
+
+def generate_signing_task(build_task_id, apks, date, is_staging):
+    artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
+
+    signing_format = 'autograph_apk'
+    index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly'
+    routes = [
+        "index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(index_release, date.year, date.month, date.day),
+        "index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format(index_release, date.year, date.month, date.day, HEAD_REV),
+        "index.project.mobile.fenix.{}.nightly.latest".format(index_release),
+    ]
+    scopes = [
+        "project:mobile:fenix:releng:signing:format:{}".format(signing_format),
+        "project:mobile:fenix:releng:signing:cert:{}".format('dep-signing' if is_staging else 'release-signing')
+    ]
+
+    return taskcluster.slugId(), BUILDER.build_signing_task(
+        build_task_id,
+        name="(Fenix) Signing task",
+        description="Sign release builds of Fenix",
+        apks=artifacts,
+        scopes=scopes,
+        routes=routes,
+        signing_format=signing_format,
+        is_staging=is_staging
+    )
+
+
+def generate_push_task(signing_task_id, apks, commit, is_staging):
+    artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
+
+    return taskcluster.slugId(), BUILDER.build_push_task(
+        signing_task_id,
+        name="(Fenix) Push task",
+        description="Upload signed release builds of Fenix to Google Play",
+        apks=artifacts,
+        scopes=[
+            "project:mobile:fenix:releng:googleplay:product:fenix{}".format(':dep' if is_staging else '')
+        ],
+        commit=commit,
+        is_staging=is_staging
+    )
+
+
+def populate_chain_of_trust_required_but_unused_files():
+    # These files are needed to keep chainOfTrust happy. However, they have no need for Reference Browser
+    # at the moment. For more details, see: https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
+
+    for file_name in ('actions.json', 'parameters.yml'):
+        with open(file_name, 'w') as f:
+            json.dump({}, f)
+
+
+def nightly(apks, track, commit, date_string):
+    queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
+    date = arrow.get(date_string)
+    is_staging = track == 'staging-nightly'
+
+    task_graph = {}
+
+    build_task_id, build_task = generate_build_task(apks)
+    lib.tasks.schedule_task(queue, build_task_id, build_task)
+
+    task_graph[build_task_id] = {}
+    task_graph[build_task_id]['task'] = queue.task(build_task_id)
+
+    sign_task_id, sign_task = generate_signing_task(build_task_id, apks, date, is_staging)
+    lib.tasks.schedule_task(queue, sign_task_id, sign_task)
+
+    task_graph[sign_task_id] = {}
+    task_graph[sign_task_id]['task'] = queue.task(sign_task_id)
+
+    # TODO enable once first apk uploaded to Google Play manually
+    # push_task_id, push_task = generate_push_task(sign_task_id, apks, commit, is_staging)
+    # lib.tasks.schedule_task(queue, push_task_id, push_task)
+    #
+    # task_graph[push_task_id] = {}
+    # task_graph[push_task_id]['task'] = queue.task(push_task_id)
+
+    print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
+
+    with open('task-graph.json', 'w') as f:
+        json.dump(task_graph, f)
+
+    populate_chain_of_trust_required_but_unused_files()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description='Create a release pipeline (build, sign, publish) on taskcluster.')
+
+    parser.add_argument('--track', dest="track", action="store", choices=['nightly', 'staging-nightly'], required=True)
+    parser.add_argument('--commit', action="store_true", help="commit the google play transaction")
+    parser.add_argument('--apk', dest="apks", metavar="path", action="append", help="Path to APKs to sign and upload",
+                        required=True)
+    parser.add_argument('--output', metavar="path", action="store", help="Path to the build output",
+                        required=True)
+    parser.add_argument('--date', action="store", help="ISO8601 timestamp for build")
+
+    result = parser.parse_args()
+    apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
+    nightly(apks, result.track, result.commit, result.date)