Nightly build setup

README.md

@@ -4,6 +4,15 @@
 
 A full-featured browser reference implementation using [Mozilla Android Components](https://github.com/mozilla-mobile/android-components).
 
+# Download Nightly builds
+
+Signed Nightly builds can be downloaded from:
+
+* [⬇️ ARM devices (Android 5+)](https://index.taskcluster.net/v1/task/project.mobile.reference-browser.nightly.latest/artifacts/public/reference-browser-arm.apk)
+* [⬇️ x86  devices (Android 5+)](https://index.taskcluster.net/v1/task/project.mobile.reference-browser.nightly.latest/artifacts/public/reference-browser-x86.apk)
+
+Note that all builds are signed with a non-production / throw-away key. The latest Nightly build task can be found [here](https://tools.taskcluster.net/index/project.mobile.reference-browser.nightly/latest).
+
 # Getting Involved
 
 We encourage you to participate in this open source project. We love pull requests, bug reports, ideas, (security) code reviews or any kind of positive contribution.

app/build.gradle

@@ -24,6 +24,9 @@ android {
             minifyEnabled false
             proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
         }
+        debug {
+            applicationIdSuffix ".debug"
+        }
     }
 
     flavorDimensions "engine", "abi"

automation/taskcluster/actions/nightly.sh

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+########################################################################
+# Build nightly, sign it with a throw-away key and attach it as artifact
+# to the taskcluster task.
+########################################################################
+
+# If a command fails then do not proceed and fail this script too.
+set -ex
+
+# Fetch sentry token for crash reporting
+python automation/taskcluster/helper/get-secret.py -s project/mobile/reference-browser/sentry -k dsn -f .sentry_token
+
+# First build and test everything
+./gradlew --no-daemon -PcrashReportEnabled=true clean assembleRelease test
+
+# Fetch preview/throw-away key from secrets service
+python automation/taskcluster/helper/get-secret.py -s project/mobile/reference-browser/preview-key-store -k keyStoreFile -f .store --decode
+python automation/taskcluster/helper/get-secret.py -s project/mobile/reference-browser/preview-key-store -k keyStorePassword -f .store_token
+python automation/taskcluster/helper/get-secret.py -s project/mobile/reference-browser/preview-key-store -k keyPassword -f .key_token
+
+# Sign APKs with preview/throw-away key
+python automation/taskcluster/helper/sign-builds.py --zipalign --path ./app/build/outputs/apk --store .store --store-token .store_token --key-alias preview-key --key-token .key_token --archive ./preview
+
+# Copy release APKs to separate folder for attaching as artifacts
+mkdir release
+cp preview/app-geckoNightly-arm-armeabi-v7a-release-signed-aligned.apk release/reference-browser-arm.apk
+cp preview/app-geckoNightly-x86-x86-release-signed-aligned.apk release/reference-browser-x86.apk

automation/taskcluster/decision_task_nightly.py

@@ -0,0 +1,97 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Decision task for releases
+"""
+
+from __future__ import print_function
+import datetime
+import json
+import os
+import taskcluster
+
+import lib.tasks
+
+TASK_ID = os.environ.get('TASK_ID')
+HEAD_REV = os.environ.get('MOBILE_HEAD_REV')
+
+def generate_build_task():
+    created = datetime.datetime.now()
+    expires = taskcluster.fromNow('1 year')
+    deadline = taskcluster.fromNow('1 day')
+
+    command = "automation/taskcluster/actions/nightly.sh"
+
+    return {
+        "workerType": 'gecko-focus',
+        "taskGroupId": TASK_ID,
+        "expires": taskcluster.stringDate(expires),
+        "retries": 5,
+        "created": taskcluster.stringDate(created),
+        "tags": {},
+        "priority": "lowest",
+        "schedulerId": "focus-nightly-sched",
+        "deadline": taskcluster.stringDate(deadline),
+        "dependencies": [ TASK_ID ],
+        "routes": [
+            "index.project.mobile.reference-browser.nightly.latest"
+        ],
+        "scopes": [
+            "queue:route:index.project.mobile.reference-browser.nightly.*",
+            "secrets:get:project/mobile/reference-browser/preview-key-store",
+            "secrets:get:project/mobile/reference-browser/sentry"
+        ],
+        "requires": "all-completed",
+        "payload": {
+            "features": {
+                'taskclusterProxy': True
+            },
+            "maxRunTime": 7200,
+            "image": "mozillamobile/android-components:1.9",
+            "command": [
+                "/bin/bash",
+                "--login",
+                "-cx",
+                "cd .. && git clone https://github.com/mozilla-mobile/reference-browser.git && cd reference-browser && %s" % (command)
+            ],
+            "artifacts": {
+                "public": {
+                    "type": "directory",
+                    "path": "/build/reference-browser/release",
+                    "expires": taskcluster.stringDate(expires)
+                }
+            },
+            "env": {
+                "TASK_GROUP_ID": TASK_ID
+            }
+        },
+        "provisionerId": "aws-provisioner-v1",
+        "metadata": {
+            "name": "build",
+            "description": "Building reference browser nightly",
+            "owner": "skaspari@mozilla.com",
+            "source": "https://github.com/mozilla-mobile/android-components"
+        }
+    }
+
+def nightly():
+    queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
+
+    task_graph = {}
+    build_task_id = taskcluster.slugId()
+    build_task = generate_build_task()
+    lib.tasks.schedule_task(queue, build_task_id, build_task)
+
+    task_graph[build_task_id] = {}
+    task_graph[build_task_id]["task"] = queue.task(build_task_id)
+
+    print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
+
+    task_graph_path = "task-graph.json"
+    with open(task_graph_path, 'w') as f:
+        json.dump(task_graph, f)
+
+if __name__ == "__main__":
+    nightly()

automation/taskcluster/helper/get-secret.py

@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import base64
+import os
+import taskcluster
+
+def write_secret_to_file(path, data, key, base64decode=False, append=False, prefix=''):
+    path = os.path.join(os.path.dirname(__file__), '../../../' + path)
+    with open(path, 'a' if append else 'w') as f:
+        value = data['secret'][key]
+        if base64decode:
+            value = base64.b64decode(value)
+        f.write(prefix + value)
+
+
+def fetch_secret_from_taskcluster(name):
+    secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
+    return secrets.get(name)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Fetch a taskcluster secret value and save it to a file.')
+
+    parser.add_argument('-s', dest="secret", action="store", help="name of the secret")
+    parser.add_argument('-k', dest='key', action="store", help='key of the secret')
+    parser.add_argument('-f', dest="path", action="store", help='file to save secret to')
+    parser.add_argument('--decode', dest="decode", action="store_true", default=False, help='base64 decode secret before saving to file')
+    parser.add_argument('--append', dest="append", action="store_true", default=False, help='append secret to existing file')
+    parser.add_argument('--prefix', dest="prefix", action="store", default="", help='add prefix when writing secret to file')
+
+    result = parser.parse_args()
+
+    secret = fetch_secret_from_taskcluster(result.secret)
+    write_secret_to_file(result.path, secret, result.key, result.decode, result.append, result.prefix)
+
+
+if __name__ == "__main__":
+    main()

automation/taskcluster/helper/sign-builds.py

@@ -0,0 +1,89 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import fnmatch
+import os
+import subprocess
+
+
+def collect_apks(path, pattern):
+    matches = []
+    for root, dirnames, filenames in os.walk(path):
+        for filename in fnmatch.filter(filenames, pattern):
+            matches.append(os.path.join(root, filename))
+    return matches
+
+
+def zipalign(path):
+    unsigned_apks = collect_apks(path, '*-unsigned.apk')
+    print("Found {apk_count} APK(s) to zipalign in {path}".format(apk_count=len(unsigned_apks), path=path))
+    for apk in unsigned_apks:
+        print("Zipaligning", apk)
+        split = os.path.splitext(apk)
+        print(subprocess.check_output(["zipalign", "-f", "-v", "-p", "4", apk, split[0] + "-aligned" + split[1]]))
+
+
+def sign(path, store, store_token, key_alias, key_token):
+    unsigned_apks = collect_apks(path, '*-aligned.apk')
+    print("Found {apk_count} APK(s) to sign in {path}".format(apk_count=len(unsigned_apks), path=path))
+
+    for apk in unsigned_apks:
+        print("Signing", apk)
+        print(subprocess.check_output([
+            "apksigner", "sign",
+            "--ks", store,
+            "--ks-key-alias", key_alias,
+            "--ks-pass", "file:%s" % store_token,
+            "--key-pass", "file:%s" % key_token,
+            "-v",
+            "--out", apk.replace('unsigned', 'signed'), apk]))
+
+
+def archive_result(path, archive):
+    if not os.path.exists(archive):
+        os.makedirs(archive)
+
+    signed_apks = collect_apks(path, '*-signed-*.apk')
+    print("Found {apk_count} APK(s) to archive in {path}".format(apk_count=len(signed_apks), path=path))
+
+    for apk in signed_apks:
+        print("Verifying", apk)
+        print(subprocess.check_output(['apksigner', 'verify', apk]))
+
+        destination = archive + "/" + os.path.basename(apk)
+        print("Archiving", apk)
+        print(" `->", destination)
+        os.rename(apk, destination)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Zipaligns, signs and archives APKs')
+    parser.add_argument('--path', dest="path", action="store", help='Root path to search for APK files')
+    parser.add_argument('--zipalign', dest="zipalign", action="store_true", default=False,
+                        help='Zipaligns APKs before signing')
+    parser.add_argument('--archive', metavar="PATH", dest="archive", action="store", default=False,
+                        help='Path to save sign APKs to')
+
+    parser.add_argument('--store', metavar="PATH", dest="store", action="store", help='Path to keystore')
+    parser.add_argument('--store-token', metavar="PATH", dest="store_token", action="store",
+                        help='Path to keystore password file')
+    parser.add_argument('--key-alias', metavar="ALIAS", dest="key_alias", action="store", help='Key alias')
+    parser.add_argument('--key-token', metavar="PATH", dest="key_token", action="store",
+                        help='Path to key password file')
+
+    result = parser.parse_args()
+
+    if result.zipalign:
+        zipalign(result.path)
+
+    sign(result.path, result.store, result.store_token, result.key_alias, result.key_token)
+
+    if result.archive:
+        archive_result(result.path, result.archive)
+
+
+if __name__ == "__main__":
+    main()