Bug 1682829 - Automate a fuzzing smoke test in the CI r=decoder

Adds a smoke test script we can use in the CI to smoke test Firefox prior to
bigger tests.

Differential Revision: https://phabricator.services.mozilla.com/D100008

Author: billrest

build/gen_test_packages_manifest.py

@@ -25,6 +25,7 @@
     "updater-dep",
     "jsreftest",
     "perftests",
+    "fuzztest",
 ]
 
 PACKAGE_SPECIFIED_HARNESSES = [
@@ -41,6 +42,7 @@
     "jittest",
     "jsreftest",
     "perftests",
+    "fuzztest",
 ]
 
 # These packages are not present for every build configuration.

python/mozbuild/mozbuild/action/test_archive.py

@@ -98,6 +98,7 @@
                 "jit-test/**",
                 "jittest/**",  # To make the ignore checker happy
                 "perftests/**",
+                "fuzztest/**",
             ],
         },
         {
@@ -670,6 +671,12 @@
             "pattern": "jsreftest/**",
         },
     ],
+    "fuzztest": [
+        {
+            "source": buildconfig.topsrcdir,
+            "pattern": "tools/fuzzing/smoke/**",
+        },
+    ],
     "jittest": [
         {
             "source": buildconfig.topsrcdir,

python/mozbuild/mozbuild/schedules.py

@@ -49,6 +49,7 @@
     "condprofile",
     "cppunittest",
     "firefox-ui",
+    "fuzztest",
     "geckoview-junit",
     "gtest",
     "marionette",

taskcluster/ci/config.yml

@@ -174,6 +174,7 @@ treeherder:
         'GhS': 'GitHub Synchronization'
         'perftest': 'Performance tests'
         'perftest-http3': 'Performance tests with HTTP/3'
+        'fuzzing': 'Fuzzing checks'
 
 index:
     products:

taskcluster/ci/fuzzing/kind.yml

@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - fetch
+    - toolchain
+    - build
+
+job-defaults:
+    platform: linux64-shippable/opt
+    require-build:
+        linux64-shippable/opt: build-linux64-shippable/opt
+    dependencies:
+        linux64-opt: build-linux64/opt
+    worker-type: b-linux
+    worker:
+        max-run-time: 3600
+        docker-image: {in-tree: debian10-amd64-build}
+        env:
+            LD_LIBRARY_PATH: /builds/worker/fetches
+            JSSHELL: ./js
+    treeherder:
+        kind: test
+        tier: 3
+    fetches:
+        build:
+            - target.jsshell.zip
+            - target.fuzztest.tests.tar.gz
+
+jobs:
+    simple:
+        description: Simple Fuzzing Test
+        treeherder:
+            symbol: simple-fuzzing
+        run:
+            using: run-task
+            command: >-
+                cd $MOZ_FETCHES_DIR &&
+                python tools/fuzzing/smoke/smoke.py

taskcluster/docs/kinds.rst

@@ -715,4 +715,9 @@ Push tasks to try to test new scriptworker deployments.
 
 updatebot
 ------------------
-Check for updates to (supported) third party libraries, and manage their lifecycle.
+Check for updates to (supported) third party libraries, and manage their lifecycle.
+
+fuzzing
+-------
+
+Performs fuzzing smoke tests

testing/testsuite-targets.mk

@@ -122,6 +122,7 @@ TEST_PKGS_TARGZ := \
   jsreftest \
   jittest \
   perftests \
+  fuzztest \
   $(NULL)
 
 ifdef LINK_GTEST_DURING_COMPILE

tools/fuzzing/smoke/js.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Hello I am a fake jsshell for testing purpose.
+Add more features!
+"""
+from __future__ import absolute_import
+import argparse
+import sys
+
+
+def run():
+    parser = argparse.ArgumentParser(description="Process some integers.")
+    parser.add_argument("-e", type=str, default=None)
+
+    parser.add_argument("--fuzzing-safe", action="store_true", default=False)
+
+    args = parser.parse_args()
+
+    if args.e is not None:
+        if "crash()" in args.e:
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    run()

tools/fuzzing/smoke/smoke.py

@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""" Smoke test script for Fuzzing
+
+This script can be used to perform simple calls using `jsshell`
+or whatever other tools you may add.
+
+The call is done via `taskcluster/ci/fuzzing/kind.yml` and
+files contained in the `target.jsshell.zip` and `target.fuzztest.tests.tar.gz`
+build artifacts are downloaded to run things.
+
+Everything included in this directory will be added in
+`target.fuzztest.tests.tar.gz` at build time, so you can add more scripts and
+tools if you need. They will be located in `$MOZ_FETCHES_DIR` and follow the
+same directory structure than the source tree.
+"""
+from __future__ import absolute_import
+from distutils.spawn import find_executable
+import os
+import os.path
+import subprocess
+import shlex
+import sys
+
+
+def run_jsshell(command, label=None):
+    """Invokes `jsshell` with command.
+
+    This function will use the `JSSHELL` environment variable,
+    and fallback to a `js` executable if it finds one
+    """
+    shell = os.environ.get("JSSHELL")
+    if shell is None:
+        shell = find_executable("js")
+        if shell is None:
+            raise FileNotFoundError(shell)
+    else:
+        if not os.path.exists(shell) or not os.path.isfile(shell):
+            raise FileNotFoundError(shell)
+
+    if label is None:
+        label = command
+    sys.stdout.write(label)
+    cmd = [shell] + shlex.split(command)
+    sys.stdout.flush()
+    try:
+        subprocess.check_call(cmd)
+    finally:
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+
+
+def smoke_test():
+    # first, let's make sure it catches crashes so we don't have false
+    # positives.
+    try:
+        run_jsshell("-e 'crash();'", "Testing for crash\n")
+    except subprocess.CalledProcessError:
+        pass
+    else:
+        raise Exception("Could not get the process to crash")
+
+    # now let's proceed with some tests
+    run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'", "Simple Fuzzing...")
+
+    # add more smoke tests here
+
+
+if __name__ == "__main__":
+    # if this calls raises an error, the job will turn red in the CI.
+    smoke_test()

tools/fuzzing/smoke/tests.py

@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+import os
+import pytest
+from contextlib import contextmanager
+
+import smoke
+
+JS = os.path.join(os.path.dirname(__file__), "js.py")
+
+
+@contextmanager
+def fake_js():
+    os.environ["JSSHELL"] = JS
+    try:
+        yield
+    finally:
+        del os.environ["JSSHELL"]
+
+
+def test_run_no_jsshell():
+    with pytest.raises(FileNotFoundError):
+        smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'")
+
+
+def test_run_jsshell_set():
+    with fake_js():
+        smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'")
+
+
+def test_smoke_test():
+    with fake_js():
+        smoke.smoke_test()