add EMULATED_ANDROID platform (#2267)

* add EMULATED_ANDROID platform

This allows downloading an emulator bundle (from the
ANDROID_EMULATOR_BUCKET_PATH variable) and running it (using the
ANDROID_EMULATOR_SCRIPT_PATH variable) before fuzzing.

The emulated Android instance is controlled using 'adb', just as a
physical Android device would. The new EmulatedAndroidLibFuzzerRunner
shares most logic with the existing AndroidLibFuzzerRunner.

* fix import order

* fix formatting using 'butler.py format'

* fix lint errors

* resolve first round of feedback

* change execute_command() to run_command() for 'adb devices' call

* move emulator startup to build_manager.py

This also scans the stdout of the emulator process to get the expected
device serial, which is determined at runtime. This allows us to set the
ANDROID_SERIAL environment variable.

* fix test errors from unused imports

* wait for emulated device to come online

* bail out early from functions when using emulated Android device

* add case to wait_until_fully_booted() for emulators

* fix lint error - unused import

* clean up platform check, check for correct env var

configs/test/pubsub/queues.yaml

@@ -44,6 +44,8 @@ resources:
     type: queue.jinja
   - name: jobs-android-x86
     type: queue.jinja
+  - name: jobs-android-emulator
+    type: queue.jinja
   - name: high-end-jobs-android-x86
     type: queue.jinja
   - name: jobs-android-local-devices

src/python/base/tasks.py

@@ -63,6 +63,7 @@
     'ANDROID_KERNEL': 'Android Kernel',
     'ANDROID_AUTO': 'Android Auto',
     'ANDROID_X86': 'Android (x86)',
+    'ANDROID_EMULATOR': 'Android (Emulated)',
     'CHROMEOS': 'Chrome OS',
     'FUCHSIA': 'Fuchsia OS',
     'MAC': 'Mac',

src/python/bot/tasks/update_task.py

@@ -170,7 +170,10 @@ def run_platform_init_scripts():
   logs.log('Running platform initialization scripts.')
 
   plt = environment.platform()
-  if environment.is_android():
+  if environment.is_android_emulator():
+    # Nothing to do here since emulator is not started yet.
+    pass
+  elif environment.is_android():
     android_init.run()
   elif plt == 'CHROMEOS':
     chromeos_init.run()

src/python/build_management/build_manager.py

@@ -791,6 +791,19 @@ def setup(self):
     return result
 
 
+class AndroidEmulatorBuild(RegularBuild):
+  """Represents an Android Emulator build."""
+
+  def setup(self):
+    """Android Emulator build setup."""
+    emu_proc = android.emulator.EmulatorProcess()
+    emu_proc.create()
+    emu_proc.run()
+    android.adb.run_as_root()
+
+    return super().setup()
+
+
 class SymbolizedBuild(Build):
   """Symbolized build."""
 
@@ -1252,6 +1265,8 @@ def setup_regular_build(revision,
     build_class = build_setup_host.RemoteRegularBuild
   elif environment.platform() == 'FUCHSIA':
     build_class = FuchsiaBuild
+  elif environment.is_android_emulator():
+    build_class = AndroidEmulatorBuild
 
   build = build_class(
       base_build_dir,

src/python/crash_analysis/stack_parsing/stack_analyzer.py

@@ -27,8 +27,8 @@ def linkify_kernel_or_lkl_stacktrace_if_needed(crash_info):
   """Linkify Android Kernel or lkl stacktrace."""
   kernel_prefix = ''
   kernel_hash = ''
-  if environment.is_android() and (crash_info.found_android_kernel_crash or
-                                   crash_info.is_kasan):
+  if (environment.is_android() and not environment.is_android_emulator() and
+      (crash_info.found_android_kernel_crash or crash_info.is_kasan)):
     kernel_prefix, kernel_hash = \
       android_kernel.get_kernel_prefix_and_full_hash()
 

src/python/platforms/android/__init__.py

@@ -18,6 +18,7 @@
 from . import battery
 from . import constants
 from . import device
+from . import emulator
 from . import fetch_artifact
 from . import flash
 from . import gestures

src/python/platforms/android/adb.py

@@ -134,7 +134,7 @@ def run():
   thread = threading.Thread(target=run)
   thread.start()
   thread.join(timeout)
-  if thread.isAlive():
+  if thread.is_alive():
     try:
       pipe.kill()
     except OSError:
@@ -149,7 +149,7 @@ def run():
 
 def factory_reset():
   """Reset device to factory state."""
-  if is_gce():
+  if is_gce() or environment.is_android_emulator():
     # We cannot recover from this since there can be cases like userdata image
     # corruption in /data/data. Till the bug is fixed, we just need to wait
     # for reimage in next iteration.
@@ -561,7 +561,7 @@ def run_command(cmd,
     timeout = ADB_TIMEOUT
 
   output = execute_command(get_adb_command_line(cmd), timeout, log_error)
-  if not recover:
+  if not recover or environment.is_android_emulator():
     if log_output:
       logs.log('Output: (%s)' % output)
     return output
@@ -734,6 +734,11 @@ def package_manager_ready():
     if is_drive_ready and is_package_manager_ready and is_boot_completed:
       return True
 
+    # is_boot_completed and is_package_manager_ready may never happen on
+    # emulated devices.
+    if is_drive_ready and environment.is_android_emulator():
+      return True
+
     time.sleep(BOOT_WAIT_INTERVAL)
 
   factory_reset()

src/python/platforms/android/emulator.py

@@ -0,0 +1,104 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helper functions for running commands on emulated Android devices."""
+
+import os
+import re
+import subprocess
+import tempfile
+
+from google_cloud_utils import storage
+from metrics import logs
+from platforms.android import adb
+from system import archive
+from system import environment
+from system import new_process
+from system import shell
+
+# Output pattern to parse stdout for serial number
+DEVICE_SERIAL_RE = re.compile(r'DEVICE_SERIAL: (.+)')
+
+
+class EmulatorError(Exception):
+  """Error for errors handling the Android emulator."""
+
+
+class EmulatorProcess(object):
+  """A EmulatorProcess encapsulates the creation, running, and destruction
+  of Android emulator processes."""
+
+  def __init__(self):
+    self.process_runner = None
+    self.process = None
+    self.logfile = None
+
+    log_path = os.path.join(tempfile.gettempdir(), 'android-emulator.log')
+    self.logfile = open(log_path, 'wb')
+
+  def create(self):
+    """Configures a emulator process which can subsequently be `run`."""
+    # Download emulator image.
+    if not environment.get_value('ANDROID_EMULATOR_BUCKET_PATH'):
+      logs.log_error('ANDROID_EMULATOR_BUCKET_PATH is not set.')
+      return
+    temp_directory = environment.get_value('BOT_TMPDIR')
+    archive_src_path = environment.get_value('ANDROID_EMULATOR_BUCKET_PATH')
+    archive_dst_path = os.path.join(temp_directory, 'emulator_bundle.zip')
+    storage.copy_file_from(archive_src_path, archive_dst_path)
+
+    # Extract emulator image.
+    self.emulator_path = os.path.join(temp_directory, 'emulator')
+    archive.unpack(archive_dst_path, self.emulator_path)
+    shell.remove_file(archive_dst_path)
+
+    # Run emulator.
+    script_path = os.path.join(self.emulator_path, 'run')
+    self.process_runner = new_process.ProcessRunner(script_path)
+
+  def run(self):
+    """Actually runs a emulator, assuming `create` has already been called."""
+    if not self.process_runner:
+      raise EmulatorError('Attempted to `run` emulator before calling `create`')
+
+    logs.log('Starting emulator.')
+    self.process = self.process_runner.run(
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    device_serial = None
+    while not device_serial:
+      line = self.process.popen.stdout.readline().decode()
+      match = DEVICE_SERIAL_RE.match(line)
+      if match:
+        device_serial = match.group(1)
+
+    logs.log('Found serial ID: %s.' % device_serial)
+    environment.set_value('ANDROID_SERIAL', device_serial)
+
+    logs.log('Waiting on device')
+    adb.wait_until_fully_booted()
+    logs.log('Device is online')
+
+  def kill(self):
+    """ Kills the currently-running emulator, if there is one. """
+    if self.process:
+      logs.log('Stopping emulator.')
+      self.process.kill()
+      self.process = None
+
+    if self.logfile:
+      self.logfile.close()
+      self.logfile = None
+
+    if self.emulator_path:
+      shell.remove_directory(self.emulator_path)

src/python/system/environment.py

@@ -200,7 +200,7 @@ def get_asan_options(redzone_size, malloc_context_size, quarantine_size_mb,
 
 def get_cpu_arch():
   """Return cpu architecture."""
-  if is_android():
+  if is_android() and not is_android_emulator():
     # FIXME: Handle this import in a cleaner way.
     from platforms import android
     return android.settings.get_cpu_arch()
@@ -470,6 +470,8 @@ def get_msan_options():
 def get_platform_id():
   """Return a platform id as a lowercase string."""
   bot_platform = platform()
+  if is_android_emulator():
+    return bot_platform.lower()
   if is_android(bot_platform):
     # FIXME: Handle this import in a cleaner way.
     from platforms import android
@@ -1054,6 +1056,11 @@ def is_android(plt=None):
   return 'ANDROID' in (plt or platform())
 
 
+def is_android_emulator(plt=None):
+  """Return true if we are on android emulator platform."""
+  return 'ANDROID_EMULATOR' == (plt or platform())
+
+
 def is_lib():
   """Whether or not we're in libClusterFuzz."""
   return get_value('ROOT_DIR') is None

src/python/system/shell.py

@@ -144,7 +144,7 @@ def clear_data_directories_on_low_disk_space():
 
 def clear_device_temp_directories():
   """Clear device specific temp directories."""
-  if environment.is_android():
+  if environment.is_android() and environment.get_value('ANDROID_SERIAL'):
     from platforms import android
     android.device.clear_temp_directories()
 
@@ -216,7 +216,7 @@ def clear_testcase_directories():
   remove_directory(environment.get_value('FUZZ_INPUTS'), recreate=True)
   remove_directory(environment.get_value('FUZZ_INPUTS_DISK'), recreate=True)
 
-  if environment.is_android():
+  if environment.is_android() and environment.get_value('ANDROID_SERIAL'):
     from platforms import android
     android.device.clear_testcase_directory()
   if environment.platform() == 'FUCHSIA':