android: Switch to GLES layer on Android Q

renderdoc/android/android.cpp

@@ -403,6 +403,12 @@ bool RemoveRenderDocAndroidServer(const std::string &deviceID)
 void ResetCaptureSettings(const std::string &deviceID)
 {
   Android::adbExecCommand(deviceID, "shell setprop debug.vulkan.layers :", ".", true);
+  Android::adbExecCommand(deviceID, "shell settings delete global enable_gpu_debug_layers", ".",
+                          true);
+  Android::adbExecCommand(deviceID, "shell settings delete global gpu_debug_app", ".", true);
+  Android::adbExecCommand(deviceID, "shell settings delete global gpu_debug_layer_app", ".", true);
+  Android::adbExecCommand(deviceID, "shell settings delete global gpu_debug_layers", ".", true);
+  Android::adbExecCommand(deviceID, "shell settings delete global gpu_debug_layers_gles", ".", true);
 }
 
 rdcarray<rdcstr> EnumerateDevices()
@@ -937,6 +943,7 @@ struct AndroidController : public IDeviceProtocolHandler
     return &m_Inst;
   };
 };
+
 ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w, const char *c,
                                                     const rdcarray<EnvironmentModification> &env,
                                                     const CaptureOptions &opts)
@@ -977,9 +984,41 @@ ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w
     Android::adbExecCommand(m_deviceID, StringFormat::Fmt("forward --remove tcp:%i", jdwpPort));
     // force stop the package if it was running before
     Android::adbExecCommand(m_deviceID, "shell am force-stop " + packageName);
-    // enable the vulkan layer (will only be used by vulkan programs)
-    Android::adbExecCommand(m_deviceID,
-                            "shell setprop debug.vulkan.layers " RENDERDOC_VULKAN_LAYER_NAME);
+
+    bool hookWithJDWP = true;
+
+    if(Android::SupportsNativeLayers(m_deviceID))
+    {
+      RDCLOG("Using Android 10 native GPU layering");
+
+      // if we have Android 10 native layering, don't use JDWP hooking
+      hookWithJDWP = false;
+
+      // set up environment variables for the package, and point to ourselves for vulkan and GLES
+      // layers
+      std::string installedABI = Android::DetermineInstalledABI(m_deviceID, packageName);
+      std::string layerPackage = GetRenderDocPackageForABI(Android::GetABI(installedABI));
+      Android::adbExecCommand(m_deviceID, "shell settings put global enable_gpu_debug_layers 1");
+      Android::adbExecCommand(m_deviceID, "shell settings put global gpu_debug_app " + packageName);
+      Android::adbExecCommand(m_deviceID,
+                              "shell settings put global gpu_debug_layer_app " + layerPackage);
+      Android::adbExecCommand(
+          m_deviceID, "shell settings put global gpu_debug_layers " RENDERDOC_VULKAN_LAYER_NAME);
+      Android::adbExecCommand(
+          m_deviceID, "shell settings put global gpu_debug_layers_gles " RENDERDOC_ANDROID_LIBRARY);
+    }
+    else
+    {
+      RDCLOG("Using pre-Android 10 Vulkan layering and JDWP injection");
+
+      // use JDWP hooking to inject our library for GLES
+      hookWithJDWP = true;
+
+      // enable the vulkan layer (will only be used by vulkan programs)
+      Android::adbExecCommand(m_deviceID,
+                              "shell setprop debug.vulkan.layers " RENDERDOC_VULKAN_LAYER_NAME);
+    }
+
     // if in VR mode, enable frame delimiter markers
     Android::adbExecCommand(m_deviceID, "shell setprop debug.vr.profiler 1");
     // create the data directory we will use for storing, in case the application doesn't
@@ -1007,8 +1046,6 @@ ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w
     if(RDCLib.find("/lib/*/" RENDERDOC_ANDROID_LIBRARY) != std::string::npos)
       RDCLib.clear();
 
-    bool injectLibraries = true;
-
     if(RDCLib.empty())
     {
       RDCLOG("No library found in %s/lib/*/" RENDERDOC_ANDROID_LIBRARY
@@ -1017,15 +1054,15 @@ ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w
     }
     else
     {
-      injectLibraries = false;
+      hookWithJDWP = false;
       RDCLOG("Library found, no injection required: %s", RDCLib.c_str());
     }
 
     int pid = 0;
 
     RDCLOG("Launching package '%s' with activity '%s'", packageName.c_str(), activityName.c_str());
 
-    if(injectLibraries)
+    if(hookWithJDWP)
     {
       RDCLOG("Setting up to launch the application as a debugger to inject.");
 
@@ -1043,8 +1080,7 @@ ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w
     }
     else
     {
-      RDCLOG(
-          "Not doing any injection - assuming APK is pre-loaded with RenderDoc capture library.");
+      RDCLOG("Launching APK with no debugger or direct injection.");
 
       // start the activity in this package with debugging enabled and force-stop after starting
       Android::adbExecCommand(m_deviceID,

renderdoc/android/android_patch.cpp

@@ -390,37 +390,6 @@ bool CheckPatchingRequirements()
   return true;
 }
 
-std::string DetermineInstalledABI(const std::string &deviceID, const std::string &packageName)
-{
-  RDCLOG("Checking installed ABI for %s", packageName.c_str());
-  std::string abi;
-
-  std::string dump = adbExecCommand(deviceID, "shell pm dump " + packageName).strStdout;
-  if(dump.empty())
-    RDCERR("Unable to pm dump %s", packageName.c_str());
-
-  // Walk through the output and look for primaryCpuAbi
-  std::istringstream contents(dump);
-  std::string line;
-  std::string prefix("primaryCpuAbi=");
-  while(std::getline(contents, line))
-  {
-    line = trim(line);
-    if(line.compare(0, prefix.size(), prefix) == 0)
-    {
-      // Extract the abi
-      abi = line.substr(line.find_last_of("=") + 1);
-      RDCLOG("primaryCpuAbi found: %s", abi.c_str());
-      break;
-    }
-  }
-
-  if(abi.empty())
-    RDCERR("Unable to determine installed abi for: %s", packageName.c_str());
-
-  return abi;
-}
-
 bool PullAPK(const std::string &deviceID, const std::string &pkgPath, const std::string &apk)
 {
   RDCLOG("Pulling APK to patch");

renderdoc/android/android_utils.cpp

@@ -24,6 +24,7 @@
 
 #include "android_utils.h"
 #include <algorithm>
+#include <sstream>
 #include "core/core.h"
 #include "strings/string_utils.h"
 
@@ -183,6 +184,51 @@ bool IsSupported(std::string deviceID)
   return true;
 }
 
+bool SupportsNativeLayers(const rdcstr &deviceID)
+{
+  std::string api =
+      trim(Android::adbExecCommand(deviceID, "shell getprop ro.build.version.sdk").strStdout);
+
+  int apiVersion = atoi(api.c_str());
+
+  // SDK 29 == Android 10.0, the first version that included layering
+  if(apiVersion >= 29)
+    return true;
+
+  return false;
+}
+
+std::string DetermineInstalledABI(const std::string &deviceID, const std::string &packageName)
+{
+  RDCLOG("Checking installed ABI for %s", packageName.c_str());
+  std::string abi;
+
+  std::string dump = adbExecCommand(deviceID, "shell pm dump " + packageName).strStdout;
+  if(dump.empty())
+    RDCERR("Unable to pm dump %s", packageName.c_str());
+
+  // Walk through the output and look for primaryCpuAbi
+  std::istringstream contents(dump);
+  std::string line;
+  std::string prefix("primaryCpuAbi=");
+  while(std::getline(contents, line))
+  {
+    line = trim(line);
+    if(line.compare(0, prefix.size(), prefix) == 0)
+    {
+      // Extract the abi
+      abi = line.substr(line.find_last_of("=") + 1);
+      RDCLOG("primaryCpuAbi found: %s", abi.c_str());
+      break;
+    }
+  }
+
+  if(abi.empty())
+    RDCERR("Unable to determine installed abi for: %s", packageName.c_str());
+
+  return abi;
+}
+
 rdcstr GetFriendlyName(const rdcstr &deviceID)
 {
   // run adb root now, so we hit any disconnection that we're going to before trying to connect.

renderdoc/android/android_utils.h

@@ -50,6 +50,8 @@ bool toolExists(const std::string &path);
 std::string GetFirstMatchingLine(const std::string &haystack, const std::string &needle);
 
 bool IsSupported(std::string deviceID);
+bool SupportsNativeLayers(const rdcstr &deviceID);
+std::string DetermineInstalledABI(const std::string &deviceID, const std::string &packageName);
 rdcstr GetFriendlyName(const rdcstr &deviceID);
 
 // supported ABIs

renderdoc/driver/gl/egl_hooks.cpp

@@ -901,27 +901,59 @@ static void EGLHooked(void *handle)
 }
 
 #if ENABLED(RDOC_WIN32)
+
 bool ShouldHookEGL()
 {
   const char *toggle = Process::GetEnvVariable("RENDERDOC_HOOK_EGL");
 
   // if the var is set to 0, then don't hook EGL
   if(toggle && toggle[0] == '0')
+  {
+    RDCLOG(
+        "EGL hooks disabled by RENDERDOC_HOOK_EGL environment variable - "
+        "if GLES emulator is in use, underlying API will be captured");
     return false;
+  }
 
   return true;
 }
+
+#elif ENABLED(RDOC_ANDROID)
+
+bool ShouldHookEGL()
+{
+  void *egl_handle = dlopen("libEGL.so", RTLD_LAZY);
+  PFN_eglQueryString query_string = (PFN_eglQueryString)dlsym(egl_handle, "eglQueryString");
+  if(!query_string)
+  {
+    RDCERR("Unable to find eglQueryString entry point, enabling EGL hooking");
+    return true;
+  }
+
+  const char *eglExts = query_string(EGL_NO_DISPLAY, EGL_EXTENSIONS);
+
+  if(eglExts && strstr(eglExts, "EGL_ANDROID_GLES_layers"))
+  {
+    RDCLOG("EGL_ANDROID_GLES_layers detected, disabling EGL hooks - GLES layering in effect");
+    return false;
+  }
+
+  return true;
+}
+
+#else
+
+bool ShouldHookEGL()
+{
+  return true;
+}
+
 #endif
 
 void EGLHook::RegisterHooks()
 {
-#if ENABLED(RDOC_WIN32)
   if(!ShouldHookEGL())
-  {
-    RDCLOG("EGL hooks disabled - if GLES emulator is in use, underlying API will be captured");
     return;
-  }
-#endif
 
   RDCLOG("Registering EGL hooks");
 
@@ -960,3 +992,49 @@ void EGLHook::RegisterHooks()
   EGL_HOOKED_SYMBOLS(EGL_REGISTER)
 #undef EGL_REGISTER
 }
+
+// Android GLES layering support
+#if ENABLED(RDOC_ANDROID)
+
+typedef __eglMustCastToProperFunctionPointerType(EGLAPIENTRY *PFNEGLGETNEXTLAYERPROCADDRESSPROC)(
+    void *, const char *funcName);
+
+HOOK_EXPORT void AndroidGLESLayer_Initialize(void *layer_id,
+                                             PFNEGLGETNEXTLAYERPROCADDRESSPROC next_gpa)
+{
+  RDCLOG("Initialising Android GLES layer with ID %p", layer_id);
+
+  // as a hook callback this is only called while capturing
+  RDCASSERT(!RenderDoc::Inst().IsReplayApp());
+
+// populate EGL dispatch table with the next layer's function pointers. Fetch all 'hooked' and
+// non-hooked functions
+#define EGL_FETCH(func, isext)                                                 \
+  EGL.func = (CONCAT(PFN_egl, func))next_gpa(layer_id, "egl" STRINGIZE(func)); \
+  if(!EGL.func)                                                                \
+    RDCWARN("Couldn't fetch function pointer for egl" STRINGIZE(func));
+  EGL_HOOKED_SYMBOLS(EGL_FETCH)
+  EGL_NONHOOKED_SYMBOLS(EGL_FETCH)
+#undef EGL_FETCH
+
+  // populate GL dispatch table with the next layer's function pointers
+  GL.PopulateWithCallback(
+      [layer_id, next_gpa](const char *f) { return (void *)next_gpa(layer_id, f); });
+}
+
+HOOK_EXPORT void *AndroidGLESLayer_GetProcAddress(const char *funcName,
+                                                  __eglMustCastToProperFunctionPointerType next)
+{
+// return our egl hooks
+#define GPA_FUNCTION(name, isext)              \
+  if(!strcmp(funcName, "egl" STRINGIZE(name))) \
+    return (void *)&CONCAT(egl, CONCAT(name, _renderdoc_hooked));
+  EGL_HOOKED_SYMBOLS(GPA_FUNCTION)
+#undef GPA_FUNCTION
+
+  // otherwise, consult our database of hooks
+  // Android GLES layer spec expects us to return next unmodified for functions we don't support
+  return HookedGetProcAddress(funcName, (void *)next);
+}
+
+#endif    // ENABLED(RDOC_ANDROID)

renderdoc/driver/gl/egl_layer_android.cpp

@@ -0,0 +1,26 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "driver/gl/egl_dispatch_table.h"
+#include "driver/gl/gl_driver.h"

renderdoc/driver/gl/gl_common.h

@@ -49,20 +49,17 @@ DECLARE_REFLECTION_ENUM(RDCGLenum);
 #define RENDERDOC_SUPPORT_GL
 #define RENDERDOC_SUPPORT_GLES
 
-// checks a runtime opt-out option to disallow hooking EGL on windows. This means that the
-// underlying GL or D3D calls will be captured instead.
-bool ShouldHookEGL();
-
 #else
 
 // other cases are defined by build-time configuration
 
-// for consistency, we always enable this on other platforms - if GLES support is disabled at
-// compile time then it does nothing.
-#define RENDERDOC_HOOK_EGL OPTION_ON
-
 #endif
 
+// checks a runtime opt-out option to disallow hooking EGL. On Windows with no EGL support then the
+// EGL library is an emulator, so this means we'll capture the underlying calls it makes to the
+// native API. On Android this is useful if we detect that GLES layering is supported.
+bool ShouldHookEGL();
+
 #define GL_APICALL
 
 // official headers

renderdoc/driver/gl/gl_hooks.cpp

@@ -243,6 +243,15 @@ static void GLHooked(void *handle)
 
 void GLHook::RegisterHooks()
 {
+#if ENABLED(RDOC_ANDROID)
+  // on android if EGL hooking is disabled we're using GLES layering, don't register any GL hooks
+  if(!ShouldHookEGL())
+  {
+    RDCLOG("Not registering OpenGL hooks for Android");
+    return;
+  }
+#endif
+
   RDCLOG("Registering OpenGL hooks");
 
 // pick the 'primary' library we consider GL functions to come from. This is mostly important on