Download symbol file for .oat files on android

On android .oat files (compiled java code) don't have symbol
information but on SDK 23+ it can be generated by the oatdump tool
(based on the dex information).

This CL adds logic to download this information and store it in the
module cache.

Differential revision: http://reviews.llvm.org/D11936

llvm-svn: 244738

lldb/include/lldb/Target/Platform.h

@@ -1092,6 +1092,10 @@ class ModuleCache;
                              const uint64_t src_offset,
                              const uint64_t src_size,
                              const FileSpec& dst_file_spec);
+
+        virtual Error
+        DownloadSymbolFile (const lldb::ModuleSP& module_sp,
+                            const FileSpec& dst_file_spec);
 
         virtual const char *
         GetCacheHostname ();

lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp

@@ -11,7 +11,9 @@
 // C++ Includes
 // Other libraries and framework includes
 #include "lldb/Core/Log.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Section.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Host/StringConvert.h"
 #include "Utility/UriParser.h"
@@ -302,3 +304,78 @@ PlatformAndroid::GetSdkVersion()
     m_sdk_version = StringConvert::ToUInt32(version_string.c_str());
     return m_sdk_version;
 }
+
+Error
+PlatformAndroid::DownloadSymbolFile (const lldb::ModuleSP& module_sp,
+                                     const FileSpec& dst_file_spec)
+{
+    // For oat file we can try to fetch additional debug info from the device
+    if (module_sp->GetFileSpec().GetFileNameExtension() != ConstString("oat"))
+        return Error("Symbol file downloading only supported for oat files");
+
+    // If we have no information about the platform file we can't execute oatdump
+    if (!module_sp->GetPlatformFileSpec())
+        return Error("No platform file specified");
+
+    // Symbolizer isn't available before SDK version 23
+    if (GetSdkVersion() < 23)
+        return Error("Symbol file generation only supported on SDK 23+");
+
+    // If we already have symtab then we don't have to try and generate one
+    if (module_sp->GetSectionList()->FindSectionByName(ConstString(".symtab")) != nullptr)
+        return Error("Symtab already available in the module");
+
+    int status = 0;
+    std::string tmpdir;
+    StreamString command;
+    command.Printf("mktemp --directory --tmpdir %s", GetWorkingDirectory().GetCString());
+    Error error = RunShellCommand(command.GetData(),
+                                  GetWorkingDirectory(),
+                                  &status,
+                                  nullptr,
+                                  &tmpdir,
+                                  5 /* timeout (s) */);
+
+    if (error.Fail() || status != 0 || tmpdir.empty())
+        return Error("Failed to generate temporary directory on the device (%s)", error.AsCString());
+    tmpdir.erase(tmpdir.size() - 1); // Remove trailing new line
+
+    // Create file remover for the temporary directory created on the device
+    std::unique_ptr<std::string, std::function<void(std::string*)>> tmpdir_remover(
+        &tmpdir,
+        [this](std::string* s) {
+            StreamString command;
+            command.Printf("rm -rf %s", s->c_str());
+            Error error = this->RunShellCommand(command.GetData(),
+                                                GetWorkingDirectory(),
+                                                nullptr,
+                                                nullptr,
+                                                nullptr,
+                                                5 /* timeout (s) */);
+
+            Log *log(GetLogIfAllCategoriesSet (LIBLLDB_LOG_PLATFORM));
+            if (error.Fail())
+                log->Printf("Failed to remove temp directory: %s", error.AsCString());
+        }
+    );
+
+    FileSpec symfile_platform_filespec(tmpdir.c_str(), false);
+    symfile_platform_filespec.AppendPathComponent("symbolized.oat");
+
+    // Execute oatdump on the remote device to generate a file with symtab
+    command.Clear();
+    command.Printf("oatdump --symbolize=%s --output=%s",
+                   module_sp->GetPlatformFileSpec().GetCString(false),
+                   symfile_platform_filespec.GetCString(false));
+    error = RunShellCommand(command.GetData(),
+                            GetWorkingDirectory(),
+                            &status,
+                            nullptr,
+                            nullptr,
+                            60 /* timeout (s) */);
+    if (error.Fail() || status != 0)
+        return Error("Oatdump failed: %s", error.AsCString());
+
+    // Download the symbolfile from the remote device
+    return GetFile(symfile_platform_filespec, dst_file_spec);
+}

lldb/source/Plugins/Platform/Android/PlatformAndroid.h

@@ -90,6 +90,10 @@ namespace platform_android {
                              const uint64_t src_size,
                              const FileSpec &dst_file_spec) override;
 
+        Error
+        DownloadSymbolFile (const lldb::ModuleSP& module_sp,
+                            const FileSpec& dst_file_spec) override;
+
     private:
         std::string m_device_id;
         uint32_t m_sdk_version;

lldb/source/Target/Platform.cpp

@@ -1811,7 +1811,7 @@ Platform::GetRemoteSharedModule (const ModuleSpec &module_spec,
     {
         // Try to get module information from the process
         if (process->GetModuleSpec (module_spec.GetFileSpec (), module_spec.GetArchitecture (), resolved_module_spec))
-          got_module_spec = true;
+            got_module_spec = true;
     }
 
     if (!got_module_spec)
@@ -1848,14 +1848,18 @@ Platform::GetCachedSharedModule (const ModuleSpec &module_spec,
         GetModuleCacheRoot (),
         GetCacheHostname (),
         module_spec,
-        [=](const ModuleSpec &module_spec, const FileSpec &tmp_download_file_spec)
+        [this](const ModuleSpec &module_spec, const FileSpec &tmp_download_file_spec)
         {
             return DownloadModuleSlice (module_spec.GetFileSpec (),
                                         module_spec.GetObjectOffset (),
                                         module_spec.GetObjectSize (),
                                         tmp_download_file_spec);
 
         },
+        [this](const ModuleSP& module_sp, const FileSpec& tmp_download_file_spec)
+        {
+            return DownloadSymbolFile (module_sp, tmp_download_file_spec);
+        },
         module_sp,
         did_create_ptr);
     if (error.Success ())
@@ -1918,6 +1922,12 @@ Platform::DownloadModuleSlice (const FileSpec& src_file_spec,
     return error;
 }
 
+Error
+Platform::DownloadSymbolFile (const lldb::ModuleSP& module_sp, const FileSpec& dst_file_spec)
+{
+    return Error ("Symbol file downloading not supported by the default platform.");
+}
+
 FileSpec
 Platform::GetModuleCacheRoot ()
 {

lldb/source/Utility/ModuleCache.cpp

@@ -29,6 +29,8 @@ namespace {
 const char* kModulesSubdir = ".cache";
 const char* kLockFileName = ".lock";
 const char* kTempFileName = ".temp";
+const char* kTempSymFileName = ".symtemp";
+const char* kSymFileExtension = ".sym";
 
 FileSpec
 JoinPath (const FileSpec &path1, const char* path2)
@@ -74,24 +76,31 @@ CreateHostSysRootModuleLink (const FileSpec &root_dir_spec, const char *hostname
     return FileSystem::Hardlink(sysroot_module_path_spec, local_module_spec);
 }
 
+FileSpec
+GetSymbolFileSpec(const FileSpec& module_file_spec)
+{
+    return FileSpec((module_file_spec.GetPath() + kSymFileExtension).c_str(), false);
+}
+
 }  // namespace
 
 Error
 ModuleCache::Put (const FileSpec &root_dir_spec,
                   const char *hostname,
                   const ModuleSpec &module_spec,
-                  const FileSpec &tmp_file)
+                  const FileSpec &tmp_file,
+                  const FileSpec &target_file)
 {
     const auto module_spec_dir = GetModuleDirectory (root_dir_spec, module_spec.GetUUID ());
-    const auto module_file_path = JoinPath (module_spec_dir, module_spec.GetFileSpec ().GetFilename ().AsCString ());
+    const auto module_file_path = JoinPath (module_spec_dir, target_file.GetFilename ().AsCString ());
 
     const auto tmp_file_path = tmp_file.GetPath ();
     const auto err_code = llvm::sys::fs::rename (tmp_file_path.c_str (), module_file_path.GetPath ().c_str ());
     if (err_code)
         return Error ("Failed to rename file %s to %s: %s",
                       tmp_file_path.c_str (), module_file_path.GetPath ().c_str (), err_code.message ().c_str ());
 
-    const auto error = CreateHostSysRootModuleLink(root_dir_spec, hostname, module_spec.GetFileSpec(), module_file_path);
+    const auto error = CreateHostSysRootModuleLink(root_dir_spec, hostname, target_file, module_file_path);
     if (error.Fail ())
         return Error ("Failed to create link to %s: %s", module_file_path.GetPath ().c_str (), error.AsCString ());
     return Error ();
@@ -131,6 +140,11 @@ ModuleCache::Get (const FileSpec &root_dir_spec,
     cached_module_spec.GetFileSpec () = module_file_path;
     cached_module_spec.GetPlatformFileSpec () = module_spec.GetFileSpec ();
     cached_module_sp.reset (new Module (cached_module_spec));
+
+    FileSpec symfile_spec = GetSymbolFileSpec(cached_module_sp->GetFileSpec ());
+    if (symfile_spec.Exists ())
+        cached_module_sp->SetSymbolFileFileSpec (symfile_spec);
+
     if (did_create_ptr)
         *did_create_ptr = true;
 
@@ -143,7 +157,8 @@ Error
 ModuleCache::GetAndPut (const FileSpec &root_dir_spec,
                         const char *hostname,
                         const ModuleSpec &module_spec,
-                        const Downloader &downloader,
+                        const ModuleDownloader &module_downloader,
+                        const SymfileDownloader &symfile_downloader,
                         lldb::ModuleSP &cached_module_sp,
                         bool *did_create_ptr)
 {
@@ -171,16 +186,37 @@ ModuleCache::GetAndPut (const FileSpec &root_dir_spec,
         return error;
 
     const auto tmp_download_file_spec = JoinPath (module_spec_dir, kTempFileName);
-    error = downloader (module_spec, tmp_download_file_spec);
+    error = module_downloader (module_spec, tmp_download_file_spec);
     llvm::FileRemover tmp_file_remover (tmp_download_file_spec.GetPath ().c_str ());
     if (error.Fail ())
         return Error("Failed to download module: %s", error.AsCString ());
 
     // Put downloaded file into local module cache.
-    error = Put (root_dir_spec, hostname, module_spec, tmp_download_file_spec);
+    error = Put (root_dir_spec, hostname, module_spec, tmp_download_file_spec, module_spec.GetFileSpec ());
     if (error.Fail ())
         return Error ("Failed to put module into cache: %s", error.AsCString ());
 
     tmp_file_remover.releaseFile ();
-    return Get (root_dir_spec, hostname, module_spec, cached_module_sp, did_create_ptr);
+    error = Get (root_dir_spec, hostname, module_spec, cached_module_sp, did_create_ptr);
+    if (error.Fail ())
+        return error;
+
+    // Fetching a symbol file for the module
+    const auto tmp_download_sym_file_spec = JoinPath (module_spec_dir, kTempSymFileName);
+    error = symfile_downloader (cached_module_sp, tmp_download_sym_file_spec);
+    llvm::FileRemover tmp_symfile_remover (tmp_download_sym_file_spec.GetPath ().c_str ());
+    if (error.Fail ())
+        // Failed to download a symfile but fetching the module was successful. The module might
+        // contain the neccessary symbols and the debugging is also possible without a symfile.
+        return Error ();
+
+    FileSpec symfile_spec = GetSymbolFileSpec (cached_module_sp->GetFileSpec ());
+    error = Put (root_dir_spec, hostname, module_spec, tmp_download_sym_file_spec, symfile_spec);
+    if (error.Fail ())
+        return Error ("Failed to put symbol file into cache: %s", error.AsCString ());
+
+    tmp_symfile_remover.releaseFile();
+
+    cached_module_sp->SetSymbolFileFileSpec (symfile_spec);
+    return Error ();
 }

lldb/source/Utility/ModuleCache.h

@@ -46,13 +46,15 @@ class UUID;
 class ModuleCache
 {
 public:
-    using Downloader = std::function<Error (const ModuleSpec&, const FileSpec&)>;
+    using ModuleDownloader = std::function<Error (const ModuleSpec&, const FileSpec&)>;
+    using SymfileDownloader = std::function<Error (const lldb::ModuleSP&, const FileSpec&)>;
 
     Error
     GetAndPut(const FileSpec &root_dir_spec,
               const char *hostname,
               const ModuleSpec &module_spec,
-              const Downloader &downloader,
+              const ModuleDownloader &module_downloader,
+              const SymfileDownloader &symfile_downloader,
               lldb::ModuleSP &cached_module_sp,
               bool *did_create_ptr);
 
@@ -61,7 +63,8 @@ class ModuleCache
     Put (const FileSpec &root_dir_spec,
          const char *hostname,
          const ModuleSpec &module_spec,
-         const FileSpec &tmp_file);
+         const FileSpec &tmp_file,
+         const FileSpec &target_file);
 
     Error
     Get (const FileSpec &root_dir_spec,