Skip to content

Commit

Permalink
[llvm] [Debuginfo] Debuginfod client library.
Browse files Browse the repository at this point in the history
This adds a Debuginfod library containing the `fetchDebuginfo` function which queries servers specified by the `DEBUGINFOD_URLS` environment variable for the debuginfo, executable, or a specified source file associated with a given build id.

This diff was split out from D111252.

Reviewed By: dblaikie

Differential Revision: https://reviews.llvm.org/D112758
  • Loading branch information
noajshu committed Dec 6, 2021
1 parent b1eb6a3 commit af69947
Show file tree
Hide file tree
Showing 8 changed files with 316 additions and 3 deletions.
71 changes: 71 additions & 0 deletions llvm/include/llvm/Debuginfod/Debuginfod.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file contains the declarations of getCachedOrDownloadArtifact and
/// several convenience functions for specific artifact types:
/// getCachedOrDownloadSource, getCachedOrDownloadExecutable, and
/// getCachedOrDownloadDebuginfo. This file also declares
/// getDefaultDebuginfodUrls and getDefaultDebuginfodCacheDirectory.
///
///
//===----------------------------------------------------------------------===//

#ifndef LLVM_DEBUGINFOD_DEBUGINFOD_H
#define LLVM_DEBUGINFOD_DEBUGINFOD_H

#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"

namespace llvm {

typedef ArrayRef<uint8_t> BuildIDRef;

typedef SmallVector<uint8_t, 10> BuildID;

/// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS
/// environment variable.
Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls();

/// Finds a default local file caching directory for the debuginfod client,
/// first checking DEBUGINFOD_CACHE_PATH.
Expected<std::string> getDefaultDebuginfodCacheDirectory();

/// Finds a default timeout for debuginfod HTTP requests. Checks
/// DEBUGINFOD_TIMEOUT environment variable, default is 90 seconds (90000 ms).
std::chrono::milliseconds getDefaultDebuginfodTimeout();

/// Fetches a specified source file by searching the default local cache
/// directory and server URLs.
Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
StringRef SourceFilePath);

/// Fetches an executable by searching the default local cache directory and
/// server URLs.
Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID);

/// Fetches a debug binary by searching the default local cache directory and
/// server URLs.
Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID);

/// Fetches any debuginfod artifact using the default local cache directory and
/// server URLs.
Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
StringRef UrlPath);

/// Fetches any debuginfod artifact using the specified local cache directory,
/// server URLs, and request timeout (in milliseconds). If the artifact is
/// found, uses the UniqueKey for the local cache file.
Expected<std::string> getCachedOrDownloadArtifact(
StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout);

} // end namespace llvm

#endif
7 changes: 4 additions & 3 deletions llvm/include/llvm/Support/Caching.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,10 @@ using AddBufferFn =
/// the cache directory if it does not already exist. The cache name appears in
/// error messages for errors during caching. The temporary file prefix is used
/// in the temporary file naming scheme used when writing files atomically.
Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
Twine CacheDirectoryPathRef,
AddBufferFn AddBuffer);
Expected<FileCache> localCache(
Twine CacheNameRef, Twine TempFilePrefixRef, Twine CacheDirectoryPathRef,
AddBufferFn AddBuffer = [](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
});
} // namespace llvm

#endif
1 change: 1 addition & 0 deletions llvm/lib/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ add_subdirectory(Object)
add_subdirectory(ObjectYAML)
add_subdirectory(Option)
add_subdirectory(Remarks)
add_subdirectory(Debuginfod)
add_subdirectory(DebugInfo)
add_subdirectory(DWP)
add_subdirectory(ExecutionEngine)
Expand Down
9 changes: 9 additions & 0 deletions llvm/lib/Debuginfod/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
add_llvm_component_library(LLVMDebuginfod
Debuginfod.cpp

ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod

LINK_COMPONENTS
Support
)
176 changes: 176 additions & 0 deletions llvm/lib/Debuginfod/Debuginfod.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
///
/// This file defines the fetchInfo function, which retrieves
/// any of the three supported artifact types: (executable, debuginfo, source
/// file) associated with a build-id from debuginfod servers. If a source file
/// is to be fetched, its absolute path must be specified in the Description
/// argument to fetchInfo.
///
//===----------------------------------------------------------------------===//

#include "llvm/Debuginfod/Debuginfod.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/CachePruning.h"
#include "llvm/Support/Caching.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/HTTPClient.h"
#include "llvm/Support/xxhash.h"

namespace llvm {
static std::string uniqueKey(llvm::StringRef S) { return utostr(xxHash64(S)); }

// Returns a binary BuildID as a normalized hex string.
// Uses lowercase for compatibility with common debuginfod servers.
static std::string buildIDToString(BuildIDRef ID) {
return llvm::toHex(ID, /*LowerCase=*/true);
}

Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() {
const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS");
if (DebuginfodUrlsEnv == NULL)
return SmallVector<StringRef>();

SmallVector<StringRef> DebuginfodUrls;
StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " ");
return DebuginfodUrls;
}

Expected<std::string> getDefaultDebuginfodCacheDirectory() {
if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH"))
return CacheDirectoryEnv;

SmallString<64> CacheDirectory;
if (!sys::path::cache_directory(CacheDirectory))
return createStringError(
errc::io_error, "Unable to determine appropriate cache directory.");
return std::string(CacheDirectory);
}

std::chrono::milliseconds getDefaultDebuginfodTimeout() {
long Timeout;
const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT");
if (DebuginfodTimeoutEnv &&
to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10))
return std::chrono::milliseconds(Timeout * 1000);

return std::chrono::milliseconds(90 * 1000);
}

/// The following functions fetch a debuginfod artifact to a file in a local
/// cache and return the cached file path. They first search the local cache,
/// followed by the debuginfod servers.

Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
StringRef SourceFilePath) {
SmallString<64> UrlPath;
sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
buildIDToString(ID), "source",
sys::path::convert_to_slash(SourceFilePath));
return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
}

Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) {
SmallString<64> UrlPath;
sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
buildIDToString(ID), "executable");
return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
}

Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) {
SmallString<64> UrlPath;
sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
buildIDToString(ID), "debuginfo");
return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
}

// General fetching function.
Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
StringRef UrlPath) {
SmallString<10> CacheDir;

Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory();
if (Error Err = CacheDirOrErr.takeError())
return Err;
CacheDir = *CacheDirOrErr;

Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr =
getDefaultDebuginfodUrls();
if (Error Err = DebuginfodUrlsOrErr.takeError())
return Err;
SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr;
return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
DebuginfodUrls,
getDefaultDebuginfodTimeout());
}

Expected<std::string> getCachedOrDownloadArtifact(
StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) {
SmallString<64> AbsCachedArtifactPath;
sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath,
"llvmcache-" + UniqueKey);

Expected<FileCache> CacheOrErr =
localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath);
if (Error Err = CacheOrErr.takeError())
return Err;

FileCache Cache = *CacheOrErr;
// We choose an arbitrary Task parameter as we do not make use of it.
unsigned Task = 0;
Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey);
if (Error Err = CacheAddStreamOrErr.takeError())
return Err;
AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
if (!CacheAddStream)
return std::string(AbsCachedArtifactPath);
// The artifact was not found in the local cache, query the debuginfod
// servers.
if (!HTTPClient::isAvailable())
return createStringError(errc::io_error,
"No working HTTP client is available.");

HTTPClient Client;
Client.setTimeout(Timeout);
for (StringRef ServerUrl : DebuginfodUrls) {
SmallString<64> ArtifactUrl;
sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath);

Expected<HTTPResponseBuffer> ResponseOrErr = Client.get(ArtifactUrl);
if (Error Err = ResponseOrErr.takeError())
return Err;

HTTPResponseBuffer &Response = *ResponseOrErr;
if (Response.Code != 200)
continue;

// We have retrieved the artifact from this server, and now add it to the
// file cache.
Expected<std::unique_ptr<CachedFileStream>> FileStreamOrErr =
CacheAddStream(Task);
if (Error Err = FileStreamOrErr.takeError())
return Err;
std::unique_ptr<CachedFileStream> &FileStream = *FileStreamOrErr;
if (!Response.Body)
return createStringError(
errc::io_error, "Unallocated MemoryBuffer in HTTPResponseBuffer.");

*FileStream->OS << StringRef(Response.Body->getBufferStart(),
Response.Body->getBufferSize());

// Return the path to the artifact on disk.
return std::string(AbsCachedArtifactPath);
}

return createStringError(errc::argument_out_of_domain, "build id not found");
}
} // namespace llvm
1 change: 1 addition & 0 deletions llvm/unittests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ add_subdirectory(Bitcode)
add_subdirectory(Bitstream)
add_subdirectory(CodeGen)
add_subdirectory(DebugInfo)
add_subdirectory(Debuginfod)
add_subdirectory(Demangle)
add_subdirectory(ExecutionEngine)
add_subdirectory(FileCheck)
Expand Down
9 changes: 9 additions & 0 deletions llvm/unittests/Debuginfod/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
set(LLVM_LINK_COMPONENTS
Debuginfod
)

add_llvm_unittest(DebuginfodTests
DebuginfodTests.cpp
)

target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport)
45 changes: 45 additions & 0 deletions llvm/unittests/Debuginfod/DebuginfodTests.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//===-- llvm/unittest/Support/DebuginfodTests.cpp - unit tests --*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "llvm/Debuginfod/Debuginfod.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/HTTPClient.h"
#include "llvm/Support/Path.h"
#include "llvm/Testing/Support/Error.h"
#include "gtest/gtest.h"

using namespace llvm;

// Check that the Debuginfod client can find locally cached artifacts.
TEST(DebuginfodClient, CacheHit) {
int FD;
SmallString<64> CachedFilePath;
sys::fs::createTemporaryFile("llvmcache-key", "temp", FD, CachedFilePath);
StringRef CacheDir = sys::path::parent_path(CachedFilePath);
StringRef UniqueKey = sys::path::filename(CachedFilePath);
EXPECT_TRUE(UniqueKey.consume_front("llvmcache-"));
raw_fd_ostream OF(FD, true, /*unbuffered=*/true);
OF << "contents\n";
OF << CacheDir << "\n";
OF.close();
Expected<std::string> PathOrErr = getCachedOrDownloadArtifact(
UniqueKey, /*UrlPath=*/"/null", CacheDir,
/*DebuginfodUrls=*/{}, /*Timeout=*/std::chrono::milliseconds(1));
EXPECT_THAT_EXPECTED(PathOrErr, HasValue(CachedFilePath));
}

// Check that the Debuginfod client returns an Error when it fails to find an
// artifact.
TEST(DebuginfodClient, CacheMiss) {
// Ensure there are no urls to guarantee a cache miss.
setenv("DEBUGINFOD_URLS", "", /*replace=*/1);
HTTPClient::initialize();
Expected<std::string> PathOrErr = getCachedOrDownloadArtifact(
/*UniqueKey=*/"nonexistent-key", /*UrlPath=*/"/null");
EXPECT_THAT_EXPECTED(PathOrErr, Failed<StringError>());
}

0 comments on commit af69947

Please sign in to comment.