[clangd] Add path mappings functionality

Summary: Add path mappings to clangd which translate file URIs on inbound and outbound LSP messages. This mapping allows clangd to run in a remote environment (e.g. docker), where the source files and dependencies may be at different locations than the host. See http://lists.llvm.org/pipermail/clangd-dev/2019-January/000231.htm for more.

Patch by William Wagner!

Reviewers: sammccall, ilya-biryukov

Reviewed By: sammccall

Subscribers: usaxena95, ormris, mgorny, MaskRay, jkorous, arphaman, kadircet, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D64305

clang-tools-extra/clangd/CMakeLists.txt

@@ -62,6 +62,7 @@ add_clang_library(clangDaemon
   IncludeFixer.cpp
   JSONTransport.cpp
   Logger.cpp
+  PathMapping.cpp
   Protocol.cpp
   Quality.cpp
   ParsedAST.cpp

clang-tools-extra/clangd/PathMapping.cpp

@@ -0,0 +1,199 @@
+//===--- PathMapping.cpp - apply path mappings to LSP messages -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "PathMapping.h"
+#include "Transport.h"
+#include "URI.h"
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Errno.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Path.h"
+#include <algorithm>
+#include <tuple>
+
+namespace clang {
+namespace clangd {
+llvm::Optional<std::string> doPathMapping(llvm::StringRef S,
+                                          PathMapping::Direction Dir,
+                                          const PathMappings &Mappings) {
+  // Retrun early to optimize for the common case, wherein S is not a file URI
+  if (!S.startswith("file://"))
+    return llvm::None;
+  auto Uri = URI::parse(S);
+  if (!Uri) {
+    llvm::consumeError(Uri.takeError());
+    return llvm::None;
+  }
+  for (const auto &Mapping : Mappings) {
+    const std::string &From = Dir == PathMapping::Direction::ClientToServer
+                                  ? Mapping.ClientPath
+                                  : Mapping.ServerPath;
+    const std::string &To = Dir == PathMapping::Direction::ClientToServer
+                                ? Mapping.ServerPath
+                                : Mapping.ClientPath;
+    llvm::StringRef Body = Uri->body();
+    if (Body.consume_front(From) && (Body.empty() || Body.front() == '/')) {
+      std::string MappedBody = (To + Body).str();
+      return URI(Uri->scheme(), Uri->authority(), MappedBody.c_str())
+          .toString();
+    }
+  }
+  return llvm::None;
+}
+
+void applyPathMappings(llvm::json::Value &V, PathMapping::Direction Dir,
+                       const PathMappings &Mappings) {
+  using Kind = llvm::json::Value::Kind;
+  Kind K = V.kind();
+  if (K == Kind::Object) {
+    llvm::json::Object *Obj = V.getAsObject();
+    llvm::json::Object MappedObj;
+    // 1. Map all the Keys
+    for (auto &KV : *Obj) {
+      if (llvm::Optional<std::string> MappedKey =
+              doPathMapping(KV.first.str(), Dir, Mappings)) {
+        MappedObj.try_emplace(std::move(*MappedKey), std::move(KV.second));
+      } else {
+        MappedObj.try_emplace(std::move(KV.first), std::move(KV.second));
+      }
+    }
+    *Obj = std::move(MappedObj);
+    // 2. Map all the values
+    for (auto &KV : *Obj)
+      applyPathMappings(KV.second, Dir, Mappings);
+  } else if (K == Kind::Array) {
+    for (llvm::json::Value &Val : *V.getAsArray())
+      applyPathMappings(Val, Dir, Mappings);
+  } else if (K == Kind::String) {
+    if (llvm::Optional<std::string> Mapped =
+            doPathMapping(*V.getAsString(), Dir, Mappings))
+      V = std::move(*Mapped);
+  }
+}
+
+namespace {
+
+class PathMappingMessageHandler : public Transport::MessageHandler {
+public:
+  PathMappingMessageHandler(MessageHandler &Handler,
+                            const PathMappings &Mappings)
+      : WrappedHandler(Handler), Mappings(Mappings) {}
+
+  bool onNotify(llvm::StringRef Method, llvm::json::Value Params) override {
+    applyPathMappings(Params, PathMapping::Direction::ClientToServer, Mappings);
+    return WrappedHandler.onNotify(Method, std::move(Params));
+  }
+
+  bool onCall(llvm::StringRef Method, llvm::json::Value Params,
+              llvm::json::Value ID) override {
+    applyPathMappings(Params, PathMapping::Direction::ClientToServer, Mappings);
+    return WrappedHandler.onCall(Method, std::move(Params), std::move(ID));
+  }
+
+  bool onReply(llvm::json::Value ID,
+               llvm::Expected<llvm::json::Value> Result) override {
+    if (Result)
+      applyPathMappings(*Result, PathMapping::Direction::ClientToServer,
+                        Mappings);
+    return WrappedHandler.onReply(std::move(ID), std::move(Result));
+  }
+
+private:
+  Transport::MessageHandler &WrappedHandler;
+  const PathMappings &Mappings;
+};
+
+// Apply path mappings to all LSP messages by intercepting all params/results
+// and then delegating to the normal transport
+class PathMappingTransport : public Transport {
+public:
+  PathMappingTransport(std::unique_ptr<Transport> Transp, PathMappings Mappings)
+      : WrappedTransport(std::move(Transp)), Mappings(std::move(Mappings)) {}
+
+  void notify(llvm::StringRef Method, llvm::json::Value Params) override {
+    applyPathMappings(Params, PathMapping::Direction::ServerToClient, Mappings);
+    WrappedTransport->notify(Method, std::move(Params));
+  }
+
+  void call(llvm::StringRef Method, llvm::json::Value Params,
+            llvm::json::Value ID) override {
+    applyPathMappings(Params, PathMapping::Direction::ServerToClient, Mappings);
+    WrappedTransport->call(Method, std::move(Params), std::move(ID));
+  }
+
+  void reply(llvm::json::Value ID,
+             llvm::Expected<llvm::json::Value> Result) override {
+    if (Result)
+      applyPathMappings(*Result, PathMapping::Direction::ServerToClient,
+                        Mappings);
+    WrappedTransport->reply(std::move(ID), std::move(Result));
+  }
+
+  llvm::Error loop(MessageHandler &Handler) override {
+    PathMappingMessageHandler WrappedHandler(Handler, Mappings);
+    return WrappedTransport->loop(WrappedHandler);
+  }
+
+private:
+  std::unique_ptr<Transport> WrappedTransport;
+  PathMappings Mappings;
+};
+
+// Converts a unix/windows path to the path portion of a file URI
+// e.g. "C:\foo" -> "/C:/foo"
+llvm::Expected<std::string> parsePath(llvm::StringRef Path) {
+  namespace path = llvm::sys::path;
+  if (path::is_absolute(Path, path::Style::posix)) {
+    return Path;
+  } else if (path::is_absolute(Path, path::Style::windows)) {
+    std::string Converted = path::convert_to_slash(Path, path::Style::windows);
+    if (Converted.front() != '/')
+      Converted = "/" + Converted;
+    return Converted;
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "Path not absolute: " + Path);
+}
+
+} // namespace
+
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const PathMapping &M) {
+  return OS << M.ClientPath << "=" << M.ServerPath;
+}
+
+llvm::Expected<PathMappings>
+parsePathMappings(llvm::StringRef RawPathMappings) {
+  llvm::StringRef ClientPath, ServerPath, PathPair, Rest = RawPathMappings;
+  PathMappings ParsedMappings;
+  while (!Rest.empty()) {
+    std::tie(PathPair, Rest) = Rest.split(",");
+    std::tie(ClientPath, ServerPath) = PathPair.split("=");
+    if (ClientPath.empty() || ServerPath.empty())
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Not a valid path mapping pair: " +
+                                         PathPair);
+    llvm::Expected<std::string> ParsedClientPath = parsePath(ClientPath);
+    if (!ParsedClientPath)
+      return ParsedClientPath.takeError();
+    llvm::Expected<std::string> ParsedServerPath = parsePath(ServerPath);
+    if (!ParsedServerPath)
+      return ParsedServerPath.takeError();
+    ParsedMappings.push_back(
+        {std::move(*ParsedClientPath), std::move(*ParsedServerPath)});
+  }
+  return ParsedMappings;
+}
+
+std::unique_ptr<Transport>
+createPathMappingTransport(std::unique_ptr<Transport> Transp,
+                           PathMappings Mappings) {
+  return std::make_unique<PathMappingTransport>(std::move(Transp), Mappings);
+}
+
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/PathMapping.h

@@ -0,0 +1,67 @@
+//===--- PathMapping.h - apply path mappings to LSP messages -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+
+class Transport;
+
+/// PathMappings are a collection of paired client and server paths.
+/// These pairs are used to alter file:// URIs appearing in inbound and outbound
+/// LSP messages, as the client's environment may have source files or
+/// dependencies at different locations than the server. Therefore, both
+/// paths are stored as they appear in file URI bodies, e.g. /usr/include or
+/// /C:/config
+///
+/// For example, if the mappings were {{"/home/user", "/workarea"}}, then
+/// a client-to-server LSP message would have file:///home/user/foo.cpp
+/// remapped to file:///workarea/foo.cpp, and the same would happen for replies
+/// (in the opposite order).
+struct PathMapping {
+  std::string ClientPath;
+  std::string ServerPath;
+  enum class Direction { ClientToServer, ServerToClient };
+};
+using PathMappings = std::vector<PathMapping>;
+
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const PathMapping &M);
+
+/// Parse the command line \p RawPathMappings (e.g. "/client=/server") into
+/// pairs. Returns an error if the mappings are malformed, i.e. not absolute or
+/// not a proper pair.
+llvm::Expected<PathMappings> parsePathMappings(llvm::StringRef RawPathMappings);
+
+/// Returns a modified \p S with the first matching path in \p Mappings
+/// substituted, if applicable
+llvm::Optional<std::string> doPathMapping(llvm::StringRef S,
+                                          PathMapping::Direction Dir,
+                                          const PathMappings &Mappings);
+
+/// Applies the \p Mappings to all the file:// URIs in \p Params.
+/// NOTE: The first matching mapping will be applied, otherwise \p Params will
+/// be untouched.
+void applyPathMappings(llvm::json::Value &Params, PathMapping::Direction Dir,
+                       const PathMappings &Mappings);
+
+/// Creates a wrapping transport over \p Transp that applies the \p Mappings to
+/// all inbound and outbound LSP messages. All calls are then delegated to the
+/// regular transport (e.g. XPC, JSON).
+std::unique_ptr<Transport>
+createPathMappingTransport(std::unique_ptr<Transport> Transp,
+                           PathMappings Mappings);
+
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/test/Inputs/path-mappings/server/foo.h

@@ -0,0 +1,4 @@
+#ifndef FOO_H
+#define FOO_H
+int foo() { return 42; }
+#endif

clang-tools-extra/clangd/test/path-mappings.test

@@ -0,0 +1,64 @@
+# Copy over the server file into test workspace
+# RUN: rm -rf %t
+# RUN: cp -r %S/Inputs/path-mappings %t
+#
+# RUN: clangd --path-mappings 'C:\client=%t/server' -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+---
+{
+  "jsonrpc": "2.0",
+  "method": "textDocument/didOpen",
+  "params": {
+    "textDocument": {
+      "uri": "file:///C:/client/bar.cpp",
+      "languageId": "cpp",
+      "version": 1,
+      "text": "#include \"foo.h\"\nint main(){\nreturn foo();\n}"
+    }
+  }
+}
+# Ensure that the client gets back the same client path (clangd thinks it edited %t/server/bar.cpp)
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [],
+# CHECK-NEXT:    "uri": "file:///C:/client/bar.cpp"
+# CHECK-NEXT:  }
+---
+# We're editing bar.cpp, which includes foo.h, where foo.h "exists" at a server location 
+# With path mappings, when we go to definition on foo(), we get back a client file uri
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "textDocument/definition",
+  "params": {
+    "textDocument": {
+      "uri": "file:///C:/client/bar.cpp"
+    },
+    "position": {
+      "line": 2,
+      "character": 8
+    }
+  }
+}
+#      CHECK:  "id": 1,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "range": {
+# CHECK-NEXT:        "end": {
+# CHECK-NEXT:          "character": {{[0-9]+}},
+# CHECK-NEXT:          "line": {{[0-9]+}} 
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "start": {
+# CHECK-NEXT:          "character": {{[0-9]+}},
+# CHECK-NEXT:          "line": {{[0-9]+}}
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "uri": "file:///C:/client/foo.h"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+#
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}

clang-tools-extra/clangd/tool/ClangdMain.cpp

@@ -10,6 +10,7 @@
 #include "CodeComplete.h"
 #include "Features.inc"
 #include "Path.h"
+#include "PathMapping.h"
 #include "Protocol.h"
 #include "Shutdown.h"
 #include "Trace.h"
@@ -350,6 +351,18 @@ opt<bool> EnableTestScheme{
     Hidden,
 };
 
+opt<std::string> PathMappingsArg{
+    "path-mappings",
+    cat(Protocol),
+    desc(
+        "Translates between client paths (as seen by a remote editor) and "
+        "server paths (where clangd sees files on disk). "
+        "Comma separated list of '<client_path>=<server_path>' pairs, the "
+        "first entry matching a given path is used. "
+        "e.g. /home/project/incl=/opt/include,/home/project=/workarea/project"),
+    init(""),
+};
+
 opt<Path> InputMirrorFile{
     "input-mirror-file",
     cat(Protocol),
@@ -654,7 +667,15 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
         InputMirrorStream ? InputMirrorStream.getPointer() : nullptr,
         PrettyPrint, InputStyle);
   }
-
+  if (!PathMappingsArg.empty()) {
+    auto Mappings = parsePathMappings(PathMappingsArg);
+    if (!Mappings) {
+      elog("Invalid -path-mappings: {0}", Mappings.takeError());
+      return 1;
+    }
+    TransportLayer = createPathMappingTransport(std::move(TransportLayer),
+                                                std::move(*Mappings));
+  }
   // Create an empty clang-tidy option.
   std::mutex ClangTidyOptMu;
   std::unique_ptr<tidy::ClangTidyOptionsProvider>

clang-tools-extra/clangd/unittests/CMakeLists.txt

@@ -55,6 +55,7 @@ add_unittest(ClangdUnitTests ClangdTests
   IndexTests.cpp
   JSONTransportTests.cpp
   ParsedASTTests.cpp
+  PathMappingTests.cpp
   PrintASTTests.cpp
   QualityTests.cpp
   RenameTests.cpp