[CodeComplete] Add completions for filenames in #include directives.

Summary:
The dir component ("somedir" in #include <somedir/fo...>) is considered fixed.
We append "foo" to each directory on the include path, and then list its files.

Completions are of the forms:
 #include <somedir/fo^
                   foo.h>
                   fox/

The filter is set to the filename part ("fo"), so fuzzy matching can be
applied to the filename only.

No fancy scoring/priorities are set, and no information is added to
CodeCompleteResult to make smart scoring possible. Could be in future.

Reviewers: ilya-biryukov

Subscribers: cfe-commits

Differential Revision: https://reviews.llvm.org/D52076

llvm-svn: 342449

clang/include/clang-c/Index.h

@@ -5605,10 +5605,15 @@ enum CXCompletionContext {
    */
   CXCompletionContext_NaturalLanguage = 1 << 21,
 
+  /**
+   * #include file completions should be included in the results.
+   */
+  CXCompletionContext_IncludedFile = 1 << 22,
+
   /**
    * The current context is unknown, so set all contexts.
    */
-  CXCompletionContext_Unknown = ((1 << 22) - 1)
+  CXCompletionContext_Unknown = ((1 << 23) - 1)
 };
 
 /**

clang/include/clang/Lex/CodeCompletionHandler.h

@@ -14,6 +14,8 @@
 #ifndef LLVM_CLANG_LEX_CODECOMPLETIONHANDLER_H
 #define LLVM_CLANG_LEX_CODECOMPLETIONHANDLER_H
 
+#include "llvm/ADT/StringRef.h"
+
 namespace clang {
 
 class IdentifierInfo;
@@ -60,6 +62,11 @@ class CodeCompletionHandler {
                                          MacroInfo *MacroInfo,
                                          unsigned ArgumentIndex) { }
 
+  /// Callback invoked when performing code completion inside the filename
+  /// part of an #include directive. (Also #import, #include_next, etc).
+  /// \p Dir is the directory relative to the include path, e.g. "a" for <a/b.
+  virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled) {}
+
   /// Callback invoked when performing code completion in a part of the
   /// file where we expect natural language, e.g., a comment, string, or
   /// \#error directive.

clang/include/clang/Lex/Lexer.h

@@ -711,6 +711,9 @@ class Lexer : public PreprocessorLexer {
 
   bool isHexaLiteral(const char *Start, const LangOptions &LangOpts);
 
+  void codeCompleteIncludedFile(const char *PathStart,
+                                const char *CompletionPoint, bool IsAngled);
+
   /// Read a universal character name.
   ///
   /// \param StartPtr The position in the source buffer after the initial '\'.

clang/include/clang/Lex/Preprocessor.h

@@ -1128,6 +1128,10 @@ class Preprocessor {
     CodeComplete = nullptr;
   }
 
+  /// Hook used by the lexer to invoke the "included file" code
+  /// completion point.
+  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
+
   /// Hook used by the lexer to invoke the "natural language" code
   /// completion point.
   void CodeCompleteNaturalLanguage();

clang/include/clang/Parse/Parser.h

@@ -2969,6 +2969,7 @@ class Parser : public CodeCompletionHandler {
   void CodeCompletePreprocessorExpression() override;
   void CodeCompleteMacroArgument(IdentifierInfo *Macro, MacroInfo *MacroInfo,
                                  unsigned ArgumentIndex) override;
+  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled) override;
   void CodeCompleteNaturalLanguage() override;
 };
 

clang/include/clang/Sema/CodeCompleteConsumer.h

@@ -323,6 +323,9 @@ class CodeCompletionContext {
     /// Code completion where an Objective-C category name is expected.
     CCC_ObjCCategoryName,
 
+    /// Code completion inside the filename part of a #include directive.
+    CCC_IncludedFile,
+
     /// An unknown context, in which we are recovering from a parsing
     /// error and don't know which completions we should give.
     CCC_Recovery

clang/include/clang/Sema/Sema.h

@@ -10345,6 +10345,7 @@ class Sema {
                                              IdentifierInfo *Macro,
                                              MacroInfo *MacroInfo,
                                              unsigned Argument);
+  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
   void CodeCompleteNaturalLanguage();
   void CodeCompleteAvailabilityPlatformName();
   void GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,

clang/lib/Frontend/ASTUnit.cpp

@@ -1976,6 +1976,7 @@ static void CalculateHiddenNames(const CodeCompletionContext &Context,
   case CodeCompletionContext::CCC_ObjCInstanceMessage:
   case CodeCompletionContext::CCC_ObjCClassMessage:
   case CodeCompletionContext::CCC_ObjCCategoryName:
+  case CodeCompletionContext::CCC_IncludedFile:
     // We're looking for nothing, or we're looking for names that cannot
     // be hidden.
     return;

clang/lib/Lex/Lexer.cpp

@@ -1896,6 +1896,7 @@ const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
 /// either " or L" or u8" or u" or U".
 bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
                              tok::TokenKind Kind) {
+  const char *AfterQuote = CurPtr;
   // Does this string contain the \0 character?
   const char *NulCharacter = nullptr;
 
@@ -1924,8 +1925,11 @@ bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
 
     if (C == 0) {
       if (isCodeCompletionPoint(CurPtr-1)) {
-        PP->CodeCompleteNaturalLanguage();
-        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        if (ParsingFilename)
+          codeCompleteIncludedFile(AfterQuote, CurPtr - 1, /*IsAngled=*/false);
+        else
+          PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
         cutOffLexing();
         return true;
       }
@@ -2043,16 +2047,21 @@ bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
     if (C == '\\')
       C = getAndAdvanceChar(CurPtr, Result);
 
-    if (C == '\n' || C == '\r' ||             // Newline.
-        (C == 0 && (CurPtr-1 == BufferEnd ||  // End of file.
-                    isCodeCompletionPoint(CurPtr-1)))) {
+    if (C == '\n' || C == '\r' ||                // Newline.
+        (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file.
       // If the filename is unterminated, then it must just be a lone <
       // character.  Return this as such.
       FormTokenWithChars(Result, AfterLessPos, tok::less);
       return true;
     }
 
     if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr - 1)) {
+        codeCompleteIncludedFile(AfterLessPos, CurPtr - 1, /*IsAngled=*/true);
+        cutOffLexing();
+        FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
+        return true;
+      }
       NulCharacter = CurPtr-1;
     }
     C = getAndAdvanceChar(CurPtr, Result);
@@ -2069,6 +2078,34 @@ bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
   return true;
 }
 
+void Lexer::codeCompleteIncludedFile(const char *PathStart,
+                                     const char *CompletionPoint,
+                                     bool IsAngled) {
+  // Completion only applies to the filename, after the last slash.
+  StringRef PartialPath(PathStart, CompletionPoint - PathStart);
+  auto Slash = PartialPath.find_last_of(LangOpts.MSVCCompat ? "/\\" : "/");
+  StringRef Dir =
+      (Slash == StringRef::npos) ? "" : PartialPath.take_front(Slash);
+  const char *StartOfFilename =
+      (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1;
+  // Code completion filter range is the filename only, up to completion point.
+  PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
+      StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
+  // We should replace the characters up to the closing quote, if any.
+  while (CompletionPoint < BufferEnd) {
+    char Next = *(CompletionPoint + 1);
+    if (Next == 0 || Next == '\r' || Next == '\n')
+      break;
+    ++CompletionPoint;
+    if (Next == (IsAngled ? '>' : '"'))
+      break;
+  }
+  PP->setCodeCompletionTokenRange(
+      FileLoc.getLocWithOffset(StartOfFilename - BufferStart),
+      FileLoc.getLocWithOffset(CompletionPoint - BufferStart));
+  PP->CodeCompleteIncludedFile(Dir, IsAngled);
+}
+
 /// LexCharConstant - Lex the remainder of a character constant, after having
 /// lexed either ' or L' or u8' or u' or U'.
 bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,

clang/lib/Lex/Preprocessor.cpp

@@ -445,6 +445,13 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
   return false;
 }
 
+void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
+                                            bool IsAngled) {
+  if (CodeComplete)
+    CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
+  setCodeCompletionReached();
+}
+
 void Preprocessor::CodeCompleteNaturalLanguage() {
   if (CodeComplete)
     CodeComplete->CodeCompleteNaturalLanguage();

clang/lib/Parse/Parser.cpp

@@ -1967,6 +1967,10 @@ void Parser::CodeCompleteMacroArgument(IdentifierInfo *Macro,
                                                 ArgumentIndex);
 }
 
+void Parser::CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled) {
+  Actions.CodeCompleteIncludedFile(Dir, IsAngled);
+}
+
 void Parser::CodeCompleteNaturalLanguage() {
   Actions.CodeCompleteNaturalLanguage();
 }

clang/lib/Sema/CodeCompleteConsumer.cpp

@@ -80,6 +80,7 @@ bool CodeCompletionContext::wantConstructorResults() const {
   case CCC_ObjCClassMessage:
   case CCC_ObjCInterfaceName:
   case CCC_ObjCCategoryName:
+  case CCC_IncludedFile:
     return false;
   }
 
@@ -155,6 +156,8 @@ StringRef clang::getCompletionKindString(CodeCompletionContext::Kind Kind) {
     return "ObjCInterfaceName";
   case CCKind::CCC_ObjCCategoryName:
     return "ObjCCategoryName";
+  case CCKind::CCC_IncludedFile:
+    return "IncludedFile";
   case CCKind::CCC_Recovery:
     return "Recovery";
   }
@@ -522,7 +525,8 @@ bool PrintingCodeCompleteConsumer::isResultFilteredOut(StringRef Filter,
   case CodeCompletionResult::RK_Macro:
     return !Result.Macro->getName().startswith(Filter);
   case CodeCompletionResult::RK_Pattern:
-    return !StringRef(Result.Pattern->getAsString()).startswith(Filter);
+    return !(Result.Pattern->getTypedText() &&
+             StringRef(Result.Pattern->getTypedText()).startswith(Filter));
   }
   llvm_unreachable("Unknown code completion result Kind.");
 }

clang/lib/Sema/SemaCodeComplete.cpp

@@ -32,6 +32,8 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Path.h"
 #include <list>
 #include <map>
 #include <vector>
@@ -7994,6 +7996,115 @@ void Sema::CodeCompletePreprocessorMacroArgument(Scope *S,
   // for the expanded tokens.
 }
 
+// This handles completion inside an #include filename, e.g. #include <foo/ba
+// We look for the directory "foo" under each directory on the include path,
+// list its files, and reassemble the appropriate #include.
+void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) {
+  // RelDir should use /, but unescaped \ is possible on windows!
+  // Our completions will normalize to / for simplicity, this case is rare.
+  std::string RelDir = llvm::sys::path::convert_to_slash(Dir);
+  // We need the native slashes for the actual file system interactions.
+  SmallString<128> NativeRelDir = StringRef(RelDir);
+  llvm::sys::path::native(NativeRelDir);
+  auto FS = getSourceManager().getFileManager().getVirtualFileSystem();
+
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompleter->getCodeCompletionTUInfo(),
+                        CodeCompletionContext::CCC_IncludedFile);
+  llvm::DenseSet<StringRef> SeenResults; // To deduplicate results.
+
+  // Helper: adds one file or directory completion result.
+  auto AddCompletion = [&](StringRef Filename, bool IsDirectory) {
+    SmallString<64> TypedChunk = Filename;
+    // Directory completion is up to the slash, e.g. <sys/
+    TypedChunk.push_back(IsDirectory ? '/' : Angled ? '>' : '"');
+    auto R = SeenResults.insert(TypedChunk);
+    if (R.second) { // New completion
+      const char *InternedTyped = Results.getAllocator().CopyString(TypedChunk);
+      *R.first = InternedTyped; // Avoid dangling StringRef.
+      CodeCompletionBuilder Builder(CodeCompleter->getAllocator(),
+                                    CodeCompleter->getCodeCompletionTUInfo());
+      Builder.AddTypedTextChunk(InternedTyped);
+      // The result is a "Pattern", which is pretty opaque.
+      // We may want to include the real filename to allow smart ranking.
+      Results.AddResult(CodeCompletionResult(Builder.TakeString()));
+    }
+  };
+
+  // Helper: scans IncludeDir for nice files, and adds results for each.
+  auto AddFilesFromIncludeDir = [&](StringRef IncludeDir, bool IsSystem) {
+    llvm::SmallString<128> Dir = IncludeDir;
+    if (!NativeRelDir.empty())
+      llvm::sys::path::append(Dir, NativeRelDir);
+
+    std::error_code EC;
+    unsigned Count = 0;
+    for (auto It = FS->dir_begin(Dir, EC);
+         !EC && It != vfs::directory_iterator(); It.increment(EC)) {
+      if (++Count == 2500) // If we happen to hit a huge directory,
+        break;             // bail out early so we're not too slow.
+      StringRef Filename = llvm::sys::path::filename(It->path());
+      switch (It->type()) {
+      case llvm::sys::fs::file_type::directory_file:
+        AddCompletion(Filename, /*IsDirectory=*/true);
+        break;
+      case llvm::sys::fs::file_type::regular_file:
+        // Only files that really look like headers. (Except in system dirs).
+        if (!IsSystem) {
+          // Header extensions from Types.def, which we can't depend on here.
+          if (!(Filename.endswith_lower(".h") ||
+                Filename.endswith_lower(".hh") ||
+                Filename.endswith_lower(".hpp") ||
+                Filename.endswith_lower(".inc")))
+            break;
+        }
+        AddCompletion(Filename, /*IsDirectory=*/false);
+        break;
+      default:
+        break;
+      }
+    }
+  };
+
+  // Helper: adds results relative to IncludeDir, if possible.
+  auto AddFilesFromDirLookup = [&](const DirectoryLookup &IncludeDir,
+                                   bool IsSystem) {
+    llvm::SmallString<128> Dir;
+    switch (IncludeDir.getLookupType()) {
+    case DirectoryLookup::LT_HeaderMap:
+      // header maps are not (currently) enumerable.
+      break;
+    case DirectoryLookup::LT_NormalDir:
+      AddFilesFromIncludeDir(IncludeDir.getDir()->getName(), IsSystem);
+      break;
+    case DirectoryLookup::LT_Framework:
+      AddFilesFromIncludeDir(IncludeDir.getFrameworkDir()->getName(), IsSystem);
+      break;
+    }
+  };
+
+  // Finally with all our helpers, we can scan the include path.
+  // Do this in standard order so deduplication keeps the right file.
+  // (In case we decide to add more details to the results later).
+  const auto &S = PP.getHeaderSearchInfo();
+  using llvm::make_range;
+  if (!Angled) {
+    // The current directory is on the include path for "quoted" includes.
+    auto *CurFile = PP.getCurrentFileLexer()->getFileEntry();
+    if (CurFile && CurFile->getDir())
+      AddFilesFromIncludeDir(CurFile->getDir()->getName(), false);
+    for (const auto &D : make_range(S.quoted_dir_begin(), S.quoted_dir_end()))
+      AddFilesFromDirLookup(D, false);
+  }
+  for (const auto &D : make_range(S.angled_dir_begin(), S.angled_dir_end()))
+    AddFilesFromDirLookup(D, true);
+  for (const auto &D : make_range(S.system_dir_begin(), S.system_dir_end()))
+    AddFilesFromDirLookup(D, true);
+
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
+                            Results.data(), Results.size());
+}
+
 void Sema::CodeCompleteNaturalLanguage() {
   HandleCodeCompleteResults(this, CodeCompleter,
                             CodeCompletionContext::CCC_NaturalLanguage,

clang/test/CodeCompletion/included-files.cpp

@@ -0,0 +1,29 @@
+// RUN: rm -rf %t && mkdir %t && cp %s %t/main.cc && mkdir %t/a
+// RUN: touch %t/foo.h && touch %t/foo.cc && touch %t/a/foosys %t/a/foosys.h
+
+// Quoted string shows header-ish files from CWD, and all from system.
+#include "foo.h"
+// RUN: %clang -fsyntax-only -isystem %t/a -Xclang -code-completion-at=%t/main.cc:5:13 %t/main.cc | FileCheck -check-prefix=CHECK-1 %s
+// CHECK-1-NOT: foo.cc"
+// CHECK-1: foo.h"
+// CHECK-1: foosys"
+
+// Quoted string with dir shows header-ish files in that subdir.
+#include "a/foosys"
+// RUN: %clang -fsyntax-only -isystem %t/a -Xclang -code-completion-at=%t/main.cc:12:13 %t/main.cc | FileCheck -check-prefix=CHECK-2 %s
+// CHECK-2-NOT: foo.h"
+// CHECK-2: foosys.h"
+// CHECK-2-NOT: foosys"
+
+// Angled string showes all files, but only in system dirs.
+#include <foosys>
+// RUN: %clang -fsyntax-only -isystem %t/a -Xclang -code-completion-at=%t/main.cc:19:13 %t/main.cc | FileCheck -check-prefix=CHECK-3 %s
+// CHECK-3-NOT: foo.cc>
+// CHECK-3-NOT: foo.h>
+// CHECK-3: foosys>
+
+// Backslash handling.
+#include "a\foosys"
+// RUN: %clang -fsyntax-only -isystem %t/a -Xclang -code-completion-at=%t/main.cc:26:13 %t/main.cc -fms-compatibility | FileCheck -check-prefix=CHECK-4 %s
+// CHECK-4: foosys.h"
+

clang/tools/libclang/CIndexCodeCompletion.cpp

@@ -499,6 +499,10 @@ static unsigned long long getContextsForContextKind(
       contexts = CXCompletionContext_NaturalLanguage;
       break;
     }
+    case CodeCompletionContext::CCC_IncludedFile: {
+      contexts = CXCompletionContext_IncludedFile;
+      break;
+    }
     case CodeCompletionContext::CCC_SelectorName: {
       contexts = CXCompletionContext_ObjCSelectorName;
       break;