diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h index 88fe5f39c2a8e..62d7e16e1251c 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h @@ -14,9 +14,9 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CASID.h" #include "llvm/CAS/CASReference.h" -#include "llvm/CAS/ThreadSafeFileSystem.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/VirtualFileSystem.h" @@ -32,8 +32,12 @@ namespace clang { namespace tooling { namespace dependencies { -class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem { +class DependencyScanningCASFilesystem + : public llvm::RTTIExtends { public: + static const char ID; + DependencyScanningCASFilesystem( IntrusiveRefCntPtr WorkerFS, llvm::cas::ActionCache &Cache); @@ -59,13 +63,13 @@ class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem { return FS->isLocal(Path, Result); } - IntrusiveRefCntPtr + IntrusiveRefCntPtr createThreadSafeProxyFS() override; llvm::ErrorOr status(const Twine &Path) override; bool exists(const Twine &Path) override; - llvm::ErrorOr> - openFileForRead(const Twine &Path) override; + llvm::Expected> + openCASBackedFileForRead(const Twine &Path) override; /// \returns The scanned preprocessor directive tokens of the file that are /// used to speed up preprocessing, if available. diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index e2e0ec37f0429..2f8ec3b4cda1d 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -20,6 +20,7 @@ #include "clang/Basic/FileSystemStatCache.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CASReference.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/FileSystem.h" @@ -536,10 +537,10 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile, auto Result = Entry->File->getBuffer(Filename, FileSize, RequiresNullTerminator, isVolatile); if (CASContents) { - auto CASRef = Entry->File->getObjectRefForContent(); - if (!CASRef) - return CASRef.getError(); - *CASContents = *CASRef; + if (auto *CASFile = dyn_cast(Entry->File.get())) + *CASContents = CASFile->getObjectRefForContent(); + else + *CASContents = std::nullopt; } Entry->closeFile(); return Result; @@ -555,25 +556,52 @@ FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize, bool isVolatile, bool RequiresNullTerminator, bool IsText, std::optional *CASContents) const { + auto getBufferImpl = [&](StringRef Name) + -> llvm::ErrorOr> { + auto F = FS->openFileForRead(Name); + if (!F) + return F.getError(); + + if (CASContents) { + if (auto *CASFile = dyn_cast(F->get())) + *CASContents = CASFile->getObjectRefForContent(); + else + *CASContents = std::nullopt; + } + return FS->getBufferForFile(Name, FileSize, RequiresNullTerminator, + isVolatile, IsText); + }; + if (FileSystemOpts.WorkingDir.empty()) - return FS->getBufferForFile(Filename, FileSize, RequiresNullTerminator, - isVolatile, IsText, CASContents); + return getBufferImpl(Filename); SmallString<128> FilePath(Filename); FixupRelativePath(FilePath); - return FS->getBufferForFile(FilePath, FileSize, RequiresNullTerminator, - isVolatile, IsText, CASContents); + return getBufferImpl(FilePath); } llvm::ErrorOr> FileManager::getObjectRefForFileContent(const Twine &Filename) { + auto getObjectRefImpl = + [&](const Twine &Name) -> llvm::ErrorOr> { + auto F = FS->openFileForRead(Name); + if (!F) + return F.getError(); + + auto *CASFile = dyn_cast(F->get()); + if (!CASFile) + return std::nullopt; + + return CASFile->getObjectRefForContent(); + }; + if (FileSystemOpts.WorkingDir.empty()) - return FS->getObjectRefForFileContent(Filename); + return getObjectRefImpl(Filename); SmallString<128> FilePath; Filename.toVector(FilePath); FixupRelativePath(FilePath); - return FS->getObjectRefForFileContent(FilePath); + return getObjectRefImpl(FilePath); } /// getStatValue - Get the 'stat' information for the specified path, diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index d2450b7f86ab6..8cb2ad4c2de7d 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -9,6 +9,7 @@ #include "clang/CAS/IncludeTree.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallBitVector.h" +#include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" @@ -899,11 +900,11 @@ namespace { /// of the preprocessor, for creating \p FileEntries using a file path, while /// "replaying" an \p IncludeTreeRoot. It is not intended to be a complete /// implementation of a file system. -class IncludeTreeFileSystem : public llvm::vfs::FileSystem { +class IncludeTreeFileSystem final : public llvm::cas::CASBackedFileSystem { llvm::cas::ObjectStore &CAS; public: - class IncludeTreeFile : public llvm::vfs::File { + class IncludeTreeFile final : public llvm::cas::CASBackedFile { llvm::vfs::Status Stat; StringRef Contents; cas::ObjectRef ContentsRef; @@ -924,10 +925,7 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { Name.toStringRef(NameBuf)); } - llvm::ErrorOr> - getObjectRefForContent() override { - return ContentsRef; - } + cas::ObjectRef getObjectRefForContent() override { return ContentsRef; } std::error_code close() override { return std::error_code(); } }; @@ -972,13 +970,13 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { return llvm::errorToErrorCode(fileError(Filename)); } - llvm::ErrorOr> - openFileForRead(const Twine &Path) override { + llvm::Expected> + openCASBackedFileForRead(const Twine &Path) override { SmallString<128> Filename; getPath(Path, Filename); auto MaterializedFile = materialize(Filename); if (!MaterializedFile) - return llvm::errorToErrorCode(MaterializedFile.takeError()); + return MaterializedFile.takeError(); llvm::vfs::Status Stat = makeStatus( Filename, MaterializedFile->Contents.size(), MaterializedFile->UniqueID, llvm::sys::fs::file_type::regular_file); @@ -1037,6 +1035,10 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { std::error_code setCurrentWorkingDirectory(const Twine &Path) override { return llvm::errc::operation_not_permitted; } + IntrusiveRefCntPtr + createThreadSafeProxyFS() override { + llvm::report_fatal_error("not implemented"); + } }; } // namespace diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp index c4c2bf920ba37..a32afcc55c715 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp @@ -40,6 +40,7 @@ DependencyScanningCASFilesystem::DependencyScanningCASFilesystem( : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), Cache(Cache) { } +const char DependencyScanningCASFilesystem::ID = 0; DependencyScanningCASFilesystem::~DependencyScanningCASFilesystem() = default; static Expected @@ -276,19 +277,18 @@ bool DependencyScanningCASFilesystem::exists(const Twine &Path) { return getCachingFS().exists(Path); } -IntrusiveRefCntPtr +IntrusiveRefCntPtr DependencyScanningCASFilesystem::createThreadSafeProxyFS() { llvm::report_fatal_error("not implemented"); } namespace { -class DepScanFile final : public llvm::vfs::File { +class DepScanFile final : public llvm::cas::CASBackedFile { public: - DepScanFile(StringRef Buffer, std::optional CASContents, + DepScanFile(StringRef Buffer, cas::ObjectRef CASContents, llvm::vfs::Status Stat) - : Buffer(Buffer), CASContents(std::move(CASContents)), - Stat(std::move(Stat)) {} + : Buffer(Buffer), CASContents(CASContents), Stat(std::move(Stat)) {} llvm::ErrorOr status() override { return Stat; } @@ -299,36 +299,35 @@ class DepScanFile final : public llvm::vfs::File { return llvm::MemoryBuffer::getMemBuffer(Buffer, Name.toStringRef(Storage)); } - llvm::ErrorOr> - getObjectRefForContent() override { - return CASContents; - } + cas::ObjectRef getObjectRefForContent() override { return CASContents; } std::error_code close() override { return {}; } private: StringRef Buffer; - std::optional CASContents; + cas::ObjectRef CASContents; llvm::vfs::Status Stat; }; } // end anonymous namespace -llvm::ErrorOr> -DependencyScanningCASFilesystem::openFileForRead(const Twine &Path) { +Expected> +DependencyScanningCASFilesystem::openCASBackedFileForRead(const Twine &Path) { LookupPathResult Result = lookupPath(Path); if (!Result.Entry) { if (std::error_code EC = Result.Status.getError()) - return EC; + return llvm::errorCodeToError(EC); assert(Result.Status->getType() == llvm::sys::fs::file_type::directory_file); - return std::make_error_code(std::errc::is_a_directory); + return llvm::createFileError( + Path, std::make_error_code(std::errc::is_a_directory)); } if (Result.Entry->EC) - return Result.Entry->EC; + return llvm::errorCodeToError(Result.Entry->EC); + assert(Result.Entry->CASContents); return std::make_unique( - *Result.Entry->Buffer, Result.Entry->CASContents, Result.Entry->Status); + *Result.Entry->Buffer, *Result.Entry->CASContents, Result.Entry->Status); } std::optional> diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 0c2000b337d23..f606b9c3e990e 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "llvm/CAS/CASFileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Threading.h" #include @@ -33,10 +34,9 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) { return MaybeBuffer.getError(); auto Buffer = std::move(*MaybeBuffer); - auto MaybeCASContents = File->getObjectRefForContent(); - if (!MaybeCASContents) - return MaybeCASContents.getError(); - auto CASContents = std::move(*MaybeCASContents); + std::optional CASContents; + if (auto *CASFile = dyn_cast(File.get())) + CASContents = CASFile->getObjectRefForContent(); // If the file size changed between read and stat, pretend it didn't. if (Stat.getSize() != Buffer->getBufferSize()) @@ -362,9 +362,8 @@ namespace { class DepScanFile final : public llvm::vfs::File { public: DepScanFile(std::unique_ptr Buffer, - std::optional CASContents, llvm::vfs::Status Stat) - : Buffer(std::move(Buffer)), CASContents(std::move(CASContents)), - Stat(std::move(Stat)) {} + llvm::vfs::Status Stat) + : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} static llvm::ErrorOr> create(EntryRef Entry); @@ -376,16 +375,37 @@ class DepScanFile final : public llvm::vfs::File { return std::move(Buffer); } - llvm::ErrorOr> - getObjectRefForContent() override { - return CASContents; + std::error_code close() override { return {}; } + +private: + std::unique_ptr Buffer; + llvm::vfs::Status Stat; +}; + +class DepScanCASFile final : public llvm::cas::CASBackedFile { +public: + DepScanCASFile(std::unique_ptr Buffer, + cas::ObjectRef CASContents, llvm::vfs::Status Stat) + : Buffer(std::move(Buffer)), CASContents(CASContents), + Stat(std::move(Stat)) {} + + static llvm::ErrorOr> create(EntryRef Entry); + + llvm::ErrorOr status() override { return Stat; } + + llvm::ErrorOr> + getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, + bool IsVolatile) override { + return std::move(Buffer); } + cas::ObjectRef getObjectRefForContent() override { return CASContents; } + std::error_code close() override { return {}; } private: std::unique_ptr Buffer; - std::optional CASContents; + cas::ObjectRef CASContents; llvm::vfs::Status Stat; }; @@ -398,14 +418,22 @@ DepScanFile::create(EntryRef Entry) { if (Entry.isDirectory()) return std::make_error_code(std::errc::is_a_directory); - auto Result = std::make_unique( + std::unique_ptr Result; + if (Entry.getObjectRefForContent()) + Result = std::make_unique( llvm::MemoryBuffer::getMemBuffer(Entry.getContents(), Entry.getStatus().getName(), /*RequiresNullTerminator=*/false), - Entry.getObjectRefForContent(), Entry.getStatus()); + *Entry.getObjectRefForContent(), Entry.getStatus()); + else + Result = std::make_unique( + llvm::MemoryBuffer::getMemBuffer(Entry.getContents(), + Entry.getStatus().getName(), + /*RequiresNullTerminator=*/false), + Entry.getStatus()); + - return llvm::ErrorOr>( - std::unique_ptr(std::move(Result))); + return Result; } llvm::ErrorOr> diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp index 930f037f8d8a3..2bc236b4a053e 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp @@ -263,17 +263,21 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { { DependencyScanningWorkerFilesystem DepFS(Service.getSharedCache(), std::move(CASFS)); - std::optional CASContents; - auto Buf = DepFS.getBufferForFile(Path, /*FileSize*/ -1, - /*RequiresNullTerminator*/ false, - /*IsVolatile*/ false, /*IsText*/ true, - &CASContents); + llvm::ErrorOr> File = + DepFS.openFileForRead(Path); + ASSERT_TRUE(File); + llvm::cas::CASBackedFile *CASFile = + dyn_cast(File->get()); + ASSERT_TRUE(CASFile); + auto Buf = CASFile->getBuffer(Path, /*FileSize*/ -1, + /*RequiresNullTerminator*/ false, + /*IsVolatile*/ false); ASSERT_TRUE(Buf); EXPECT_EQ(Contents, (*Buf)->getBuffer()); - ASSERT_TRUE(CASContents); std::optional BlobContents; - ASSERT_THAT_ERROR(DB->getProxy(*CASContents).moveInto(BlobContents), - llvm::Succeeded()); + ASSERT_THAT_ERROR( + DB->getProxy(CASFile->getObjectRefForContent()).moveInto(BlobContents), + llvm::Succeeded()); EXPECT_EQ(BlobContents->getData(), Contents); } { @@ -286,13 +290,12 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { llvm::ErrorOr> File = DepFS.openFileForRead(Path); ASSERT_TRUE(File); - ASSERT_TRUE(*File); - llvm::ErrorOr> Ref = - (*File)->getObjectRefForContent(); - ASSERT_TRUE(Ref); - ASSERT_TRUE(*Ref); + llvm::cas::CASBackedFile *CASFile = + dyn_cast(File->get()); + ASSERT_TRUE(CASFile); + ObjectRef Ref = CASFile->getObjectRefForContent(); std::optional BlobContents; - ASSERT_THAT_ERROR(DB->getProxy(**Ref).moveInto(BlobContents), + ASSERT_THAT_ERROR(DB->getProxy(Ref).moveInto(BlobContents), llvm::Succeeded()); EXPECT_EQ(BlobContents->getData(), Contents); } diff --git a/llvm/include/llvm/CAS/CASFileSystem.h b/llvm/include/llvm/CAS/CASFileSystem.h index 896db1cdd6ed6..eb328b11d85f8 100644 --- a/llvm/include/llvm/CAS/CASFileSystem.h +++ b/llvm/include/llvm/CAS/CASFileSystem.h @@ -9,14 +9,59 @@ #ifndef LLVM_CAS_CASFILESYSTEM_H #define LLVM_CAS_CASFILESYSTEM_H -#include -#include +#include "llvm/Support/Error.h" +#include "llvm/Support/ExtensibleRTTI.h" +#include "llvm/Support/VirtualFileSystem.h" -namespace llvm { -namespace cas { +namespace llvm::cas { class ObjectStore; class CASID; +/// Abstract class represents an open file backed by a CAS. +class CASBackedFile : public RTTIExtends { +public: + static const char ID; + /// Get the CAS reference for the contents of the file. + virtual cas::ObjectRef getObjectRefForContent() = 0; +}; + +/// Abstract class represents a CAS backed file system. +class CASBackedFileSystem + : public llvm::RTTIExtends { +public: + static const char ID; + + /// This is a convenience method that opens a file, gets its content and then + /// closes the file. It returns MemoryBuffer and ObjectRef in one call to avoid + /// open the file twice. + /// The IsText parameter is used to distinguish whether the file should be + /// opened as a binary or text file. + llvm::Expected, cas::ObjectRef>> + getBufferAndObjectRefForFile(const Twine &Name, int64_t FileSize = -1, + bool RequiresNullTerminator = true, + bool IsVolatile = false, bool IsText = true); + + /// Get ObjectRef of a file from its path. + llvm::Expected getObjectRefForFileContent(const Twine &Name); + + /// Implementation for openFileForRead using CASBackedFile. + ErrorOr> + openFileForRead(const Twine &Path) override { + auto F = openCASBackedFileForRead(Path); + if (!F) + return errorToErrorCode(F.takeError()); + return std::move(*F); + } + + /// Get CASBackedFile for read. + virtual llvm::Expected> + openCASBackedFileForRead(const Twine &Path) = 0; + + /// Get a proxy FS that has an independent working directory. + virtual IntrusiveRefCntPtr + createThreadSafeProxyFS() = 0; +}; + Expected> createCASFileSystem(std::shared_ptr DB, const CASID &RootID, sys::path::Style PathStyle = sys::path::Style::native); @@ -25,7 +70,6 @@ Expected> createCASFileSystem(ObjectStore &DB, const CASID &RootID, sys::path::Style PathStyle = sys::path::Style::native); -} // namespace cas -} // namespace llvm +} // namespace llvm::cas #endif // LLVM_CAS_CASFILESYSTEM_H diff --git a/llvm/include/llvm/CAS/CachingOnDiskFileSystem.h b/llvm/include/llvm/CAS/CachingOnDiskFileSystem.h index c6c5d00b3e80b..7d37198331414 100644 --- a/llvm/include/llvm/CAS/CachingOnDiskFileSystem.h +++ b/llvm/include/llvm/CAS/CachingOnDiskFileSystem.h @@ -9,11 +9,9 @@ #ifndef LLVM_CAS_CACHINGONDISKFILESYSTEM_H #define LLVM_CAS_CACHINGONDISKFILESYSTEM_H -#include "llvm/CAS/FileSystemCache.h" -#include "llvm/CAS/ThreadSafeFileSystem.h" +#include "llvm/CAS/CASFileSystem.h" #include "llvm/Support/Error.h" #include "llvm/Support/VirtualFileSystem.h" -#include #include namespace llvm { @@ -29,10 +27,13 @@ class ObjectProxy; /// working directory. This allows a single caching on-disk filesystem to be /// used across the filesystem, with each thread using a different proxy to set /// the working directory. -class CachingOnDiskFileSystem : public ThreadSafeFileSystem { +class CachingOnDiskFileSystem + : public RTTIExtends { void anchor() override; public: + static const char ID; + /// An extra API to pull out the \a CASID if \p Path refers to a file. virtual ErrorOr statusAndFileID(const Twine &Path, std::optional &FileID) = 0; @@ -123,10 +124,18 @@ class CachingOnDiskFileSystem : public ThreadSafeFileSystem { /// same thread-safe cache. virtual IntrusiveRefCntPtr createProxyFS() = 0; - IntrusiveRefCntPtr createThreadSafeProxyFS() final { + IntrusiveRefCntPtr createThreadSafeProxyFS() final { return createProxyFS(); } + virtual llvm::Expected> + openCASBackedFileForReadImpl(const Twine &Path) = 0; + + llvm::Expected> + openCASBackedFileForRead(const Twine &Path) final { + return openCASBackedFileForReadImpl(Path); + } + protected: CachingOnDiskFileSystem(std::shared_ptr DB); CachingOnDiskFileSystem(ObjectStore &DB); diff --git a/llvm/include/llvm/CAS/ThreadSafeFileSystem.h b/llvm/include/llvm/CAS/ThreadSafeFileSystem.h deleted file mode 100644 index b6de16458a259..0000000000000 --- a/llvm/include/llvm/CAS/ThreadSafeFileSystem.h +++ /dev/null @@ -1,37 +0,0 @@ -//===- llvm/CAS/ThreadSafeFileSystem.h ------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CAS_THREADSAFEFILESYSTEM_H -#define LLVM_CAS_THREADSAFEFILESYSTEM_H - -#include "llvm/Support/Error.h" -#include "llvm/Support/VirtualFileSystem.h" - -namespace llvm { -namespace cas { - -class ObjectStore; -class CASID; - -/// For thread-safe filesystem implementations. -class ThreadSafeFileSystem - : public llvm::RTTIExtends { - virtual void anchor() override; - -public: - static const char ID; - - /// Get a proxy FS that has an independent working directory. - virtual IntrusiveRefCntPtr - createThreadSafeProxyFS() = 0; -}; - -} // namespace cas -} // namespace llvm - -#endif // LLVM_CAS_THREADSAFEFILESYSTEM_H diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index 6b87808f9de80..99a3d797e800e 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -18,9 +18,6 @@ #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -// FIXME: Bringing in "CASReference.h" because Swift/C++ interop complains about -// `ObjectRef` being undefined. -#include "llvm/CAS/CASReference.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Errc.h" @@ -118,8 +115,10 @@ class Status { }; /// Represents an open file. -class LLVM_ABI File { +class LLVM_ABI File : public RTTIExtends { public: + static const char ID; + /// Destroy the file after closing it (if open). /// Sub-classes should generally call close() inside their destructors. We /// cannot do that from the base class, since close is virtual. @@ -141,11 +140,6 @@ class LLVM_ABI File { getBuffer(const Twine &Name, int64_t FileSize = -1, bool RequiresNullTerminator = true, bool IsVolatile = false) = 0; - /// Get the CAS reference for the contents of the file. - /// \returns \p None if the underlying \p FileSystem doesn't support providing - /// CAS references. - virtual llvm::ErrorOr> getObjectRefForContent(); - /// Closes the file. virtual std::error_code close() = 0; @@ -307,15 +301,7 @@ class LLVM_ABI FileSystem : public llvm::ThreadSafeRefCountedBase, llvm::ErrorOr> getBufferForFile(const Twine &Name, int64_t FileSize = -1, bool RequiresNullTerminator = true, bool IsVolatile = false, - bool IsText = true, - std::optional *CASContents = nullptr); - - /// This is a convenience method that opens a file, gets the \p cas::ObjectRef - /// for its contents if supported by the file system, and then closes the - /// file. If both the buffer and its `cas::ObjectRef` are needed use \p - /// getBufferForFile to avoid the extra file lookup. - llvm::ErrorOr> - getObjectRefForFileContent(const Twine &Name); + bool IsText = true); /// Get a directory_iterator for \p Dir. /// \note The 'end' iterator is directory_iterator(). diff --git a/llvm/include/module.modulemap b/llvm/include/module.modulemap index 67d42a9c9852b..8866896d3fc9e 100644 --- a/llvm/include/module.modulemap +++ b/llvm/include/module.modulemap @@ -111,7 +111,6 @@ module LLVM_CAS { requires cplusplus umbrella "llvm/CAS" module * { export * } - textual header "llvm/CAS/CASReference.h" } module LLVM_Config { diff --git a/llvm/lib/CAS/CASFileSystem.cpp b/llvm/lib/CAS/CASFileSystem.cpp index 6262f188e7856..f5c4675c7501f 100644 --- a/llvm/lib/CAS/CASFileSystem.cpp +++ b/llvm/lib/CAS/CASFileSystem.cpp @@ -7,18 +7,46 @@ //===----------------------------------------------------------------------===// #include "llvm/CAS/CASFileSystem.h" -#include "llvm/ADT/StringMap.h" #include "llvm/CAS/FileSystemCache.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/CAS/TreeSchema.h" -#include "llvm/Support/AlignOf.h" -#include "llvm/Support/Allocator.h" using namespace llvm; using namespace llvm::cas; +const char CASBackedFile::ID = 0; +const char CASBackedFileSystem::ID = 0; + +llvm::Expected, cas::ObjectRef>> +CASBackedFileSystem::getBufferAndObjectRefForFile(const Twine &Name, + int64_t FileSize, + bool RequiresNullTerminator, + bool IsVolatile, + bool IsText) { + auto F = openCASBackedFileForRead(Name); + if (!F) + return F.takeError(); + + auto Buf = + (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile); + if (!Buf) + return errorCodeToError(Buf.getError()); + + auto CASRef = (*F)->getObjectRefForContent(); + return std::pair{std::move(*Buf), CASRef}; +} + +llvm::Expected +CASBackedFileSystem::getObjectRefForFileContent(const Twine &Name) { + auto F = openCASBackedFileForRead(Name); + if (!F) + return F.takeError(); + return (*F)->getObjectRefForContent(); +} + namespace { -class CASFileSystem : public ThreadSafeFileSystem { +class CASFileSystem final + : public RTTIExtends { struct WorkingDirectoryType { FileSystemCache::DirectoryEntry *Entry; @@ -46,7 +74,8 @@ class CASFileSystem : public ThreadSafeFileSystem { bool FollowSymlinks = true); ErrorOr status(const Twine &Path) final; - ErrorOr> openFileForRead(const Twine &Path) final; + Expected> + openCASBackedFileForRead(const Twine &Path) final; Expected getDirectoryEntry(const Twine &Path, bool FollowSymlinks) const final; @@ -75,7 +104,7 @@ class CASFileSystem : public ThreadSafeFileSystem { CASFileSystem(ObjectStore &DB, sys::path::Style PathStyle) : DB(DB), PathStyle(PathStyle) {} - IntrusiveRefCntPtr createThreadSafeProxyFS() final { + IntrusiveRefCntPtr createThreadSafeProxyFS() final { return makeIntrusiveRefCnt(*this); } CASFileSystem(const CASFileSystem &FS) = default; @@ -92,7 +121,7 @@ class CASFileSystem : public ThreadSafeFileSystem { }; } // namespace -class CASFileSystem::VFSFile : public vfs::File { +class CASFileSystem::VFSFile final : public CASBackedFile { public: ErrorOr status() final { return Entry->getStatus(Name); } @@ -109,9 +138,7 @@ class CASFileSystem::VFSFile : public vfs::File { return Object->getMemoryBuffer(Name.toStringRef(Storage)); } - llvm::ErrorOr> getObjectRefForContent() final { - return Entry->getRef(); - } + cas::ObjectRef getObjectRefForContent() final { return *Entry->getRef(); } /// Closes the file. std::error_code close() final { return std::error_code(); } @@ -121,6 +148,7 @@ class CASFileSystem::VFSFile : public vfs::File { : DB(DB), Name(Name.str()), Entry(&Entry) { assert(Entry.isFile()); assert(Entry.hasNode()); + assert(Entry.getRef()); } private: @@ -264,22 +292,23 @@ CASFileSystem::getDirectoryEntry(const Twine &Path, bool FollowSymlinks) const { return const_cast(this)->lookupPath(PathRef, FollowSymlinks); } -ErrorOr> -CASFileSystem::openFileForRead(const Twine &Path) { +Expected> +CASFileSystem::openCASBackedFileForRead(const Twine &Path) { PathStorage PathStorage(Path, PathStyle); StringRef PathRef = PathStorage.Path; Expected ExpectedEntry = lookupPath(PathRef); if (!ExpectedEntry) - return errorToErrorCode(ExpectedEntry.takeError()); + return ExpectedEntry.takeError(); DirectoryEntry *Entry = *ExpectedEntry; if (!Entry->isFile()) - return std::errc::invalid_argument; + return createFileError(Path, + std::make_error_code(std::errc::invalid_argument)); if (!Entry->hasNode()) if (Error E = loadFile(*Entry)) - return errorToErrorCode(std::move(E)); + return std::move(E); return std::make_unique(DB, *Entry, PathRef); } diff --git a/llvm/lib/CAS/CASProvidingFileSystem.cpp b/llvm/lib/CAS/CASProvidingFileSystem.cpp index 36095903d3249..80f516a572790 100644 --- a/llvm/lib/CAS/CASProvidingFileSystem.cpp +++ b/llvm/lib/CAS/CASProvidingFileSystem.cpp @@ -6,71 +6,95 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/VirtualFileSystem.h" using namespace llvm; using namespace llvm::cas; +using namespace llvm::vfs; namespace { -class CASProvidingFile final : public vfs::File { - std::shared_ptr DB; +class CASProvidingFile final : public CASBackedFile { std::unique_ptr UnderlyingFile; + ObjectRef Ref; public: - CASProvidingFile(std::shared_ptr DB, - std::unique_ptr UnderlyingFile) - : DB(std::move(DB)), UnderlyingFile(std::move(UnderlyingFile)) {} + CASProvidingFile(ObjectRef Ref, std::unique_ptr UnderlyingFile) + : UnderlyingFile(std::move(UnderlyingFile)), Ref(Ref) {} - ErrorOr status() override { return UnderlyingFile->status(); } + ErrorOr status() final { return UnderlyingFile->status(); } ErrorOr> getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, - bool IsVolatile) override { + bool IsVolatile) final { return UnderlyingFile->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile); } - ErrorOr> getObjectRefForContent() override { - auto UnderlyingCASRef = UnderlyingFile->getObjectRefForContent(); - if (!UnderlyingCASRef || *UnderlyingCASRef) - return UnderlyingCASRef; - - auto Buffer = UnderlyingFile->getBuffer("", /*FileSize*/ -1, - /*RequiresNullTerminator*/ false); - if (!Buffer) - return Buffer.getError(); - auto Blob = DB->storeFromString({}, (*Buffer)->getBuffer()); - if (!Blob) - return errorToErrorCode(Blob.takeError()); - return *Blob; - } + cas::ObjectRef getObjectRefForContent() final { return Ref; } - std::error_code close() override { return UnderlyingFile->close(); } + std::error_code close() final { return UnderlyingFile->close(); } }; -class CASProvidingFileSystem : public vfs::ProxyFileSystem { +class CASProvidingFileSystem final : public CASBackedFileSystem { std::shared_ptr DB; + IntrusiveRefCntPtr FS; public: CASProvidingFileSystem(std::shared_ptr DB, IntrusiveRefCntPtr FS) - : ProxyFileSystem(std::move(FS)), DB(std::move(DB)) {} - - ErrorOr> - openFileForRead(const Twine &Path) override { - auto UnderlyingFile = ProxyFileSystem::openFileForRead(Path); - if (!UnderlyingFile) - return UnderlyingFile.getError(); - return std::make_unique(DB, std::move(*UnderlyingFile)); + : DB(std::move(DB)), FS(std::move(FS)) {} + + llvm::ErrorOr status(const Twine &Path) final { + return FS->status(Path); + } + bool exists(const Twine &Path) final { return FS->exists(Path); } + directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) final { + return FS->dir_begin(Dir, EC); + } + llvm::ErrorOr getCurrentWorkingDirectory() const final { + return FS->getCurrentWorkingDirectory(); + } + std::error_code setCurrentWorkingDirectory(const Twine &Path) final { + return FS->setCurrentWorkingDirectory(Path); + } + std::error_code getRealPath(const Twine &Path, + SmallVectorImpl &Output) final { + return FS->getRealPath(Path, Output); + } + std::error_code isLocal(const Twine &Path, bool &Result) final { + return FS->isLocal(Path, Result); + } + + llvm::Expected> + openCASBackedFileForRead(const Twine &Path) final { + auto F = FS->openFileForRead(Path); + if (!F) + return errorCodeToError(F.getError()); + + auto Buffer = (*F)->getBuffer("", /*FileSize*/ -1, + /*RequiresNullTerminator*/ false); + if (!Buffer) + return errorCodeToError(Buffer.getError()); + + auto Blob = DB->storeFromString({}, (*Buffer)->getBuffer()); + if (!Blob) + return Blob.takeError(); + + return std::make_unique(*Blob, std::move(*F)); } -}; + IntrusiveRefCntPtr createThreadSafeProxyFS() final { + return makeIntrusiveRefCnt(DB, FS); + } +}; } // namespace + std::unique_ptr cas::createCASProvidingFileSystem( std::shared_ptr DB, IntrusiveRefCntPtr UnderlyingFS) { diff --git a/llvm/lib/CAS/CachingOnDiskFileSystem.cpp b/llvm/lib/CAS/CachingOnDiskFileSystem.cpp index bb082536b971a..83b1b97601be0 100644 --- a/llvm/lib/CAS/CachingOnDiskFileSystem.cpp +++ b/llvm/lib/CAS/CachingOnDiskFileSystem.cpp @@ -9,6 +9,7 @@ #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/ScopeExit.h" +#include "llvm/CAS/FileSystemCache.h" #include "llvm/CAS/HierarchicalTreeBuilder.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Config/config.h" @@ -18,8 +19,7 @@ using namespace llvm; using namespace llvm::cas; -const char ThreadSafeFileSystem::ID = 0; -void ThreadSafeFileSystem::anchor() {} +const char CachingOnDiskFileSystem::ID = 0; void CachingOnDiskFileSystem::anchor() {} namespace { @@ -86,7 +86,8 @@ class CachingOnDiskFileSystemImpl final : public CachingOnDiskFileSystem { std::optional &FileID) final; ErrorOr status(const Twine &Path) final; bool exists(const Twine &Path) final; - ErrorOr> openFileForRead(const Twine &Path) final; + Expected> + openCASBackedFileForReadImpl(const Twine &Path) final; vfs::directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) final { auto IterOr = getDirectoryIterator(Dir); @@ -209,7 +210,7 @@ CachingOnDiskFileSystem::CachingOnDiskFileSystem( CachingOnDiskFileSystem::CachingOnDiskFileSystem(ObjectStore &DB) : DB(DB) {} -class CachingOnDiskFileSystemImpl::VFSFile : public vfs::File { +class CachingOnDiskFileSystemImpl::VFSFile final : public CASBackedFile { public: ErrorOr status() final { return Entry->getStatus(Name); } @@ -226,9 +227,7 @@ class CachingOnDiskFileSystemImpl::VFSFile : public vfs::File { return Object->getMemoryBuffer(RequestedName.toStringRef(Storage)); } - llvm::ErrorOr> getObjectRefForContent() final { - return Entry->getRef(); - } + cas::ObjectRef getObjectRefForContent() final { return *Entry->getRef(); } /// Closes the file. std::error_code close() final { return std::error_code(); } @@ -453,18 +452,19 @@ bool CachingOnDiskFileSystemImpl::exists(const Twine &Path) { return true; } -ErrorOr> -CachingOnDiskFileSystemImpl::openFileForRead(const Twine &Path) { +Expected> +CachingOnDiskFileSystemImpl::openCASBackedFileForReadImpl(const Twine &Path) { PathStorage PathStorage(Path); StringRef PathRef = PathStorage.Path; Expected ExpectedEntry = lookupPath(PathRef); if (!ExpectedEntry) - return errorToErrorCode(ExpectedEntry.takeError()); + return ExpectedEntry.takeError(); DirectoryEntry *Entry = *ExpectedEntry; if (!Entry->isFile()) - return std::errc::invalid_argument; + return createFileError(Path, + std::make_error_code(std::errc::invalid_argument)); return std::make_unique(DB, *Entry, PathRef); } diff --git a/llvm/lib/Support/PrefixMappingFileSystem.cpp b/llvm/lib/Support/PrefixMappingFileSystem.cpp index 8c2919a0fdaff..e5c1bfcec1db0 100644 --- a/llvm/lib/Support/PrefixMappingFileSystem.cpp +++ b/llvm/lib/Support/PrefixMappingFileSystem.cpp @@ -35,9 +35,6 @@ class PrefixMappingFile : public vfs::File { return Underlying->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile); } - llvm::ErrorOr> getObjectRefForContent() final { - return Underlying->getObjectRefForContent(); - } std::error_code close() final { return Underlying->close(); } }; diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp index f2e93e336902b..cfe3ce4fee930 100644 --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -113,38 +113,19 @@ bool Status::exists() const { File::~File() = default; -llvm::ErrorOr> File::getObjectRefForContent() { - return std::nullopt; -} - FileSystem::~FileSystem() = default; ErrorOr> FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile, - bool IsText, - std::optional *CASContents) { + bool IsText) { auto F = IsText ? openFileForRead(Name) : openFileForReadBinary(Name); if (!F) return F.getError(); - if (CASContents) { - auto CASRef = (*F)->getObjectRefForContent(); - if (!CASRef) - return CASRef.getError(); - *CASContents = *CASRef; - } return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile); } -llvm::ErrorOr> -FileSystem::getObjectRefForFileContent(const Twine &Name) { - auto F = openFileForRead(Name); - if (!F) - return F.getError(); - return (*F)->getObjectRefForContent(); -} - std::error_code FileSystem::makeAbsolute(SmallVectorImpl &Path) const { if (llvm::sys::path::is_absolute(Path)) return {}; @@ -3004,6 +2985,7 @@ void TracingFileSystem::printImpl(raw_ostream &OS, PrintType Type, getUnderlyingFS().print(OS, Type, IndentLevel + 1); } +const char File::ID = 0; const char FileSystem::ID = 0; const char OverlayFileSystem::ID = 0; const char ProxyFileSystem::ID = 0; diff --git a/llvm/tools/llvm-cas/llvm-cas.cpp b/llvm/tools/llvm-cas/llvm-cas.cpp index 3afe6e22fd548..74456c5387b0b 100644 --- a/llvm/tools/llvm-cas/llvm-cas.cpp +++ b/llvm/tools/llvm-cas/llvm-cas.cpp @@ -622,12 +622,13 @@ int getCASIDForFile(ObjectStore &CAS, const CASID &ID, if (!FS) ExitOnErr(FS.takeError()); - auto FileRef = (*FS)->getObjectRefForFileContent(Path.front()); + auto *CASFS = cast(FS->get()); + auto FileRef = CASFS->getObjectRefForFileContent(Path.front()); if (!FileRef) ExitOnErr(errorCodeToError( std::make_error_code(std::errc::no_such_file_or_directory))); - CASID FileID = CAS.getID(**FileRef); + CASID FileID = CAS.getID(*FileRef); outs() << FileID << "\n"; return 0; } diff --git a/llvm/unittests/CAS/CASProvidingFileSystemTest.cpp b/llvm/unittests/CAS/CASProvidingFileSystemTest.cpp index f4da5578cb7d2..a104af9401d8e 100644 --- a/llvm/unittests/CAS/CASProvidingFileSystemTest.cpp +++ b/llvm/unittests/CAS/CASProvidingFileSystemTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/CAS/CASProvidingFileSystem.h" +#include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Testing/Support/Error.h" @@ -25,40 +26,39 @@ TEST(CASProvidingFileSystemTest, Basic) { FS->addFile(Path1, 0, MemoryBuffer::getMemBuffer(Contents1)); FS->addFile(Path2, 0, MemoryBuffer::getMemBuffer(Contents2)); - std::unique_ptr CASFS = createCASProvidingFileSystem(DB, FS); - ASSERT_TRUE(CASFS); + std::unique_ptr VFS = createCASProvidingFileSystem(DB, FS); + ASSERT_TRUE(VFS); + auto &CASFS = cast(*VFS); { - ErrorOr> File = CASFS->openFileForRead(Path1); + std::unique_ptr File; + ASSERT_THAT_ERROR(CASFS.openCASBackedFileForRead(Path1).moveInto(File), + Succeeded()); ASSERT_TRUE(File); - ASSERT_TRUE(*File); - ErrorOr> Ref = - (*File)->getObjectRefForContent(); - ASSERT_TRUE(Ref); - ASSERT_TRUE(*Ref); + cas::ObjectRef Ref = File->getObjectRefForContent(); std::optional BlobContents; - ASSERT_THAT_ERROR(DB->getProxy(**Ref).moveInto(BlobContents), Succeeded()); + ASSERT_THAT_ERROR(DB->getProxy(Ref).moveInto(BlobContents), Succeeded()); EXPECT_EQ(BlobContents->getData(), Contents1); } { - ErrorOr> Ref = - CASFS->getObjectRefForFileContent(Path1); - ASSERT_TRUE(Ref); - ASSERT_TRUE(*Ref); + std::optional Ref; + ASSERT_THAT_ERROR(CASFS.getObjectRefForFileContent(Path1).moveInto(Ref), + Succeeded()); std::optional BlobContents; - ASSERT_THAT_ERROR(DB->getProxy(**Ref).moveInto(BlobContents), Succeeded()); + ASSERT_THAT_ERROR(DB->getProxy(*Ref).moveInto(BlobContents), Succeeded()); EXPECT_EQ(BlobContents->getData(), Contents1); } { - std::optional CASContents; - auto Buf = CASFS->getBufferForFile(Path2, /*FileSize*/ -1, - /*RequiresNullTerminator*/ false, - /*IsVolatile*/ false, /*IsText*/ true, - &CASContents); - ASSERT_TRUE(Buf); - EXPECT_EQ(Contents2, (*Buf)->getBuffer()); - ASSERT_TRUE(CASContents); + std::optional, cas::ObjectRef>> Val; + ASSERT_THAT_ERROR( + CASFS + .getBufferAndObjectRefForFile(Path2, /*FileSize*/ -1, + /*RequiresNullTerminator*/ false, + /*IsVolatile*/ false, /*IsText*/ true) + .moveInto(Val), + Succeeded()); + EXPECT_EQ(Contents2, Val->first->getBuffer()); std::optional BlobContents; - ASSERT_THAT_ERROR(DB->getProxy(*CASContents).moveInto(BlobContents), + ASSERT_THAT_ERROR(DB->getProxy(Val->second).moveInto(BlobContents), Succeeded()); EXPECT_EQ(BlobContents->getData(), Contents2); } @@ -75,25 +75,21 @@ TEST(CASProvidingFileSystemTest, WithCASSupportingFS) { ASSERT_TRUE(UnderlyingFS); std::shared_ptr DB = createInMemoryCAS(); - std::unique_ptr CASFS = + std::unique_ptr VFS = createCASProvidingFileSystem(DB, std::move(UnderlyingFS)); - ASSERT_TRUE(CASFS); + ASSERT_TRUE(VFS); + auto &CASFS = cast(*VFS); - ErrorOr> File = CASFS->openFileForRead(Path); - ASSERT_TRUE(File); - ASSERT_TRUE(*File); - ErrorOr> Ref = - (*File)->getObjectRefForContent(); - ASSERT_TRUE(Ref); - ASSERT_TRUE(*Ref); - std::optional BlobContents; - ASSERT_THAT_ERROR(UnderlyingDB->getProxy(**Ref).moveInto(BlobContents), + std::unique_ptr File; + ASSERT_THAT_ERROR(CASFS.openCASBackedFileForRead(Path).moveInto(File), Succeeded()); + cas::ObjectRef Ref = File->getObjectRefForContent(); + std::optional BlobContents; + ASSERT_THAT_ERROR(DB->getProxy(Ref).moveInto(BlobContents), Succeeded()); EXPECT_EQ(BlobContents->getData(), Contents); - CASID ID = UnderlyingDB->getID(**Ref); + CASID ID = DB->getID(Ref); std::optional Proxy; - // It didn't have to ingest in DB because the underlying FS provided a CAS - // reference. - ASSERT_THAT_ERROR(DB->getProxy(ID).moveInto(Proxy), Failed()); + // The file also ingested into the underlying FS. + ASSERT_THAT_ERROR(UnderlyingDB->getProxy(ID).moveInto(Proxy), Succeeded()); }