diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index d26a452cf94cc..9295b2bee115c 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -24,6 +24,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Support/BuryPointer.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" #include #include #include @@ -92,6 +93,9 @@ class CompilerInstance : public ModuleLoader { /// The file manager. IntrusiveRefCntPtr FileMgr; + /// The output context. + IntrusiveRefCntPtr TheOutputBackend; + /// The source manager. IntrusiveRefCntPtr SourceMgr; @@ -164,22 +168,8 @@ class CompilerInstance : public ModuleLoader { /// The stream for verbose output. raw_ostream *VerboseOutputStream = &llvm::errs(); - /// Holds information about the output file. - /// - /// If TempFilename is not empty we must rename it to Filename at the end. - /// TempFilename may be empty and Filename non-empty if creating the temporary - /// failed. - struct OutputFile { - std::string Filename; - std::optional File; - - OutputFile(std::string filename, - std::optional file) - : Filename(std::move(filename)), File(std::move(file)) {} - }; - /// The list of active output files. - std::list OutputFiles; + std::list OutputFiles; /// Force an output buffer. std::unique_ptr OutputStream; @@ -430,6 +420,22 @@ class CompilerInstance : public ModuleLoader { /// Replace the current file manager and virtual file system. void setFileManager(FileManager *Value); + /// @name Output Backend. + /// { + + /// Set the output backend. + void + setOutputBackend(IntrusiveRefCntPtr NewOutputs); + + /// Create an output manager. + void createOutputBackend(); + + bool hasOutputBackend() const { return bool(TheOutputBackend); } + + llvm::vfs::OutputBackend &getOutputBackend(); + llvm::vfs::OutputBackend &getOrCreateOutputBackend(); + + /// } /// @} /// @name Source Manager diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index d18371f21a9d8..56bb53f4e3bbf 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -54,6 +54,8 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/Support/VirtualOutputError.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Host.h" #include @@ -516,6 +518,10 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) { collectVFSEntries(*this, ModuleDepCollector); } + // Modules need an output manager. + if (!hasOutputBackend()) + createOutputBackend(); + for (auto &Listener : DependencyCollectors) Listener->attachToPreprocessor(*PP); @@ -759,32 +765,19 @@ void CompilerInstance::createSema(TranslationUnitKind TUKind, void CompilerInstance::clearOutputFiles(bool EraseFiles) { // The ASTConsumer can own streams that write to the output files. assert(!hasASTConsumer() && "ASTConsumer should be reset"); - // Ignore errors that occur when trying to discard the temp file. - for (OutputFile &OF : OutputFiles) { - if (EraseFiles) { - if (OF.File) - consumeError(OF.File->discard()); - if (!OF.Filename.empty()) - llvm::sys::fs::remove(OF.Filename); - continue; - } - - if (!OF.File) - continue; - - if (OF.File->TmpName.empty()) { - consumeError(OF.File->discard()); - continue; - } - - llvm::Error E = OF.File->keep(OF.Filename); - if (!E) - continue; - - getDiagnostics().Report(diag::err_unable_to_rename_temp) - << OF.File->TmpName << OF.Filename << std::move(E); - - llvm::sys::fs::remove(OF.File->TmpName); + if (!EraseFiles) { + for (auto &O : OutputFiles) + llvm::handleAllErrors( + O.keep(), + [&](const llvm::vfs::TempFileOutputError &E) { + getDiagnostics().Report(diag::err_unable_to_rename_temp) + << E.getTempPath() << E.getOutputPath() + << E.convertToErrorCode().message(); + }, + [&](const llvm::vfs::OutputError &E) { + getDiagnostics().Report(diag::err_fe_unable_to_open_output) + << E.getOutputPath() << E.convertToErrorCode().message(); + }); } OutputFiles.clear(); if (DeleteBuiltModules) { @@ -818,6 +811,29 @@ std::unique_ptr CompilerInstance::createNullOutputFile() { return std::make_unique(); } +void CompilerInstance::setOutputBackend( + IntrusiveRefCntPtr NewOutputs) { + assert(!TheOutputBackend && "Already has an output manager"); + TheOutputBackend = std::move(NewOutputs); +} + +void CompilerInstance::createOutputBackend() { + assert(!TheOutputBackend && "Already has an output manager"); + TheOutputBackend = + llvm::makeIntrusiveRefCnt(); +} + +llvm::vfs::OutputBackend &CompilerInstance::getOutputBackend() { + assert(TheOutputBackend); + return *TheOutputBackend; +} + +llvm::vfs::OutputBackend &CompilerInstance::getOrCreateOutputBackend() { + if (!hasOutputBackend()) + createOutputBackend(); + return getOutputBackend(); +} + std::unique_ptr CompilerInstance::createOutputFile(StringRef OutputPath, bool Binary, bool RemoveFileOnSignal, bool UseTemporary, @@ -852,98 +868,20 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary, OutputPath = *AbsPath; } - std::unique_ptr OS; - std::optional OSFile; - - if (UseTemporary) { - if (OutputPath == "-") - UseTemporary = false; - else { - llvm::sys::fs::file_status Status; - llvm::sys::fs::status(OutputPath, Status); - if (llvm::sys::fs::exists(Status)) { - // Fail early if we can't write to the final destination. - if (!llvm::sys::fs::can_write(OutputPath)) - return llvm::errorCodeToError( - make_error_code(llvm::errc::operation_not_permitted)); - - // Don't use a temporary if the output is a special file. This handles - // things like '-o /dev/null' - if (!llvm::sys::fs::is_regular_file(Status)) - UseTemporary = false; - } - } - } - - std::optional Temp; - if (UseTemporary) { - // Create a temporary file. - // Insert -%%%%%%%% before the extension (if any), and because some tools - // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build - // artifacts, also append .tmp. - StringRef OutputExtension = llvm::sys::path::extension(OutputPath); - SmallString<128> TempPath = - StringRef(OutputPath).drop_back(OutputExtension.size()); - TempPath += "-%%%%%%%%"; - TempPath += OutputExtension; - TempPath += ".tmp"; - llvm::sys::fs::OpenFlags BinaryFlags = - Binary ? llvm::sys::fs::OF_None : llvm::sys::fs::OF_Text; - Expected ExpectedFile = - llvm::sys::fs::TempFile::create( - TempPath, llvm::sys::fs::all_read | llvm::sys::fs::all_write, - BinaryFlags); - - llvm::Error E = handleErrors( - ExpectedFile.takeError(), [&](const llvm::ECError &E) -> llvm::Error { - std::error_code EC = E.convertToErrorCode(); - if (CreateMissingDirectories && - EC == llvm::errc::no_such_file_or_directory) { - StringRef Parent = llvm::sys::path::parent_path(OutputPath); - EC = llvm::sys::fs::create_directories(Parent); - if (!EC) { - ExpectedFile = llvm::sys::fs::TempFile::create( - TempPath, llvm::sys::fs::all_read | llvm::sys::fs::all_write, - BinaryFlags); - if (!ExpectedFile) - return llvm::errorCodeToError( - llvm::errc::no_such_file_or_directory); - } - } - return llvm::errorCodeToError(EC); - }); - - if (E) { - consumeError(std::move(E)); - } else { - Temp = std::move(ExpectedFile.get()); - OS.reset(new llvm::raw_fd_ostream(Temp->FD, /*shouldClose=*/false)); - OSFile = Temp->TmpName; - } - // If we failed to create the temporary, fallback to writing to the file - // directly. This handles the corner case where we cannot write to the - // directory, but can write to the file. - } - - if (!OS) { - OSFile = OutputPath; - std::error_code EC; - OS.reset(new llvm::raw_fd_ostream( - *OSFile, EC, - (Binary ? llvm::sys::fs::OF_None : llvm::sys::fs::OF_TextWithCRLF))); - if (EC) - return llvm::errorCodeToError(EC); - } - - // Add the output file -- but don't try to remove "-", since this means we are - // using stdin. - OutputFiles.emplace_back(((OutputPath != "-") ? OutputPath : "").str(), - std::move(Temp)); - - if (!Binary || OS->supportsSeeking()) - return std::move(OS); - - return std::make_unique(std::move(OS)); + using namespace llvm::vfs; + Expected O = getOrCreateOutputBackend().createFile( + OutputPath, + OutputConfig() + .setTextWithCRLF(!Binary) + .setDiscardOnSignal(RemoveFileOnSignal) + .setAtomicWrite(UseTemporary) + .setImplyCreateDirectories(UseTemporary && CreateMissingDirectories)); + if (!O) + return O.takeError(); + + O->discardOnDestroy([](llvm::Error E) { consumeError(std::move(E)); }); + OutputFiles.push_back(std::move(*O)); + return OutputFiles.back().createProxy(); } // Initialization Utilities diff --git a/llvm/include/llvm/Support/HashingOutputBackend.h b/llvm/include/llvm/Support/HashingOutputBackend.h new file mode 100644 index 0000000000000..d2e79663f5526 --- /dev/null +++ b/llvm/include/llvm/Support/HashingOutputBackend.h @@ -0,0 +1,112 @@ +//===- HashingOutputBackends.h - Hashing output backends --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H +#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/HashBuilder.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm::vfs { + +/// raw_pwrite_stream that writes to a hasher. +template +class HashingStream : public llvm::raw_pwrite_stream { +private: + SmallVector Buffer; + raw_svector_ostream OS; + + using HashBuilderT = HashBuilder; + HashBuilderT Builder; + + void write_impl(const char *Ptr, size_t Size) override { + OS.write(Ptr, Size); + } + + void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override { + OS.pwrite(Ptr, Size, Offset); + } + + uint64_t current_pos() const override { return OS.str().size(); } + +public: + HashingStream() : OS(Buffer) { SetUnbuffered(); } + + auto final() { + Builder.update(OS.str()); + return Builder.final(); + } +}; + +template class HashingOutputFile; + +/// An output backend that only generates the hash for outputs. +template class HashingOutputBackend : public OutputBackend { +private: + friend class HashingOutputFile; + void addOutputFile(StringRef Path, StringRef Hash) { + OutputHashes[Path] = std::string(Hash); + } + +protected: + IntrusiveRefCntPtr cloneImpl() const override { + return const_cast *>(this); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + return std::make_unique>(Path, *this); + } + +public: + /// Iterator for all the output file names. + auto outputFiles() const { return OutputHashes.keys(); } + + /// Get hash value for the output files in hex representation. + /// Return None if the requested path is not generated. + std::optional getHashValueForFile(StringRef Path) { + auto F = OutputHashes.find(Path); + if (F == OutputHashes.end()) + return std::nullopt; + return toHex(F->second); + } + +private: + StringMap OutputHashes; +}; + +/// HashingOutputFile. +template +class HashingOutputFile final : public OutputFileImpl { +public: + Error keep() override { + auto Result = OS.final(); + Backend.addOutputFile(OutputPath, toStringRef(Result)); + return Error::success(); + } + Error discard() override { return Error::success(); } + raw_pwrite_stream &getOS() override { return OS; } + + HashingOutputFile(StringRef OutputPath, + HashingOutputBackend &Backend) + : OutputPath(OutputPath.str()), Backend(Backend) {} + +private: + const std::string OutputPath; + HashingStream OS; + HashingOutputBackend &Backend; +}; + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H diff --git a/llvm/include/llvm/Support/VirtualOutputBackend.h b/llvm/include/llvm/Support/VirtualOutputBackend.h new file mode 100644 index 0000000000000..2328252c7054f --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputBackend.h @@ -0,0 +1,62 @@ +//===- VirtualOutputBackend.h - Output virtualization -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H +#define LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/VirtualOutputFile.h" + +namespace llvm::vfs { + +/// Interface for virtualized outputs. +/// +/// If virtual functions are added here, also add them to \a +/// ProxyOutputBackend. +class OutputBackend : public RefCountedBase { + virtual void anchor(); + +public: + /// Get a backend that points to the same destination as this one but that + /// has independent settings. + /// + /// Not thread-safe, but all operations are thread-safe when performed on + /// separate clones of the same backend. + IntrusiveRefCntPtr clone() const { return cloneImpl(); } + + /// Create a file. If \p Config is \c std::nullopt, uses the backend's default + /// OutputConfig (may match \a OutputConfig::OutputConfig(), or may + /// have been customized). + /// + /// Thread-safe. + Expected + createFile(const Twine &Path, + std::optional Config = std::nullopt); + +protected: + /// Must be thread-safe. Virtual function has a different name than \a + /// clone() so that implementations can override the return value. + virtual IntrusiveRefCntPtr cloneImpl() const = 0; + + /// Create a file for \p Path. Must be thread-safe. + /// + /// \pre \p Config is valid or std::nullopt. + virtual Expected> + createFileImpl(StringRef Path, std::optional Config) = 0; + + OutputBackend() = default; + +public: + virtual ~OutputBackend() = default; +}; + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H diff --git a/llvm/include/llvm/Support/VirtualOutputBackends.h b/llvm/include/llvm/Support/VirtualOutputBackends.h new file mode 100644 index 0000000000000..6f702000d77b3 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputBackends.h @@ -0,0 +1,110 @@ +//===- VirtualOutputBackends.h - Virtual output backends --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H +#define LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" + +namespace llvm::vfs { + +/// Create a backend that ignores all output. +IntrusiveRefCntPtr makeNullOutputBackend(); + +/// Make a backend where \a OutputBackend::createFile() forwards to +/// \p UnderlyingBackend when \p Filter is true, and otherwise returns a +/// \a NullOutput. +IntrusiveRefCntPtr makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + std::function)> Filter); + +/// Create a backend that forwards \a OutputBackend::createFile() to both \p +/// Backend1 and \p Backend2 and sends content to both places. +IntrusiveRefCntPtr +makeMirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2); + +/// A helper class for proxying another backend, with the default +/// implementation to forward to the underlying backend. +class ProxyOutputBackend : public OutputBackend { + void anchor() override; + +protected: + // Require subclass to implement cloneImpl(). + // + // IntrusiveRefCntPtr cloneImpl() const override; + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + OutputFile File; + if (Error E = UnderlyingBackend->createFile(Path, Config).moveInto(File)) + return std::move(E); + return File.takeImpl(); + } + + OutputBackend &getUnderlyingBackend() const { return *UnderlyingBackend; } + +public: + ProxyOutputBackend(IntrusiveRefCntPtr UnderlyingBackend) + : UnderlyingBackend(std::move(UnderlyingBackend)) { + assert(this->UnderlyingBackend && "Expected non-null backend"); + } + +private: + IntrusiveRefCntPtr UnderlyingBackend; +}; + +/// An output backend that creates files on disk, wrapping APIs in sys::fs. +class OnDiskOutputBackend : public OutputBackend { + void anchor() override; + +protected: + IntrusiveRefCntPtr cloneImpl() const override { + return clone(); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override; + +public: + /// Resolve an absolute path. + Error makeAbsolute(SmallVectorImpl &Path) const; + + /// On disk output settings. + struct OutputSettings { + /// Register output files to be deleted if a signal is received. Also + /// disabled for outputs with \a OutputConfig::getNoDiscardOnSignal(). + bool DisableRemoveOnSignal = false; + + /// Disable temporary files. Also disabled for outputs with \a + /// OutputConfig::getNoAtomicWrite(). + bool DisableTemporaries = false; + + // Default configuration for this backend. + OutputConfig DefaultConfig; + }; + + IntrusiveRefCntPtr clone() const { + auto Clone = makeIntrusiveRefCnt(); + Clone->Settings = Settings; + return Clone; + } + + OnDiskOutputBackend() = default; + + /// Settings for this backend. + /// + /// Access is not thread-safe. + OutputSettings Settings; +}; + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.def b/llvm/include/llvm/Support/VirtualOutputConfig.def new file mode 100644 index 0000000000000..0b6a765cbd4c0 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputConfig.def @@ -0,0 +1,26 @@ +//===- VirtualOutputConfig.def - Virtual output config defs -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef HANDLE_OUTPUT_CONFIG_FLAG +#error "Missing macro definition of HANDLE_OUTPUT_CONFIG_FLAG" +#endif + +// Define HANDLE_OUTPUT_CONFIG_FLAG before including. +// +// #define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) + +HANDLE_OUTPUT_CONFIG_FLAG(Text, false) // OF_Text. +HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF. +HANDLE_OUTPUT_CONFIG_FLAG(Append, false) // OF_Append. +HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal. +HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries. +HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true) +// Skip atomic write if existing file content is the same +HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false) + +#undef HANDLE_OUTPUT_CONFIG_FLAG diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.h b/llvm/include/llvm/Support/VirtualOutputConfig.h new file mode 100644 index 0000000000000..d93bbf5ca63a0 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputConfig.h @@ -0,0 +1,91 @@ +//===- VirtualOutputConfig.h - Virtual output configuration -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H +#define LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H + +#include "llvm/Support/Compiler.h" +#include + +namespace llvm { + +class raw_ostream; + +namespace sys { +namespace fs { +enum OpenFlags : unsigned; +} // end namespace fs +} // end namespace sys + +namespace vfs { + +namespace detail { +/// Unused and empty base class to allow OutputConfig constructor to be +/// constexpr, with commas before every field's initializer. +struct EmptyBaseClass {}; +} // namespace detail + +/// Full configuration for an output for use by the \a OutputBackend. Each +/// configuration flag is either \c true or \c false. +struct OutputConfig : detail::EmptyBaseClass { +public: + void print(raw_ostream &OS) const; + void dump() const; + +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \ + constexpr bool get##NAME() const { return NAME; } \ + constexpr bool getNo##NAME() const { return !NAME; } \ + constexpr OutputConfig &set##NAME(bool Value) { \ + NAME = Value; \ + return *this; \ + } \ + constexpr OutputConfig &set##NAME() { return set##NAME(true); } \ + constexpr OutputConfig &setNo##NAME() { return set##NAME(false); } +#include "llvm/Support/VirtualOutputConfig.def" + + constexpr OutputConfig &setBinary() { return setNoText().setNoCRLF(); } + constexpr OutputConfig &setTextWithCRLF() { return setText().setCRLF(); } + constexpr OutputConfig &setTextWithCRLF(bool Value) { + return Value ? setText().setCRLF() : setBinary(); + } + constexpr bool getTextWithCRLF() const { return getText() && getCRLF(); } + constexpr bool getBinary() const { return !getText(); } + + /// Updates Text and CRLF flags based on \a sys::fs::OF_Text and \a + /// sys::fs::OF_CRLF in \p Flags. Rejects CRLF without Text (calling + /// \a setBinary()). + OutputConfig &setOpenFlags(const sys::fs::OpenFlags &Flags); + + constexpr OutputConfig() + : EmptyBaseClass() +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) , NAME(DEFAULT) +#include "llvm/Support/VirtualOutputConfig.def" + { + } + + constexpr bool operator==(OutputConfig RHS) const { +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \ + if (NAME != RHS.NAME) \ + return false; +#include "llvm/Support/VirtualOutputConfig.def" + return true; + } + constexpr bool operator!=(OutputConfig RHS) const { return !operator==(RHS); } + +private: +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) bool NAME : 1; +#include "llvm/Support/VirtualOutputConfig.def" +}; + +} // namespace vfs + +raw_ostream &operator<<(raw_ostream &OS, vfs::OutputConfig Config); + +} // namespace llvm + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H diff --git a/llvm/include/llvm/Support/VirtualOutputError.h b/llvm/include/llvm/Support/VirtualOutputError.h new file mode 100644 index 0000000000000..5459fae4552d5 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputError.h @@ -0,0 +1,134 @@ +//===- VirtualOutputError.h - Errors for output virtualization --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTERROR_H +#define LLVM_SUPPORT_VIRTUALOUTPUTERROR_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualOutputConfig.h" + +namespace llvm::vfs { + +const std::error_category &output_category(); + +enum class OutputErrorCode { + // Error code 0 is absent. Use std::error_code() instead. + not_closed = 1, + invalid_config, + already_closed, + has_open_proxy, +}; + +inline std::error_code make_error_code(OutputErrorCode EV) { + return std::error_code(static_cast(EV), output_category()); +} + +/// Error related to an \a OutputFile. Derives from \a ECError and adds \a +/// getOutputPath(). +class OutputError : public ErrorInfo { + void anchor() override; + +public: + StringRef getOutputPath() const { return OutputPath; } + void log(raw_ostream &OS) const override { + OS << getOutputPath() << ": "; + ECError::log(OS); + } + + // Used by ErrorInfo::classID. + static char ID; + + OutputError(const Twine &OutputPath, std::error_code EC) + : ErrorInfo(EC), OutputPath(OutputPath.str()) { + assert(EC && "Cannot create OutputError from success EC"); + } + + OutputError(const Twine &OutputPath, OutputErrorCode EV) + : ErrorInfo(make_error_code(EV)), + OutputPath(OutputPath.str()) { + assert(EC && "Cannot create OutputError from success EC"); + } + +private: + std::string OutputPath; +}; + +/// Return \a Error::success() or use \p OutputPath to create an \a +/// OutputError, depending on \p EC. +inline Error convertToOutputError(const Twine &OutputPath, std::error_code EC) { + if (EC) + return make_error(OutputPath, EC); + return Error::success(); +} + +/// Error related to an OutputConfig for an \a OutputFile. Derives from \a +/// OutputError and adds \a getConfig(). +class OutputConfigError : public ErrorInfo { + void anchor() override; + +public: + OutputConfig getConfig() const { return Config; } + void log(raw_ostream &OS) const override { + OutputError::log(OS); + OS << ": " << Config; + } + + // Used by ErrorInfo::classID. + static char ID; + + OutputConfigError(OutputConfig Config, const Twine &OutputPath) + : ErrorInfo( + OutputPath, OutputErrorCode::invalid_config), + Config(Config) {} + +private: + OutputConfig Config; +}; + +/// Error related to a temporary file for an \a OutputFile. Derives from \a +/// OutputError and adds \a getTempPath(). +class TempFileOutputError : public ErrorInfo { + void anchor() override; + +public: + StringRef getTempPath() const { return TempPath; } + void log(raw_ostream &OS) const override { + OS << getTempPath() << " => "; + OutputError::log(OS); + } + + // Used by ErrorInfo::classID. + static char ID; + + TempFileOutputError(const Twine &TempPath, const Twine &OutputPath, + std::error_code EC) + : ErrorInfo(OutputPath, EC), + TempPath(TempPath.str()) {} + + TempFileOutputError(const Twine &TempPath, const Twine &OutputPath, + OutputErrorCode EV) + : ErrorInfo(OutputPath, EV), + TempPath(TempPath.str()) {} + +private: + std::string TempPath; +}; + +/// Return \a Error::success() or use \p TempPath and \p OutputPath to create a +/// \a TempFileOutputError, depending on \p EC. +inline Error convertToTempFileOutputError(const Twine &TempPath, + const Twine &OutputPath, + std::error_code EC) { + if (EC) + return make_error(TempPath, OutputPath, EC); + return Error::success(); +} + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTERROR_H diff --git a/llvm/include/llvm/Support/VirtualOutputFile.h b/llvm/include/llvm/Support/VirtualOutputFile.h new file mode 100644 index 0000000000000..0bf6c58f30484 --- /dev/null +++ b/llvm/include/llvm/Support/VirtualOutputFile.h @@ -0,0 +1,162 @@ +//===- VirtualOutputFile.h - Output file virtualization ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_VIRTUALOUTPUTFILE_H +#define LLVM_SUPPORT_VIRTUALOUTPUTFILE_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ExtensibleRTTI.h" +#include "llvm/Support/VirtualOutputError.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm::vfs { + +class OutputFileImpl : public RTTIExtends { + void anchor() override; + +public: + static char ID; + virtual ~OutputFileImpl() = default; + + virtual Error keep() = 0; + virtual Error discard() = 0; + virtual raw_pwrite_stream &getOS() = 0; +}; + +class NullOutputFileImpl final + : public RTTIExtends { + void anchor() override; + +public: + static char ID; + Error keep() final { return Error::success(); } + Error discard() final { return Error::success(); } + raw_pwrite_stream &getOS() final { return OS; } + +private: + raw_null_ostream OS; +}; + +/// A virtualized output file that writes to a specific backend. +/// +/// One of \a keep(), \a discard(), or \a discardOnDestroy() must be called +/// before destruction. +class OutputFile { +public: + StringRef getPath() const { return Path; } + + /// Check if \a keep() or \a discard() has already been called. + bool isOpen() const { return bool(Impl); } + + explicit operator bool() const { return isOpen(); } + + raw_pwrite_stream &getOS() { + assert(isOpen() && "Expected open output stream"); + return Impl->getOS(); + } + operator raw_pwrite_stream &() { return getOS(); } + template raw_ostream &operator<<(T &&V) { + return getOS() << std::forward(V); + } + + /// Keep an output. Errors if this fails. + /// + /// If it has already been closed, calls \a report_fatal_error(). + /// + /// If there's an open proxy from \a createProxy(), calls \a discard() to + /// clean up temporaries followed by \a report_fatal_error(). + Error keep(); + + /// Discard an output, cleaning up any temporary state. Errors if clean-up + /// fails. + /// + /// If it has already been closed, calls \a report_fatal_error(). + Error discard(); + + /// Discard the output when destroying it if it's still open, sending the + /// result to \a Handler. + void discardOnDestroy(unique_function Handler) { + DiscardOnDestroyHandler = std::move(Handler); + } + + /// Create a proxy stream for clients that need to pass an owned stream to a + /// producer. Errors if there's already a proxy. The proxy must be deleted + /// before calling \a keep(). The proxy will crash if it's written to after + /// calling \a discard(). + Expected> createProxy(); + + bool hasOpenProxy() const { return OpenProxy; } + + /// Take the implementation. + /// + /// \pre \a hasOpenProxy() is false. + /// \pre \a discardOnDestroy() has not been called. + std::unique_ptr takeImpl() { + assert(!hasOpenProxy() && "Unexpected open proxy"); + assert(!DiscardOnDestroyHandler && "Unexpected discard handler"); + return std::move(Impl); + } + + /// Check whether this is a null output file. + bool isNull() const { return Impl && isa(*Impl); } + + OutputFile() = default; + + explicit OutputFile(const Twine &Path, std::unique_ptr Impl) + : Path(Path.str()), Impl(std::move(Impl)) { + assert(this->Impl && "Expected open output file"); + } + + ~OutputFile() { destroy(); } + OutputFile(OutputFile &&O) { moveFrom(O); } + OutputFile &operator=(OutputFile &&O) { + destroy(); + return moveFrom(O); + } + +private: + /// Destroy \a Impl. Reports fatal error if the file is open and there's no + /// handler from \a discardOnDestroy(). + void destroy(); + OutputFile &moveFrom(OutputFile &O) { + Path = std::move(O.Path); + Impl = std::move(O.Impl); + DiscardOnDestroyHandler = std::move(O.DiscardOnDestroyHandler); + OpenProxy = O.OpenProxy; + O.OpenProxy = nullptr; + return *this; + } + + std::string Path; + std::unique_ptr Impl; + unique_function DiscardOnDestroyHandler; + + class TrackedProxy; + TrackedProxy *OpenProxy = nullptr; +}; + +/// Update \p File to silently discard itself if it's still open when it's +/// destroyed. +inline void consumeDiscardOnDestroy(OutputFile &File) { + File.discardOnDestroy(consumeError); +} + +/// Update \p File to silently discard itself if it's still open when it's +/// destroyed. +inline Expected consumeDiscardOnDestroy(Expected File) { + if (File) + consumeDiscardOnDestroy(*File); + return File; +} + +} // namespace llvm::vfs + +#endif // LLVM_SUPPORT_VIRTUALOUTPUTFILE_H diff --git a/llvm/include/llvm/Support/raw_ostream_proxy.h b/llvm/include/llvm/Support/raw_ostream_proxy.h new file mode 100644 index 0000000000000..093d0a927833f --- /dev/null +++ b/llvm/include/llvm/Support/raw_ostream_proxy.h @@ -0,0 +1,158 @@ +//===- raw_ostream_proxy.h - Proxies for raw output streams -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_RAW_OSTREAM_PROXY_H +#define LLVM_SUPPORT_RAW_OSTREAM_PROXY_H + +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +/// Common bits for \a raw_ostream_proxy_adaptor<>, split out to dedup in +/// template instantions. +class raw_ostream_proxy_adaptor_base { +protected: + raw_ostream_proxy_adaptor_base() = delete; + raw_ostream_proxy_adaptor_base(const raw_ostream_proxy_adaptor_base &) = + delete; + + explicit raw_ostream_proxy_adaptor_base(raw_ostream &OS) + : OS(&OS), PreferredBufferSize(OS.GetBufferSize()) { + // Drop OS's buffer to make this->flush() forward. This proxy will add a + // buffer in its place. + OS.SetUnbuffered(); + } + + ~raw_ostream_proxy_adaptor_base() { + assert(!OS && "Derived objects should call resetProxiedOS()"); + } + + /// Stop proxying the stream, taking the derived object by reference as \p + /// ThisProxyOS. Updates \p ThisProxyOS to stop buffering before setting \a + /// OS to \c nullptr, ensuring that future writes crash immediately. + void resetProxiedOS(raw_ostream &ThisProxyOS) { + ThisProxyOS.SetUnbuffered(); + OS = nullptr; + } + + bool hasProxiedOS() const { return OS; } + raw_ostream &getProxiedOS() const { + assert(OS && "raw_ostream_proxy_adaptor use after reset"); + return *OS; + } + size_t getPreferredBufferSize() const { return PreferredBufferSize; } + +private: + raw_ostream *OS; + + /// Caches the value of OS->GetBufferSize() at construction time. + size_t PreferredBufferSize; +}; + +/// Adaptor to create a stream class that proxies another \a raw_ostream. +/// +/// Use \a raw_ostream_proxy_adaptor<> directly to implement an abstract +/// derived class of \a raw_ostream as a proxy. Otherwise use \a +/// raw_ostream_proxy. +/// +/// Most operations are forwarded to the proxied stream. +/// +/// If the proxied stream is buffered, the buffer is dropped and moved to this +/// stream. This allows \a flush() to work correctly, flushing immediately from +/// the proxy through to the final stream, and avoids any wasteful +/// double-buffering. +/// +/// \a enable_colors() changes both the proxied stream and the proxy itself. +/// \a is_displayed() and \a has_colors() are forwarded to the proxy. \a +/// changeColor(), resetColor(), and \a reverseColor() are not forwarded, since +/// they need to call \a flush() and the buffer lives in the proxy. +template +class raw_ostream_proxy_adaptor : public RawOstreamT, + public raw_ostream_proxy_adaptor_base { + void write_impl(const char *Ptr, size_t Size) override { + getProxiedOS().write(Ptr, Size); + } + uint64_t current_pos() const override { return getProxiedOS().tell(); } + size_t preferred_buffer_size() const override { + return getPreferredBufferSize(); + } + +public: + void reserveExtraSpace(uint64_t ExtraSize) override { + getProxiedOS().reserveExtraSpace(ExtraSize); + } + bool is_displayed() const override { return getProxiedOS().is_displayed(); } + bool has_colors() const override { return getProxiedOS().has_colors(); } + void enable_colors(bool enable) override { + RawOstreamT::enable_colors(enable); + getProxiedOS().enable_colors(enable); + } + + ~raw_ostream_proxy_adaptor() override { resetProxiedOS(); } + +protected: + template + explicit raw_ostream_proxy_adaptor(raw_ostream &OS, ArgsT &&...Args) + : RawOstreamT(std::forward(Args)...), + raw_ostream_proxy_adaptor_base(OS) {} + + /// Stop proxying the stream. Flush and set up a crash for future writes. + /// + /// For example, this can simplify logic when a subclass might have a longer + /// lifetime than the stream it proxies. + void resetProxiedOS() { + raw_ostream_proxy_adaptor_base::resetProxiedOS(*this); + } + void resetProxiedOS(raw_ostream &) = delete; +}; + +/// Adaptor for creating a stream that proxies a \a raw_pwrite_stream. +template +class raw_pwrite_stream_proxy_adaptor + : public raw_ostream_proxy_adaptor { + using RawOstreamAdaptorT = raw_ostream_proxy_adaptor; + + void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override { + this->flush(); + getProxiedOS().pwrite(Ptr, Size, Offset); + } + +protected: + raw_pwrite_stream_proxy_adaptor() = default; + template + explicit raw_pwrite_stream_proxy_adaptor(raw_pwrite_stream &OS, + ArgsT &&...Args) + : RawOstreamAdaptorT(OS, std::forward(Args)...) {} + + raw_pwrite_stream &getProxiedOS() const { + return static_cast(RawOstreamAdaptorT::getProxiedOS()); + } +}; + +/// Non-owning proxy for a \a raw_ostream. Enables passing a stream into of an +/// API that takes ownership. +class raw_ostream_proxy : public raw_ostream_proxy_adaptor<> { + void anchor() override; + +public: + raw_ostream_proxy(raw_ostream &OS) : raw_ostream_proxy_adaptor<>(OS) {} +}; + +/// Non-owning proxy for a \a raw_pwrite_stream. Enables passing a stream +/// into of an API that takes ownership. +class raw_pwrite_stream_proxy : public raw_pwrite_stream_proxy_adaptor<> { + void anchor() override; + +public: + raw_pwrite_stream_proxy(raw_pwrite_stream &OS) + : raw_pwrite_stream_proxy_adaptor<>(OS) {} +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_RAW_OSTREAM_PROXY_H diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index b96d62c7a6224..2ff976882e9c1 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -247,11 +247,17 @@ add_llvm_component_library(LLVMSupport UnicodeNameToCodepointGenerated.cpp VersionTuple.cpp VirtualFileSystem.cpp + VirtualOutputBackend.cpp + VirtualOutputBackends.cpp + VirtualOutputConfig.cpp + VirtualOutputError.cpp + VirtualOutputFile.cpp WithColor.cpp YAMLParser.cpp YAMLTraits.cpp raw_os_ostream.cpp raw_ostream.cpp + raw_ostream_proxy.cpp regcomp.c regerror.c regexec.c diff --git a/llvm/lib/Support/VirtualOutputBackend.cpp b/llvm/lib/Support/VirtualOutputBackend.cpp new file mode 100644 index 0000000000000..bf50c66b0bf0c --- /dev/null +++ b/llvm/lib/Support/VirtualOutputBackend.cpp @@ -0,0 +1,38 @@ +//===- VirtualOutputBackend.cpp - Virtualize compiler outputs -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements vfs::OutputBackend. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/ADT/StringExtras.h" + +using namespace llvm; +using namespace llvm::vfs; + +void OutputBackend::anchor() {} + +Expected +OutputBackend::createFile(const Twine &Path_, + std::optional Config) { + SmallString<128> Path; + Path_.toVector(Path); + + if (Config) { + // Check for invalid configs. + if (!Config->getText() && Config->getCRLF()) + return make_error(*Config, Path); + } + + std::unique_ptr Impl; + if (Error E = createFileImpl(Path, Config).moveInto(Impl)) + return std::move(E); + assert(Impl && "Expected valid Impl or Error"); + return OutputFile(Path, std::move(Impl)); +} diff --git a/llvm/lib/Support/VirtualOutputBackends.cpp b/llvm/lib/Support/VirtualOutputBackends.cpp new file mode 100644 index 0000000000000..e3b7464dd59b6 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputBackends.cpp @@ -0,0 +1,594 @@ +//===- VirtualOutputBackends.cpp - Virtual output backends ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements vfs::OutputBackend. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/LockFileManager.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::vfs; + +void ProxyOutputBackend::anchor() {} +void OnDiskOutputBackend::anchor() {} + +IntrusiveRefCntPtr vfs::makeNullOutputBackend() { + struct NullOutputBackend : public OutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + return const_cast(this); + } + Expected> + createFileImpl(StringRef Path, std::optional) override { + return std::make_unique(); + } + }; + + return makeIntrusiveRefCnt(); +} + +IntrusiveRefCntPtr vfs::makeFilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + std::function)> Filter) { + struct FilteringOutputBackend : public ProxyOutputBackend { + Expected> + createFileImpl(StringRef Path, + std::optional Config) override { + if (Filter(Path, Config)) + return ProxyOutputBackend::createFileImpl(Path, Config); + return std::make_unique(); + } + + IntrusiveRefCntPtr cloneImpl() const override { + return makeIntrusiveRefCnt( + getUnderlyingBackend().clone(), Filter); + } + + FilteringOutputBackend( + IntrusiveRefCntPtr UnderlyingBackend, + std::function)> Filter) + : ProxyOutputBackend(std::move(UnderlyingBackend)), + Filter(std::move(Filter)) { + assert(this->Filter && "Expected a non-null function"); + } + std::function)> Filter; + }; + + return makeIntrusiveRefCnt( + std::move(UnderlyingBackend), std::move(Filter)); +} + +IntrusiveRefCntPtr +vfs::makeMirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) { + struct ProxyOutputBackend1 : public ProxyOutputBackend { + using ProxyOutputBackend::ProxyOutputBackend; + }; + struct ProxyOutputBackend2 : public ProxyOutputBackend { + using ProxyOutputBackend::ProxyOutputBackend; + }; + struct MirroringOutput final : public OutputFileImpl, raw_pwrite_stream { + Error keep() final { + flush(); + return joinErrors(F1->keep(), F2->keep()); + } + Error discard() final { + flush(); + return joinErrors(F1->discard(), F2->discard()); + } + raw_pwrite_stream &getOS() final { return *this; } + + void write_impl(const char *Ptr, size_t Size) override { + F1->getOS().write(Ptr, Size); + F2->getOS().write(Ptr, Size); + } + void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override { + this->flush(); + F1->getOS().pwrite(Ptr, Size, Offset); + F2->getOS().pwrite(Ptr, Size, Offset); + } + uint64_t current_pos() const override { return F1->getOS().tell(); } + size_t preferred_buffer_size() const override { + return PreferredBufferSize; + } + void reserveExtraSpace(uint64_t ExtraSize) override { + F1->getOS().reserveExtraSpace(ExtraSize); + F2->getOS().reserveExtraSpace(ExtraSize); + } + bool is_displayed() const override { + return F1->getOS().is_displayed() && F2->getOS().is_displayed(); + } + bool has_colors() const override { + return F1->getOS().has_colors() && F2->getOS().has_colors(); + } + void enable_colors(bool enable) override { + raw_pwrite_stream::enable_colors(enable); + F1->getOS().enable_colors(enable); + F2->getOS().enable_colors(enable); + } + + MirroringOutput(std::unique_ptr F1, + std::unique_ptr F2) + : PreferredBufferSize(std::max(F1->getOS().GetBufferSize(), + F1->getOS().GetBufferSize())), + F1(std::move(F1)), F2(std::move(F2)) { + // Don't double buffer. + this->F1->getOS().SetUnbuffered(); + this->F2->getOS().SetUnbuffered(); + } + size_t PreferredBufferSize; + std::unique_ptr F1; + std::unique_ptr F2; + }; + struct MirroringOutputBackend : public ProxyOutputBackend1, + public ProxyOutputBackend2 { + Expected> + createFileImpl(StringRef Path, + std::optional Config) override { + std::unique_ptr File1; + std::unique_ptr File2; + if (Error E = + ProxyOutputBackend1::createFileImpl(Path, Config).moveInto(File1)) + return std::move(E); + if (Error E = + ProxyOutputBackend2::createFileImpl(Path, Config).moveInto(File2)) + return joinErrors(std::move(E), File1->discard()); + + // Skip the extra indirection if one of these is a null output. + if (isa(*File1)) { + consumeError(File1->discard()); + return std::move(File2); + } + if (isa(*File2)) { + consumeError(File2->discard()); + return std::move(File1); + } + return std::make_unique(std::move(File1), + std::move(File2)); + } + + IntrusiveRefCntPtr cloneImpl() const override { + return IntrusiveRefCntPtr( + makeIntrusiveRefCnt( + ProxyOutputBackend1::getUnderlyingBackend().clone(), + ProxyOutputBackend2::getUnderlyingBackend().clone())); + } + void Retain() const { ProxyOutputBackend1::Retain(); } + void Release() const { ProxyOutputBackend1::Release(); } + + MirroringOutputBackend(IntrusiveRefCntPtr Backend1, + IntrusiveRefCntPtr Backend2) + : ProxyOutputBackend1(std::move(Backend1)), + ProxyOutputBackend2(std::move(Backend2)) {} + }; + + assert(Backend1 && "Expected actual backend"); + assert(Backend2 && "Expected actual backend"); + return IntrusiveRefCntPtr( + makeIntrusiveRefCnt(std::move(Backend1), + std::move(Backend2))); +} + +static OutputConfig +applySettings(std::optional &&Config, + const OnDiskOutputBackend::OutputSettings &Settings) { + if (!Config) + Config = Settings.DefaultConfig; + if (Settings.DisableTemporaries) + Config->setNoAtomicWrite(); + if (Settings.DisableRemoveOnSignal) + Config->setNoDiscardOnSignal(); + return *Config; +} + +namespace { +class OnDiskOutputFile final : public OutputFileImpl { +public: + Error keep() override; + Error discard() override; + raw_pwrite_stream &getOS() override { + assert(FileOS && "Expected valid file"); + if (BufferOS) + return *BufferOS; + return *FileOS; + } + + /// Attempt to open a temporary file for \p OutputPath. + /// + /// This tries to open a uniquely-named temporary file for \p OutputPath, + /// possibly also creating any missing directories if \a + /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a + /// Config. + /// + /// \post FD and \a TempPath are initialized if this is successful. + Error tryToCreateTemporary(std::optional &FD); + + Error initializeFD(std::optional &FD); + Error initializeStream(); + Error reset(); + + OnDiskOutputFile(StringRef OutputPath, std::optional Config, + const OnDiskOutputBackend::OutputSettings &Settings) + : Config(applySettings(std::move(Config), Settings)), + OutputPath(OutputPath.str()) {} + + OutputConfig Config; + const std::string OutputPath; + std::optional TempPath; + std::optional FileOS; + std::optional BufferOS; +}; +} // end namespace + +static Error createDirectoriesOnDemand(StringRef OutputPath, + OutputConfig Config, + llvm::function_ref CreateFile) { + return handleErrors(CreateFile(), [&](std::unique_ptr EC) { + if (EC->convertToErrorCode() != std::errc::no_such_file_or_directory || + Config.getNoImplyCreateDirectories()) + return Error(std::move(EC)); + + StringRef ParentPath = sys::path::parent_path(OutputPath); + if (std::error_code EC = sys::fs::create_directories(ParentPath)) + return make_error(ParentPath, EC); + return CreateFile(); + }); +} + +Error OnDiskOutputFile::tryToCreateTemporary(std::optional &FD) { + // Create a temporary file. + // Insert -%%%%%%%% before the extension (if any), and because some tools + // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build + // artifacts, also append .tmp. + StringRef OutputExtension = sys::path::extension(OutputPath); + SmallString<128> ModelPath = + StringRef(OutputPath).drop_back(OutputExtension.size()); + ModelPath += "-%%%%%%%%"; + ModelPath += OutputExtension; + ModelPath += ".tmp"; + + return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error { + int NewFD; + SmallString<128> UniquePath; + if (std::error_code EC = + sys::fs::createUniqueFile(ModelPath, NewFD, UniquePath)) + return make_error(ModelPath, OutputPath, EC); + + if (Config.getDiscardOnSignal()) + sys::RemoveFileOnSignal(UniquePath); + + TempPath = UniquePath.str().str(); + FD.emplace(NewFD); + return Error::success(); + }); +} + +Error OnDiskOutputFile::initializeFD(std::optional &FD) { + assert(OutputPath != "-" && "Unexpected request for FD of stdout"); + + // Disable temporary file for other non-regular files, and if we get a status + // object, also check if we can write and disable write-through buffers if + // appropriate. + if (Config.getAtomicWrite()) { + sys::fs::file_status Status; + sys::fs::status(OutputPath, Status); + if (sys::fs::exists(Status)) { + if (!sys::fs::is_regular_file(Status)) + Config.setNoAtomicWrite(); + + // Fail now if we can't write to the final destination. + if (!sys::fs::can_write(OutputPath)) + return make_error( + OutputPath, + std::make_error_code(std::errc::operation_not_permitted)); + } + } + + // If (still) using a temporary file, try to create it (and return success if + // that works). + if (Config.getAtomicWrite()) + if (!errorToBool(tryToCreateTemporary(FD))) + return Error::success(); + + // Not using a temporary file. Open the final output file. + return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error { + int NewFD; + sys::fs::OpenFlags OF = sys::fs::OF_None; + if (Config.getTextWithCRLF()) + OF |= sys::fs::OF_TextWithCRLF; + else if (Config.getText()) + OF |= sys::fs::OF_Text; + if (Config.getAppend()) + OF |= sys::fs::OF_Append; + if (std::error_code EC = sys::fs::openFileForWrite( + OutputPath, NewFD, sys::fs::CD_CreateAlways, OF)) + return convertToOutputError(OutputPath, EC); + FD.emplace(NewFD); + + if (Config.getDiscardOnSignal()) + sys::RemoveFileOnSignal(OutputPath); + return Error::success(); + }); +} + +Error OnDiskOutputFile::initializeStream() { + // Open the file stream. + if (OutputPath == "-") { + std::error_code EC; + FileOS.emplace(OutputPath, EC); + if (EC) + return make_error(OutputPath, EC); + } else { + std::optional FD; + if (Error E = initializeFD(FD)) + return E; + FileOS.emplace(*FD, /*shouldClose=*/true); + } + + // Buffer the stream if necessary. + if (!FileOS->supportsSeeking() && !Config.getText()) + BufferOS.emplace(*FileOS); + + return Error::success(); +} + +namespace { +class OpenFileRAII { + static const int InvalidFd = -1; + +public: + int Fd = InvalidFd; + + ~OpenFileRAII() { + if (Fd != InvalidFd) + llvm::sys::Process::SafelyCloseFileDescriptor(Fd); + } +}; + +enum class FileDifference : uint8_t { + /// The source and destination paths refer to the exact same file. + IdenticalFile, + /// The source and destination paths refer to separate files with identical + /// contents. + SameContents, + /// The source and destination paths refer to separate files with different + /// contents. + DifferentContents +}; +} // end anonymous namespace + +static Expected +areFilesDifferent(const llvm::Twine &Source, const llvm::Twine &Destination) { + if (sys::fs::equivalent(Source, Destination)) + return FileDifference::IdenticalFile; + + OpenFileRAII SourceFile; + sys::fs::file_status SourceStatus; + // If we can't open the source file, fail. + if (std::error_code EC = sys::fs::openFileForRead(Source, SourceFile.Fd)) + return convertToOutputError(Source, EC); + + // If we can't stat the source file, fail. + if (std::error_code EC = sys::fs::status(SourceFile.Fd, SourceStatus)) + return convertToOutputError(Source, EC); + + OpenFileRAII DestFile; + sys::fs::file_status DestStatus; + // If we can't open the destination file, report different. + if (std::error_code Error = + sys::fs::openFileForRead(Destination, DestFile.Fd)) + return FileDifference::DifferentContents; + + // If we can't open the destination file, report different. + if (std::error_code Error = sys::fs::status(DestFile.Fd, DestStatus)) + return FileDifference::DifferentContents; + + // If the files are different sizes, they must be different. + uint64_t Size = SourceStatus.getSize(); + if (Size != DestStatus.getSize()) + return FileDifference::DifferentContents; + + // If both files are zero size, they must be the same. + if (Size == 0) + return FileDifference::SameContents; + + // The two files match in size, so we have to compare the bytes to determine + // if they're the same. + std::error_code SourceRegionErr; + sys::fs::mapped_file_region SourceRegion( + sys::fs::convertFDToNativeFile(SourceFile.Fd), + sys::fs::mapped_file_region::readonly, Size, 0, SourceRegionErr); + if (SourceRegionErr) + return convertToOutputError(Source, SourceRegionErr); + + std::error_code DestRegionErr; + sys::fs::mapped_file_region DestRegion( + sys::fs::convertFDToNativeFile(DestFile.Fd), + sys::fs::mapped_file_region::readonly, Size, 0, DestRegionErr); + + if (DestRegionErr) + return FileDifference::DifferentContents; + + if (memcmp(SourceRegion.const_data(), DestRegion.const_data(), Size) != 0) + return FileDifference::DifferentContents; + + return FileDifference::SameContents; +} + +Error OnDiskOutputFile::reset() { + // Destroy the streams to flush them. + BufferOS.reset(); + if (!FileOS) + return Error::success(); + + // Remember the error in raw_fd_ostream to be reported later. + std::error_code EC = FileOS->error(); + // Clear the error to avoid fatal error when reset. + FileOS->clear_error(); + FileOS.reset(); + return errorCodeToError(EC); +} + +Error OnDiskOutputFile::keep() { + if (auto E = reset()) + return E; + + // Close the file descriptor and remove crash cleanup before exit. + auto RemoveDiscardOnSignal = make_scope_exit([&]() { + if (Config.getDiscardOnSignal()) + sys::DontRemoveFileOnSignal(TempPath ? *TempPath : OutputPath); + }); + + if (!TempPath) + return Error::success(); + + // See if we should append instead of move. + if (Config.getAppend() && OutputPath != "-") { + // Read TempFile for the content to append. + auto Content = MemoryBuffer::getFile(*TempPath); + if (!Content) + return convertToTempFileOutputError(*TempPath, OutputPath, + Content.getError()); + while (1) { + // Attempt to lock the output file. + // Only one process is allowed to append to this file at a time. + llvm::LockFileManager Locked(OutputPath); + switch (Locked) { + case llvm::LockFileManager::LFS_Error: { + // If we error acquiring a lock, we cannot ensure appends + // to the trace file are atomic - cannot ensure output correctness. + Locked.unsafeRemoveLockFile(); + return convertToOutputError( + OutputPath, std::make_error_code(std::errc::no_lock_available)); + } + case llvm::LockFileManager::LFS_Owned: { + // Lock acquired, perform the write and release the lock. + std::error_code EC; + llvm::raw_fd_ostream Out(OutputPath, EC, llvm::sys::fs::OF_Append); + if (EC) + return convertToOutputError(OutputPath, EC); + Out << (*Content)->getBuffer(); + Out.close(); + Locked.unsafeRemoveLockFile(); + if (Out.has_error()) + return convertToOutputError(OutputPath, Out.error()); + // Remove temp file and done. + (void)sys::fs::remove(*TempPath); + return Error::success(); + } + case llvm::LockFileManager::LFS_Shared: { + // Someone else owns the lock on this file, wait. + switch (Locked.waitForUnlock(256)) { + case llvm::LockFileManager::Res_Success: + LLVM_FALLTHROUGH; + case llvm::LockFileManager::Res_OwnerDied: { + continue; // try again to get the lock. + } + case llvm::LockFileManager::Res_Timeout: { + // We could error on timeout to avoid potentially hanging forever, but + // it may be more likely that an interrupted process failed to clear + // the lock, causing other waiting processes to time-out. Let's clear + // the lock and try again right away. If we do start seeing compiler + // hangs in this location, we will need to re-consider. + Locked.unsafeRemoveLockFile(); + continue; + } + } + break; + } + } + } + } + + if (Config.getOnlyIfDifferent()) { + auto Result = areFilesDifferent(*TempPath, OutputPath); + if (!Result) + return Result.takeError(); + switch (*Result) { + case FileDifference::IdenticalFile: + // Do nothing for a self-move. + return Error::success(); + + case FileDifference::SameContents: + // Files are identical; remove the source file. + (void)sys::fs::remove(*TempPath); + return Error::success(); + + case FileDifference::DifferentContents: + break; // Rename the file. + } + } + + // Move temporary to the final output path and remove it if that fails. + std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath); + if (!RenameEC) + return Error::success(); + + // FIXME: TempPath should be in the same directory as OutputPath but try to + // copy the output to see if makes any difference. If this path is used, + // investigate why we need to copy. + RenameEC = sys::fs::copy_file(*TempPath, OutputPath); + (void)sys::fs::remove(*TempPath); + + if (!RenameEC) + return Error::success(); + + return make_error(*TempPath, OutputPath, RenameEC); +} + +Error OnDiskOutputFile::discard() { + // Destroy the streams to flush them. + if (auto E = reset()) + return E; + + // Nothing on the filesystem to remove for stdout. + if (OutputPath == "-") + return Error::success(); + + auto discardPath = [&](StringRef Path) { + std::error_code EC = sys::fs::remove(Path); + sys::DontRemoveFileOnSignal(Path); + return EC; + }; + + // Clean up the file that's in-progress. + if (!TempPath) + return convertToOutputError(OutputPath, discardPath(OutputPath)); + return convertToTempFileOutputError(*TempPath, OutputPath, + discardPath(*TempPath)); +} + +Error OnDiskOutputBackend::makeAbsolute(SmallVectorImpl &Path) const { + return convertToOutputError(StringRef(Path.data(), Path.size()), + sys::fs::make_absolute(Path)); +} + +Expected> +OnDiskOutputBackend::createFileImpl(StringRef Path, + std::optional Config) { + SmallString<256> AbsPath; + if (Path != "-") { + AbsPath = Path; + if (Error E = makeAbsolute(AbsPath)) + return std::move(E); + Path = AbsPath; + } + + auto File = std::make_unique(Path, Config, Settings); + if (Error E = File->initializeStream()) + return std::move(E); + + return std::move(File); +} diff --git a/llvm/lib/Support/VirtualOutputConfig.cpp b/llvm/lib/Support/VirtualOutputConfig.cpp new file mode 100644 index 0000000000000..f1d3c0fe3c6e7 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputConfig.cpp @@ -0,0 +1,50 @@ +//===- VirtualOutputConfig.cpp - Virtual output configuration -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::vfs; + +OutputConfig &OutputConfig::setOpenFlags(const sys::fs::OpenFlags &Flags) { + // Ignore CRLF on its own as invalid. + using namespace llvm::sys::fs; + return Flags & OF_Text + ? setText().setCRLF(Flags & OF_CRLF).setAppend(Flags & OF_Append) + : setBinary().setAppend(Flags & OF_Append); +} + +void OutputConfig::print(raw_ostream &OS) const { + OS << "{"; + bool IsFirst = true; + auto printFlag = [&](StringRef FlagName, bool Value) { + if (IsFirst) + IsFirst = false; + else + OS << ","; + if (!Value) + OS << "No"; + OS << FlagName; + }; + +#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \ + if (get##NAME() != DEFAULT) \ + printFlag(#NAME, get##NAME()); +#include "llvm/Support/VirtualOutputConfig.def" + OS << "}"; +} + +LLVM_DUMP_METHOD void OutputConfig::dump() const { print(dbgs()); } + +raw_ostream &llvm::operator<<(raw_ostream &OS, OutputConfig Config) { + Config.print(OS); + return OS; +} diff --git a/llvm/lib/Support/VirtualOutputError.cpp b/llvm/lib/Support/VirtualOutputError.cpp new file mode 100644 index 0000000000000..74fa5e3fa0568 --- /dev/null +++ b/llvm/lib/Support/VirtualOutputError.cpp @@ -0,0 +1,53 @@ +//===- VirtualOutputError.cpp - Errors for output virtualization ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputError.h" + +using namespace llvm; +using namespace llvm::vfs; + +void OutputError::anchor() {} +void OutputConfigError::anchor() {} +void TempFileOutputError::anchor() {} + +char OutputError::ID = 0; +char OutputConfigError::ID = 0; +char TempFileOutputError::ID = 0; + +namespace { +class OutputErrorCategory : public std::error_category { +public: + const char *name() const noexcept override; + std::string message(int EV) const override; +}; +} // end namespace + +const std::error_category &vfs::output_category() { + static OutputErrorCategory ErrorCategory; + return ErrorCategory; +} + +const char *OutputErrorCategory::name() const noexcept { + return "llvm.vfs.output"; +} + +std::string OutputErrorCategory::message(int EV) const { + OutputErrorCode E = static_cast(EV); + switch (E) { + case OutputErrorCode::invalid_config: + return "invalid config"; + case OutputErrorCode::not_closed: + return "output not closed"; + case OutputErrorCode::already_closed: + return "output already closed"; + case OutputErrorCode::has_open_proxy: + return "output has open proxy"; + } + llvm_unreachable( + "An enumerator of OutputErrorCode does not have a message defined."); +} diff --git a/llvm/lib/Support/VirtualOutputFile.cpp b/llvm/lib/Support/VirtualOutputFile.cpp new file mode 100644 index 0000000000000..c043c0b455c5c --- /dev/null +++ b/llvm/lib/Support/VirtualOutputFile.cpp @@ -0,0 +1,106 @@ +//===- VirtualOutputFile.cpp - Output file virtualization -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputFile.h" +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/Support/VirtualOutputError.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/raw_ostream_proxy.h" + +using namespace llvm; +using namespace llvm::vfs; + +char OutputFileImpl::ID = 0; +char NullOutputFileImpl::ID = 0; + +void OutputFileImpl::anchor() {} +void NullOutputFileImpl::anchor() {} + +class OutputFile::TrackedProxy : public raw_pwrite_stream_proxy { +public: + void resetProxy() { + TrackingPointer = nullptr; + resetProxiedOS(); + } + + explicit TrackedProxy(TrackedProxy *&TrackingPointer, raw_pwrite_stream &OS) + : raw_pwrite_stream_proxy(OS), TrackingPointer(TrackingPointer) { + assert(!TrackingPointer && "Expected to add a proxy"); + TrackingPointer = this; + } + + ~TrackedProxy() override { resetProxy(); } + + TrackedProxy *&TrackingPointer; +}; + +Expected> OutputFile::createProxy() { + if (OpenProxy) + return make_error(getPath(), OutputErrorCode::has_open_proxy); + + return std::make_unique(OpenProxy, getOS()); +} + +Error OutputFile::keep() { + // Catch double-closing logic bugs. + if (LLVM_UNLIKELY(!Impl)) + report_fatal_error( + make_error(getPath(), OutputErrorCode::already_closed)); + + // Report a fatal error if there's an open proxy and the file is being kept. + // This is safer than relying on clients to remember to flush(). Also call + // OutputFile::discard() to give the backend a chance to clean up any + // side effects (such as temporaries). + if (LLVM_UNLIKELY(OpenProxy)) + report_fatal_error(joinErrors( + make_error(getPath(), OutputErrorCode::has_open_proxy), + discard())); + + Error E = Impl->keep(); + Impl = nullptr; + DiscardOnDestroyHandler = nullptr; + return E; +} + +Error OutputFile::discard() { + // Catch double-closing logic bugs. + if (LLVM_UNLIKELY(!Impl)) + report_fatal_error( + make_error(getPath(), OutputErrorCode::already_closed)); + + // Be lenient about open proxies since client teardown paths won't + // necessarily clean up in the right order. Reset the proxy to flush any + // current content; if there is another write, there should be quick crash on + // null dereference. + if (OpenProxy) + OpenProxy->resetProxy(); + + Error E = Impl->discard(); + Impl = nullptr; + DiscardOnDestroyHandler = nullptr; + return E; +} + +void OutputFile::destroy() { + if (!Impl) + return; + + // Clean up the file. Move the discard handler into a local since discard + // will reset it. + auto DiscardHandler = std::move(DiscardOnDestroyHandler); + Error E = discard(); + assert(!Impl && "Expected discard to destroy Impl"); + + // If there's no handler, report a fatal error. + if (LLVM_UNLIKELY(!DiscardHandler)) + llvm::report_fatal_error(joinErrors( + make_error(getPath(), OutputErrorCode::not_closed), + std::move(E))); + else if (E) + DiscardHandler(std::move(E)); +} diff --git a/llvm/lib/Support/raw_ostream_proxy.cpp b/llvm/lib/Support/raw_ostream_proxy.cpp new file mode 100644 index 0000000000000..2bbaa82f4afa7 --- /dev/null +++ b/llvm/lib/Support/raw_ostream_proxy.cpp @@ -0,0 +1,15 @@ +//===- raw_ostream_proxy.cpp - Implement the raw_ostream proxies ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/raw_ostream_proxy.h" + +using namespace llvm; + +void raw_ostream_proxy::anchor() {} + +void raw_pwrite_stream_proxy::anchor() {} diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt index dfd55b228900d..4299dbe4ac51c 100644 --- a/llvm/unittests/Support/CMakeLists.txt +++ b/llvm/unittests/Support/CMakeLists.txt @@ -93,6 +93,10 @@ add_llvm_unittest(SupportTests UnicodeTest.cpp VersionTupleTest.cpp VirtualFileSystemTest.cpp + VirtualOutputBackendTest.cpp + VirtualOutputBackendsTest.cpp + VirtualOutputConfigTest.cpp + VirtualOutputFileTest.cpp WithColorTest.cpp YAMLIOTest.cpp YAMLParserTest.cpp @@ -100,6 +104,7 @@ add_llvm_unittest(SupportTests formatted_raw_ostream_test.cpp raw_fd_stream_test.cpp raw_ostream_test.cpp + raw_ostream_proxy_test.cpp raw_pwrite_stream_test.cpp raw_sha1_ostream_test.cpp xxhashTest.cpp diff --git a/llvm/unittests/Support/VirtualOutputBackendTest.cpp b/llvm/unittests/Support/VirtualOutputBackendTest.cpp new file mode 100644 index 0000000000000..3adb671d97774 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputBackendTest.cpp @@ -0,0 +1,147 @@ +//===- VirtualOutputBackendTest.cpp - Tests for vfs::OutputBackend --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +struct MockOutputBackendData { + int Cloned = 0; + int FilesCreated = 0; + std::optional LastConfig; + unique_function FileCreator; +}; + +struct MockOutputBackend final : public OutputBackend { + struct MockFile final : public OutputFileImpl { + Error keep() override { return Error::success(); } + Error discard() override { return Error::success(); } + raw_pwrite_stream &getOS() override { return OS; } + raw_null_ostream OS; + }; + + IntrusiveRefCntPtr cloneImpl() const override { + ++Data.Cloned; + return const_cast(this); + } + + Expected> + createFileImpl(StringRef, std::optional Config) override { + ++Data.FilesCreated; + Data.LastConfig = Config; + if (Data.FileCreator) + return Data.FileCreator(); + return std::make_unique(); + } + + Expected + createAutoDiscardFile(const Twine &OutputPath, + std::optional Config = std::nullopt) { + return consumeDiscardOnDestroy(createFile(OutputPath, Config)); + } + + MockOutputBackend(MockOutputBackendData &Data) : Data(Data) {} + MockOutputBackendData &Data; +}; + +static IntrusiveRefCntPtr +createMockBackend(MockOutputBackendData &Data) { + return makeIntrusiveRefCnt(Data); +} + +static Error createCustomError() { + return createStringError(inconvertibleErrorCode(), "custom error"); +} + +TEST(VirtualOutputBackendTest, construct) { + MockOutputBackendData Data; + auto B = createMockBackend(Data); + EXPECT_EQ(0, Data.Cloned); + EXPECT_EQ(0, Data.FilesCreated); +} + +TEST(VirtualOutputBackendTest, clone) { + MockOutputBackendData Data; + auto Backend = createMockBackend(Data); + auto Clone = Backend->clone(); + EXPECT_EQ(1, Data.Cloned); + + // Confirm the clone matches what the mock's cloneImpl() does. + EXPECT_EQ(Backend.get(), Clone.get()); + + // Make another clone. + Backend->clone(); + EXPECT_EQ(2, Data.Cloned); +} + +TEST(VirtualOutputBackendTest, createFile) { + MockOutputBackendData Data; + auto Backend = createMockBackend(Data); + + StringRef FilePath = "dir/file"; + OutputFile F; + EXPECT_THAT_ERROR(Backend->createFile(Twine(FilePath)).moveInto(F), + Succeeded()); + EXPECT_EQ(1, Data.FilesCreated); + EXPECT_EQ(FilePath, F.getPath()); + EXPECT_EQ(std::nullopt, Data.LastConfig); + + // Confirm OutputBackend has not installed a discard handler. +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(F = OutputFile(), "output not closed"); +#endif + consumeError(F.discard()); + + // Create more files and specify configs. + for (OutputConfig Config : { + OutputConfig(), + OutputConfig().setNoAtomicWrite().setDiscardOnSignal(), + OutputConfig().setAtomicWrite().setNoDiscardOnSignal(), + OutputConfig().setText(), + OutputConfig().setTextWithCRLF(), + }) { + int CreatedAlready = Data.FilesCreated; + EXPECT_THAT_ERROR( + Backend->createAutoDiscardFile(Twine(FilePath), Config).takeError(), + Succeeded()); + EXPECT_EQ(Config, Data.LastConfig); + EXPECT_EQ(1 + CreatedAlready, Data.FilesCreated); + } +} + +TEST(VirtualOutputBackendTest, createFileInvalidConfigCRLF) { + MockOutputBackendData Data; + auto Backend = createMockBackend(Data); + + // Check that invalid configs don't make it to the backend. + EXPECT_THAT_ERROR( + Backend + ->createAutoDiscardFile(Twine("dir/file"), OutputConfig().setCRLF()) + .takeError(), + FailedWithMessage("dir/file: invalid config: {CRLF}")); + EXPECT_EQ(0, Data.FilesCreated); +} + +TEST(VirtualOutputBackendTest, createFileError) { + MockOutputBackendData Data; + Data.FileCreator = createCustomError; + auto Backend = createMockBackend(Data); + + // Check that invalid configs don't make it to the backend. + EXPECT_THAT_ERROR( + Backend->createAutoDiscardFile(Twine("dir/file")).takeError(), + FailedWithMessage("custom error")); + EXPECT_EQ(1, Data.FilesCreated); +} + +} // end namespace diff --git a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp new file mode 100644 index 0000000000000..630402ed354a4 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp @@ -0,0 +1,886 @@ +//===- VirtualOutputBackendsTest.cpp - Tests for vfs::OutputBackend impls -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/HashingOutputBackend.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +class OutputBackendProvider { +public: + virtual bool rejectsMissingDirectories() = 0; + + virtual IntrusiveRefCntPtr createBackend() = 0; + virtual std::string getFilePathToCreate() = 0; + virtual std::string getFilePathToCreateUnder(StringRef Parent1, + StringRef Parent2 = "") = 0; + virtual Error checkCreated(StringRef FilePath, + OutputConfig Config = OutputConfig()) = 0; + virtual Error checkWrote(StringRef FilePath, StringRef Data) = 0; + virtual Error checkFlushed(StringRef FilePath, StringRef Data) = 0; + virtual Error checkKept(StringRef FilePath, StringRef Data) = 0; + virtual Error checkDiscarded(StringRef FilePath) = 0; + + virtual ~OutputBackendProvider() = default; + + struct Generator { + std::string Name; + std::function()> Generate; + + std::unique_ptr operator()() const { + return Generate(); + } + }; +}; + +struct BackendTest + : public ::testing::TestWithParam { + std::unique_ptr Provider; + + void SetUp() override { Provider = GetParam()(); } + void TearDown() override { Provider = nullptr; } + + IntrusiveRefCntPtr createBackend() { + return Provider->createBackend(); + } +}; + +TEST_P(BackendTest, Discard) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.discard(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, DiscardNoAtomicWrite) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setNoAtomicWrite(); + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O), + Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.discard(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, Keep) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + ASSERT_TRUE(O.isOpen()); + + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, KeepFlush) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + O.getOS().flush(); + EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepFlushProxy) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + { + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(O.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + Proxy->flush(); + EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded()); + } + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepEmpty) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, ""), Succeeded()); +} + +TEST_P(BackendTest, KeepMissingDirectory) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreateUnder("missing"); + StringRef Data = "some data"; + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepMissingDirectoryNested) { + auto Backend = createBackend(); + std::string FilePath = + Provider->getFilePathToCreateUnder("missing", "nested"); + StringRef Data = "some data"; + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); +} + +TEST_P(BackendTest, KeepNoAtomicWrite) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setNoAtomicWrite(); + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O), + Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded()); + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, KeepNoAtomicWriteMissingDirectory) { + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setNoAtomicWrite(); + + OutputFile O; + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O), + Succeeded()); + consumeDiscardOnDestroy(O); + ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded()); + + O << Data; + EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded()); + + EXPECT_THAT_ERROR(O.keep(), Succeeded()); + EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded()); + EXPECT_FALSE(O.isOpen()); +} + +TEST_P(BackendTest, KeepMissingDirectoryNoImply) { + // Skip this test if the backend doesn't have a concept of missing + // directories. + if (!Provider->rejectsMissingDirectories()) + return; + + auto Backend = createBackend(); + std::string FilePath = Provider->getFilePathToCreateUnder("missing"); + std::error_code EC = errorToErrorCode( + consumeDiscardOnDestroy( + Backend->createFile(FilePath, + OutputConfig().setNoImplyCreateDirectories())) + .takeError()); + EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value()); +} + +class NullOutputBackendProvider : public OutputBackendProvider { +public: + bool rejectsMissingDirectories() override { return false; } + + IntrusiveRefCntPtr createBackend() override { + return makeNullOutputBackend(); + } + std::string getFilePathToCreate() override { return "ignored.data"; } + std::string getFilePathToCreateUnder(StringRef Parent1, + StringRef Parent2) override { + SmallString<128> Path; + sys::path::append(Path, Parent1, Parent2, getFilePathToCreate()); + return Path.str().str(); + } + Error checkCreated(StringRef, OutputConfig) override { + return Error::success(); + } + Error checkWrote(StringRef, StringRef) override { return Error::success(); } + Error checkFlushed(StringRef, StringRef) override { return Error::success(); } + Error checkKept(StringRef, StringRef) override { return Error::success(); } + Error checkDiscarded(StringRef) override { return Error::success(); } +}; + +struct OnDiskFile { + const unittest::TempDir &D; + SmallString<128> Path; + StringRef ParentPath; + StringRef Filename; + StringRef Stem; + StringRef Extension; + std::unique_ptr LastBuffer; + + OnDiskFile(const unittest::TempDir &D, const Twine &InputPath) : D(D) { + if (sys::path::is_absolute(InputPath)) + InputPath.toVector(Path); + else + sys::path::append(Path, D.path(), InputPath); + ParentPath = sys::path::parent_path(Path); + Filename = sys::path::filename(Path); + Stem = sys::path::stem(Filename); + Extension = sys::path::extension(Filename); + } + + std::optional findTemp() const; + + std::optional getCurrentUniqueID(); + + bool hasUniqueID(sys::fs::UniqueID ID) { + auto CurrentID = getCurrentUniqueID(); + if (!CurrentID) + return false; + return *CurrentID == ID; + } + + std::optional getCurrentContent() { + auto OnDiskOrErr = MemoryBuffer::getFile(Path); + if (!OnDiskOrErr) + return std::nullopt; + LastBuffer = std::move(*OnDiskOrErr); + return LastBuffer->getBuffer(); + } + + bool equalsCurrentContent(StringRef Data) { + auto CurrentContent = getCurrentContent(); + if (!CurrentContent) + return false; + return *CurrentContent == Data; + } + + bool equalsCurrentContent(std::nullopt_t) { + return getCurrentContent() == std::nullopt; + } +}; + +class OnDiskOutputBackendProvider : public OutputBackendProvider { +public: + bool rejectsMissingDirectories() override { return true; } + + std::optional D; + + IntrusiveRefCntPtr createBackend() override { + auto Backend = makeIntrusiveRefCnt(); + Backend->Settings = Settings; + return Backend; + } + void init() { + if (!D) + D.emplace("OutputBackendTest.d", /*Unique=*/true); + } + std::string getFilePathToCreate() override { + init(); + return OnDiskFile(*D, "file.data").Path.str().str(); + } + std::string getFilePathToCreateUnder(StringRef Parent1, + StringRef Parent2) override { + init(); + SmallString<128> Path; + sys::path::append(Path, D->path(), Parent1, Parent2, getFilePathToCreate()); + return Path.str().str(); + } + + Error checkCreated(StringRef FilePath, OutputConfig Config) override; + Error checkWrote(StringRef FilePath, StringRef Data) override; + Error checkFlushed(StringRef FilePath, StringRef Data) override; + Error checkKept(StringRef FilePath, StringRef Data) override; + Error checkDiscarded(StringRef FilePath) override; + + struct FileInfo { + OutputConfig Config; + std::optional F; + std::optional Temp; + std::optional UID; + std::optional TempUID; + }; + Error checkOpen(FileInfo &Info); + bool shouldUseTemporaries(const FileInfo &Info) const; + + OnDiskOutputBackendProvider() = default; + explicit OnDiskOutputBackendProvider( + const OnDiskOutputBackend::OutputSettings &Settings) + : Settings(Settings) {} + OnDiskOutputBackend::OutputSettings Settings; + + StringMap Files; + Error lookupFileInfo(StringRef FilePath, FileInfo *&Info); +}; + +bool OnDiskOutputBackendProvider::shouldUseTemporaries( + const FileInfo &Info) const { + return Info.Config.getAtomicWrite() && !Settings.DisableTemporaries; +} + +struct ProviderGeneratorList { + std::vector Generators; + ProviderGeneratorList( + std::initializer_list IL) + : Generators(IL) {} + + std::string operator()( + const ::testing::TestParamInfo &Info) { + return Info.param.Name; + } +}; + +ProviderGeneratorList BackendGenerators = { + {"Null", []() { return std::make_unique(); }}, + {"OnDisk", + []() { return std::make_unique(); }}, + {"OnDisk_DisableRemoveOnSignal", + []() { + OnDiskOutputBackend::OutputSettings Settings; + Settings.DisableRemoveOnSignal = true; + return std::make_unique(Settings); + }}, + {"OnDisk_DisableTemporaries", + []() { + OnDiskOutputBackend::OutputSettings Settings; + Settings.DisableTemporaries = true; + return std::make_unique(Settings); + }}, +}; + +INSTANTIATE_TEST_SUITE_P(VirtualOutput, BackendTest, + ::testing::ValuesIn(BackendGenerators.Generators), + BackendGenerators); + +std::optional OnDiskFile::getCurrentUniqueID() { + sys::fs::file_status Status; + sys::fs::status(Path, Status, /*follow=*/false); + if (!sys::fs::is_regular_file(Status)) + return std::nullopt; + return Status.getUniqueID(); +} + +std::optional OnDiskFile::findTemp() const { + std::error_code EC; + for (sys::fs::directory_iterator I(ParentPath, EC), E; !EC && I != E; + I.increment(EC)) { + StringRef TempPath = I->path(); + if (!TempPath.startswith(D.path())) + continue; + + // Look for "-*..tmp". + if (sys::path::extension(TempPath) != ".tmp") + continue; + + // Drop the ".tmp" and check the extension and stem. + StringRef TempStem = sys::path::stem(TempPath); + if (sys::path::extension(TempStem) != Extension) + continue; + StringRef OriginalStem = sys::path::stem(TempStem); + if (!OriginalStem.startswith(Stem)) + continue; + if (!OriginalStem.drop_front(Stem.size()).startswith("-")) + continue; + + // Found it. + return OnDiskFile(D, TempPath.drop_front(D.path().size() + 1)); + } + return std::nullopt; +} + +Error OnDiskOutputBackendProvider::lookupFileInfo(StringRef FilePath, + FileInfo *&Info) { + auto I = Files.find(FilePath); + if (Files.find(FilePath) == Files.end()) + return createStringError(inconvertibleErrorCode(), + "Missing call to checkCreated()"); + Info = &I->second; + assert(Info->F && "Expected OnDiskFile to be initialized"); + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkOpen(FileInfo &Info) { + // Collect info about filesystem state. + assert(Info.F); + std::optional UID = Info.F->getCurrentUniqueID(); + std::optional Temp = Info.F->findTemp(); + std::optional TempUID; + if (Temp) + TempUID = Temp->getCurrentUniqueID(); + + // Check if it's correct. + if (shouldUseTemporaries(Info)) { + if (!Temp) + return createStringError(inconvertibleErrorCode(), + "Missing temporary file"); + if (!TempUID) + return createStringError(inconvertibleErrorCode(), + "Missing UID for temporary"); + if (UID) + return createStringError( + inconvertibleErrorCode(), + "Unexpected final UID when temporaries should be used"); + + // Check previous data. + if (Info.Temp) + if (Temp->Path != Info.Temp->Path) + return createStringError(inconvertibleErrorCode(), + "Temporary path changed"); + if (Info.TempUID) + if (*TempUID != *Info.TempUID) + return createStringError(inconvertibleErrorCode(), + "Temporary UID changed"); + } else { + if (Temp) + return createStringError(inconvertibleErrorCode(), + "Unexpected temporary file"); + if (!UID) + return createStringError(inconvertibleErrorCode(), + "Missing UID for temporary"); + + // Check previous data. + if (Info.UID) + if (*UID != *Info.UID) + return createStringError(inconvertibleErrorCode(), "UID changed"); + } + + Info.UID = UID; + if (Temp) + Info.Temp.emplace(*D, Temp->Path); + else + Info.Temp.reset(); + Info.TempUID = TempUID; + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkCreated(StringRef FilePath, + OutputConfig Config) { + auto &Info = Files[FilePath]; + if (Info.F) { + assert(OnDiskFile(*D, FilePath).Path == Info.F->Path); + Info.UID = std::nullopt; + Info.Temp.reset(); + Info.TempUID = std::nullopt; + } else { + Info.F.emplace(*D, FilePath); + } + Info.Config = Config; + return checkOpen(Info); +} + +Error OnDiskOutputBackendProvider::checkWrote(StringRef FilePath, + StringRef Data) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + return checkOpen(*Info); +} + +Error OnDiskOutputBackendProvider::checkFlushed(StringRef FilePath, + StringRef Data) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + if (Error E = checkOpen(*Info)) + return E; + + OnDiskFile &F = shouldUseTemporaries(*Info) ? *Info->Temp : *Info->F; + if (!F.equalsCurrentContent(Data)) + return createStringError(inconvertibleErrorCode(), "content not flushed"); + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkKept(StringRef FilePath, + StringRef Data) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + + sys::fs::UniqueID UID = + shouldUseTemporaries(*Info) ? *Info->TempUID : *Info->UID; + if (!Info->F->hasUniqueID(UID)) + return createStringError(inconvertibleErrorCode(), + "File not created by keep or changed UID"); + + if (std::optional Temp = Info->F->findTemp()) + return createStringError(inconvertibleErrorCode(), + "Temporary not removed by keep"); + + return Error::success(); +} + +Error OnDiskOutputBackendProvider::checkDiscarded(StringRef FilePath) { + FileInfo *Info = nullptr; + if (Error E = lookupFileInfo(FilePath, Info)) + return E; + + if (std::optional UID = Info->F->getCurrentUniqueID()) + return createStringError(inconvertibleErrorCode(), + "File not removed by discard"); + + if (std::optional Temp = Info->F->findTemp()) + return createStringError(inconvertibleErrorCode(), + "Temporary not removed by discard"); + + return Error::success(); +} + +TEST(VirtualOutputBackendAdaptors, makeFilteringOutputBackend) { + bool ShouldCreate = false; + auto Backend = makeFilteringOutputBackend( + makeIntrusiveRefCnt(), + [&ShouldCreate](StringRef, std::optional) { + return ShouldCreate; + }); + + int Count = 0; + unittest::TempDir D("FilteringOutputBackendTest.d", /*Unique=*/true); + for (bool ShouldCreateVal : {false, true, true, false}) { + ShouldCreate = ShouldCreateVal; + OnDiskFile OnDisk(D, "file." + Twine(Count++) + "." + Twine(ShouldCreate)); + OutputFile Output; + ASSERT_THAT_ERROR(consumeDiscardOnDestroy(Backend->createFile(OnDisk.Path)) + .moveInto(Output), + Succeeded()); + EXPECT_NE(ShouldCreate, Output.isNull()); + Output << "content"; + EXPECT_THAT_ERROR(Output.keep(), Succeeded()); + + if (ShouldCreate) { + EXPECT_EQ(StringRef("content"), OnDisk.getCurrentContent()); + } else { + EXPECT_FALSE(OnDisk.getCurrentUniqueID()); + } + } + SmallString<128> Path; +} + +class AbsolutePathBackend : public ProxyOutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + llvm_unreachable("unimplemented"); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + assert(!sys::path::is_absolute(Path) && + "Expected tests to pass all relative paths"); + SmallString<256> AbsPath; + sys::path::append(AbsPath, CWD, Path); + return ProxyOutputBackend::createFileImpl(AbsPath, Config); + } + +public: + AbsolutePathBackend(const Twine &CWD, + IntrusiveRefCntPtr Backend) + : ProxyOutputBackend(std::move(Backend)), CWD(CWD.str()) { + assert(sys::path::is_absolute(this->CWD) && + "Expected tests to pass a relative path"); + } + +private: + std::string CWD; +}; + +TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackend) { + unittest::TempDir D1("MirroringOutputBackendTest.1.d", /*Unique=*/true); + unittest::TempDir D2("MirroringOutputBackendTest.2.d", /*Unique=*/true); + + IntrusiveRefCntPtr Backend; + { + auto OnDisk = makeIntrusiveRefCnt(); + Backend = makeMirroringOutputBackend( + makeIntrusiveRefCnt(D1.path(), OnDisk), + makeIntrusiveRefCnt(D2.path(), OnDisk)); + } + + OnDiskFile OnDisk1(D1, "file"); + OnDiskFile OnDisk2(D2, "file"); + OutputFile Output; + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Backend->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(OnDisk1.findTemp()); + EXPECT_TRUE(OnDisk2.findTemp()); + + Output << "content"; + Output.getOS().pwrite("ON", /*Size=*/2, /*Offset=*/1); + EXPECT_THAT_ERROR(Output.keep(), Succeeded()); + EXPECT_EQ(StringRef("cONtent"), OnDisk1.getCurrentContent()); + EXPECT_EQ(StringRef("cONtent"), OnDisk2.getCurrentContent()); + EXPECT_NE(OnDisk1.getCurrentUniqueID(), OnDisk2.getCurrentUniqueID()); +} + +/// Behaves like NullOutputFileImpl, but doesn't match the RTTI (so OutputFile +/// cannot tell). +class LikeNullOutputFile final : public OutputFileImpl { + Error keep() final { return Error::success(); } + Error discard() final { return Error::success(); } + raw_pwrite_stream &getOS() final { return OS; } + +public: + LikeNullOutputFile(raw_null_ostream &OS) : OS(OS) {} + raw_null_ostream &OS; +}; +class LikeNullOutputBackend final : public OutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + llvm_unreachable("not implemented"); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + return std::make_unique(OS); + } + +public: + raw_null_ostream OS; +}; + +TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendNull) { + // Check that null outputs are skipped by seeing that LikeNull->OS is passed + // through directly (without a mirroring proxy stream) to Output. + auto LikeNull = makeIntrusiveRefCnt(); + auto Null1 = makeNullOutputBackend(); + auto Mirror = makeMirroringOutputBackend(Null1, LikeNull); + OutputFile Output; + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(!Output.isNull()); + EXPECT_EQ(&Output.getOS(), &LikeNull->OS); + + // Check the other direction. + Mirror = makeMirroringOutputBackend(LikeNull, Null1); + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(!Output.isNull()); + EXPECT_EQ(&Output.getOS(), &LikeNull->OS); + + // Same null backend, twice. + Mirror = makeMirroringOutputBackend(Null1, Null1); + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(Output.isNull()); + + // Two null backends. + auto Null2 = makeNullOutputBackend(); + Mirror = makeMirroringOutputBackend(Null1, Null2); + ASSERT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output), + Succeeded()); + EXPECT_TRUE(Output.isNull()); +} + +class StringErrorBackend final : public OutputBackend { + IntrusiveRefCntPtr cloneImpl() const override { + llvm_unreachable("not implemented"); + } + + Expected> + createFileImpl(StringRef Path, std::optional Config) override { + return createStringError(inconvertibleErrorCode(), Msg); + } + +public: + StringErrorBackend(const Twine &Msg) : Msg(Msg.str()) {} + std::string Msg; +}; + +TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) { + auto Error1 = makeIntrusiveRefCnt("error-backend-1"); + auto Null = makeNullOutputBackend(); + + auto Mirror = makeMirroringOutputBackend(Null, Error1); + EXPECT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(), + FailedWithMessage(Error1->Msg)); + + Mirror = makeMirroringOutputBackend(Error1, Null); + EXPECT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(), + FailedWithMessage(Error1->Msg)); + + auto Error2 = makeIntrusiveRefCnt("error-backend-2"); + Mirror = makeMirroringOutputBackend(Error1, Error2); + EXPECT_THAT_ERROR( + consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(), + FailedWithMessage(Error1->Msg)); +} + +TEST(OnDiskBackendTest, OnlyIfDifferent) { + OnDiskOutputBackendProvider Provider; + auto Backend = Provider.createBackend(); + std::string FilePath = Provider.getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setOnlyIfDifferent(); + + OutputFile O1, O2, O3; + sys::fs::file_status Status1, Status2, Status3; + // Write first file. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1), + Succeeded()); + O1 << Data; + EXPECT_THAT_ERROR(O1.keep(), Succeeded()); + EXPECT_FALSE(O1.isOpen()); + EXPECT_FALSE(sys::fs::status(FilePath, Status1, /*follow=*/false)); + + // Write second with same content. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2), + Succeeded()); + O2 << Data; + EXPECT_THAT_ERROR(O2.keep(), Succeeded()); + EXPECT_FALSE(O2.isOpen()); + EXPECT_FALSE(sys::fs::status(FilePath, Status2, /*follow=*/false)); + + // Make sure the output path file is not modified with same content. + EXPECT_EQ(Status1.getUniqueID(), Status2.getUniqueID()); + + // Write third with different content. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O3), + Succeeded()); + O3 << Data << "\n"; + EXPECT_THAT_ERROR(O3.keep(), Succeeded()); + EXPECT_FALSE(O3.isOpen()); + EXPECT_FALSE(sys::fs::status(FilePath, Status3, /*follow=*/false)); + + // This should overwrite the file and create a different UniqueID. + EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID()); +} + +TEST(OnDiskBackendTest, Append) { + OnDiskOutputBackendProvider Provider; + auto Backend = Provider.createBackend(); + std::string FilePath = Provider.getFilePathToCreate(); + OutputConfig Config = OutputConfig().setAppend(); + + OutputFile O1, O2, O3; + // Write first file. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1), + Succeeded()); + O1 << "some data\n"; + EXPECT_THAT_ERROR(O1.keep(), Succeeded()); + EXPECT_FALSE(O1.isOpen()); + + OnDiskFile File1(*Provider.D, FilePath); + EXPECT_TRUE(File1.equalsCurrentContent("some data\n")); + + // Append same data. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2), + Succeeded()); + O2 << "more data\n"; + EXPECT_THAT_ERROR(O2.keep(), Succeeded()); + EXPECT_FALSE(O2.isOpen()); + + // Check data is appended. + OnDiskFile File2(*Provider.D, FilePath); + EXPECT_TRUE(File2.equalsCurrentContent("some data\nmore data\n")); + + // Non atomic append. + EXPECT_THAT_ERROR( + Backend->createFile(FilePath, Config.setNoAtomicWrite()).moveInto(O3), + Succeeded()); + O3 << "more more\n"; + EXPECT_THAT_ERROR(O3.keep(), Succeeded()); + EXPECT_FALSE(O3.isOpen()); + + // Check data is appended. + OnDiskFile File3(*Provider.D, FilePath); + EXPECT_TRUE(File3.equalsCurrentContent("some data\nmore data\nmore more\n")); +} + +TEST(HashingBackendTest, HashOutput) { + HashingOutputBackend Backend; + OutputFile O1, O2, O3, O4, O5; + EXPECT_THAT_ERROR(Backend.createFile("file1").moveInto(O1), Succeeded()); + O1 << "some data"; + EXPECT_THAT_ERROR(O1.keep(), Succeeded()); + EXPECT_THAT_ERROR(Backend.createFile("file2").moveInto(O2), Succeeded()); + O2 << "some data"; + EXPECT_THAT_ERROR(O2.keep(), Succeeded()); + EXPECT_EQ(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file2")); + + EXPECT_THAT_ERROR(Backend.createFile("file3").moveInto(O3), Succeeded()); + O3 << "some "; + O3 << "data"; + EXPECT_THAT_ERROR(O3.keep(), Succeeded()); + EXPECT_EQ(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file3")); + + EXPECT_THAT_ERROR(Backend.createFile("file4").moveInto(O4), Succeeded()); + O4 << "same data"; + O4.getOS().pwrite("o", 1, 1); + EXPECT_THAT_ERROR(O4.keep(), Succeeded()); + EXPECT_EQ(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file4")); + + EXPECT_THAT_ERROR(Backend.createFile("file5").moveInto(O5), Succeeded()); + O5 << "different data"; + EXPECT_THAT_ERROR(O5.keep(), Succeeded()); + EXPECT_NE(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file5")); +} + +} // end namespace diff --git a/llvm/unittests/Support/VirtualOutputConfigTest.cpp b/llvm/unittests/Support/VirtualOutputConfigTest.cpp new file mode 100644 index 0000000000000..cf6cd19b619aa --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputConfigTest.cpp @@ -0,0 +1,152 @@ +//===- VirtualOutputConfigTest.cpp - vfs::OutputConfig tests --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/FileSystem.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +TEST(VirtualOutputConfigTest, construct) { + // Test defaults. + EXPECT_FALSE(OutputConfig().getText()); + EXPECT_FALSE(OutputConfig().getCRLF()); + EXPECT_TRUE(OutputConfig().getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig().getAtomicWrite()); + EXPECT_TRUE(OutputConfig().getImplyCreateDirectories()); + EXPECT_FALSE(OutputConfig().getOnlyIfDifferent()); + EXPECT_FALSE(OutputConfig().getAppend()); + + // Test inverted defaults. + EXPECT_TRUE(OutputConfig().getNoText()); + EXPECT_TRUE(OutputConfig().getNoCRLF()); + EXPECT_FALSE(OutputConfig().getNoDiscardOnSignal()); + EXPECT_FALSE(OutputConfig().getNoAtomicWrite()); + EXPECT_FALSE(OutputConfig().getNoImplyCreateDirectories()); + EXPECT_TRUE(OutputConfig().getNoOnlyIfDifferent()); + EXPECT_TRUE(OutputConfig().getNoAppend()); +} + +TEST(VirtualOutputConfigTest, set) { + // Check a flag that defaults to false. Try both 'get's, all three 'set's, + // and turning back off after turning it on. + ASSERT_TRUE(OutputConfig().getNoText()); + EXPECT_TRUE(OutputConfig().setText().getText()); + EXPECT_FALSE(OutputConfig().setText().getNoText()); + EXPECT_TRUE(OutputConfig().setText(true).getText()); + EXPECT_FALSE(OutputConfig().setText().setNoText().getText()); + EXPECT_FALSE(OutputConfig().setText().setText(false).getText()); + + // Check a flag that defaults to true. Try both 'get's, all three 'set's, and + // turning back on after turning it off. + ASSERT_TRUE(OutputConfig().getDiscardOnSignal()); + EXPECT_FALSE(OutputConfig().setNoDiscardOnSignal().getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig().setNoDiscardOnSignal().getNoDiscardOnSignal()); + EXPECT_FALSE(OutputConfig().setDiscardOnSignal(false).getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig() + .setNoDiscardOnSignal() + .setDiscardOnSignal() + .getDiscardOnSignal()); + EXPECT_TRUE(OutputConfig() + .setNoDiscardOnSignal() + .setDiscardOnSignal(true) + .getDiscardOnSignal()); + + // Set multiple flags. + OutputConfig Config; + Config.setText().setNoDiscardOnSignal().setNoImplyCreateDirectories(); + EXPECT_TRUE(Config.getText()); + EXPECT_TRUE(Config.getNoDiscardOnSignal()); + EXPECT_TRUE(Config.getNoImplyCreateDirectories()); +} + +TEST(VirtualOutputConfigTest, equals) { + EXPECT_TRUE(OutputConfig() == OutputConfig()); + EXPECT_FALSE(OutputConfig() != OutputConfig()); + EXPECT_EQ(OutputConfig().setAtomicWrite(), OutputConfig().setAtomicWrite()); + EXPECT_NE(OutputConfig().setAtomicWrite(), OutputConfig().setNoAtomicWrite()); +} + +static std::string toString(OutputConfig Config) { + std::string Printed; + raw_string_ostream OS(Printed); + Config.print(OS); + return Printed; +} + +TEST(VirtualOutputConfigTest, print) { + EXPECT_EQ("{}", toString(OutputConfig())); + EXPECT_EQ("{Text}", toString(OutputConfig().setText())); + EXPECT_EQ("{Text,NoDiscardOnSignal}", + toString(OutputConfig().setText().setNoDiscardOnSignal())); + EXPECT_EQ("{Text,NoDiscardOnSignal}", + toString(OutputConfig().setNoDiscardOnSignal().setText())); +} + +TEST(VirtualOutputConfigTest, BinaryAndTextWithCRLF) { + // Test defaults. + EXPECT_TRUE(OutputConfig().getBinary()); + EXPECT_FALSE(OutputConfig().getTextWithCRLF()); + EXPECT_FALSE(OutputConfig().getText()); + EXPECT_FALSE(OutputConfig().getCRLF()); + + // Test setting. + EXPECT_TRUE(OutputConfig().setTextWithCRLF().getTextWithCRLF()); + EXPECT_TRUE(OutputConfig().setTextWithCRLF().getText()); + EXPECT_TRUE(OutputConfig().setTextWithCRLF().getCRLF()); + EXPECT_TRUE(OutputConfig().setText().setCRLF().getTextWithCRLF()); + EXPECT_FALSE(OutputConfig().setText().getBinary()); + EXPECT_FALSE(OutputConfig().setTextWithCRLF().getBinary()); + EXPECT_FALSE(OutputConfig().setTextWithCRLF().setBinary().getText()); + EXPECT_FALSE(OutputConfig().setTextWithCRLF().setBinary().getCRLF()); + + // Test setTextWithCRLF(bool). + EXPECT_TRUE(OutputConfig().setBinary().setTextWithCRLF(true).getText()); + EXPECT_TRUE(OutputConfig().setBinary().setTextWithCRLF(true).getCRLF()); + EXPECT_TRUE( + OutputConfig().setTextWithCRLF().setTextWithCRLF(false).getBinary()); + + // Test printing. + EXPECT_EQ("{Text,CRLF}", toString(OutputConfig().setTextWithCRLF())); +} + +TEST(VirtualOutputConfigTest, OpenFlags) { + using namespace llvm::sys::fs; + + // Confirm the default is binary. + ASSERT_EQ(OutputConfig().setBinary(), OutputConfig()); + + // Most flags are not supported / have no effect. + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_None)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_Delete)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_ChildInherit)); + EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_UpdateAtime)); + + // Check setting OF_Text and OF_CRLF. + for (OutputConfig Init : { + OutputConfig(), + OutputConfig().setText(), + OutputConfig().setTextWithCRLF(), + OutputConfig().setAppend(), + + // Should be overridden despite being invalid. + OutputConfig().setCRLF(), + }) { + EXPECT_EQ(OutputConfig(), Init.setOpenFlags(OF_None)); + EXPECT_EQ(OutputConfig(), Init.setOpenFlags(OF_CRLF)); + EXPECT_EQ(OutputConfig().setText(), Init.setOpenFlags(OF_Text)); + EXPECT_EQ(OutputConfig().setTextWithCRLF(), + Init.setOpenFlags(OF_TextWithCRLF)); + EXPECT_EQ(OutputConfig().setAppend(), Init.setOpenFlags(OF_Append)); + } +} + +} // anonymous namespace diff --git a/llvm/unittests/Support/VirtualOutputFileTest.cpp b/llvm/unittests/Support/VirtualOutputFileTest.cpp new file mode 100644 index 0000000000000..8712bf0489449 --- /dev/null +++ b/llvm/unittests/Support/VirtualOutputFileTest.cpp @@ -0,0 +1,342 @@ +//===- VirtualOutputFileTest.cpp - vfs::OutputFile tests ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/VirtualOutputFile.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::vfs; + +namespace { + +struct MockOutputFileData { + int Kept = 0; + int Discarded = 0; + int Handled = 0; + unique_function Keeper; + unique_function Discarder; + + void handler(Error E) { + consumeError(std::move(E)); + ++Handled; + } + unique_function getHandler() { + return [this](Error E) { handler(std::move(E)); }; + } + + SmallString<128> V; + std::optional VOS; + raw_pwrite_stream *OS = nullptr; + + MockOutputFileData() : VOS(std::in_place, V), OS(&*VOS) {} + MockOutputFileData(raw_pwrite_stream &OS) : OS(&OS) {} +}; + +struct MockOutputFile final : public OutputFileImpl { + Error keep() override { + ++Data.Kept; + if (Data.Keeper) + return Data.Keeper(); + return Error::success(); + } + + Error discard() override { + ++Data.Discarded; + if (Data.Discarder) + return Data.Discarder(); + return Error::success(); + } + + raw_pwrite_stream &getOS() override { + if (!Data.OS) + report_fatal_error("missing stream in MockOutputFile::getOS"); + return *Data.OS; + } + + MockOutputFile(MockOutputFileData &Data) : Data(Data) {} + MockOutputFileData &Data; +}; + +static std::unique_ptr +createMockOutput(MockOutputFileData &Data) { + return std::make_unique(Data); +} + +static Error createCustomError() { + return createStringError(inconvertibleErrorCode(), "custom error"); +} + +TEST(VirtualOutputFileTest, construct) { + OutputFile F; + EXPECT_EQ("", F.getPath()); + EXPECT_FALSE(F); + EXPECT_FALSE(F.isOpen()); + +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) + EXPECT_DEATH(F.getOS(), "Expected open output stream"); +#endif +} + +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) +TEST(VirtualOutputFileTest, constructNull) { + EXPECT_DEATH(OutputFile("some/file/path", nullptr), + "Expected open output file"); +} +#endif + +TEST(VirtualOutputFileTest, destroy) { + MockOutputFileData Data; + StringRef FilePath = "some/file/path"; + + // Check behaviour when destroying, first without a handler and then with + // one. The handler shouldn't be called. + std::optional F(std::in_place, FilePath, createMockOutput(Data)); + EXPECT_TRUE(F->isOpen()); + EXPECT_EQ(FilePath, F->getPath()); + EXPECT_EQ(Data.OS, &F->getOS()); +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(F.reset(), "output not closed"); +#endif + F->discardOnDestroy(Data.getHandler()); + EXPECT_EQ(0, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + F.reset(); + EXPECT_EQ(1, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + + // Try again when discard returns an error. This time the handler should be + // called. + Data.Discarder = createCustomError; + F.emplace("some/file/path", createMockOutput(Data)); + F->discardOnDestroy(Data.getHandler()); + F.reset(); + EXPECT_EQ(2, Data.Discarded); + EXPECT_EQ(1, Data.Handled); +} + +TEST(VirtualOutputFileTest, destroyProxy) { + MockOutputFileData Data; + + std::optional F(std::in_place, "some/file/path", + createMockOutput(Data)); + F->discardOnDestroy(Data.getHandler()); + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F->createProxy().moveInto(Proxy), Succeeded()); + F.reset(); +#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG) + EXPECT_DEATH(*Proxy << "data", "use after reset"); +#endif + Proxy.reset(); +} + +TEST(VirtualOutputFileTest, discard) { + StringRef Content = "some data"; + MockOutputFileData Data; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.discard(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); + +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(consumeError(F.keep()), + "some/file/path: output already closed"); + EXPECT_DEATH(consumeError(F.discard()), + "some/file/path: output already closed"); +#endif + } + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); +} + +TEST(VirtualOutputFileTest, discardError) { + StringRef Content = "some data"; + MockOutputFileData Data; + Data.Discarder = createCustomError; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + EXPECT_THAT_ERROR(F.discard(), FailedWithMessage("custom error")); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + } + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); + EXPECT_EQ(0, Data.Handled); +} + +TEST(VirtualOutputFileTest, discardProxy) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.discard(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); +} + +TEST(VirtualOutputFileTest, discardProxyFlush) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F.getOS().SetBufferSize(Content.size() * 2); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ("", Data.V); + EXPECT_THAT_ERROR(F.discard(), Succeeded()); + EXPECT_EQ(Content, Data.V); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(1, Data.Discarded); +} + +TEST(VirtualOutputFileTest, keep) { + StringRef Content = "some data"; + MockOutputFileData Data; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.keep(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(consumeError(F.keep()), + "some/file/path: output already closed"); + EXPECT_DEATH(consumeError(F.discard()), + "some/file/path: output already closed"); +#endif + } + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); +} + +TEST(VirtualOutputFileTest, keepError) { + StringRef Content = "some data"; + MockOutputFileData Data; + Data.Keeper = createCustomError; + { + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F << Content; + EXPECT_EQ(Content, Data.V); + + EXPECT_THAT_ERROR(F.keep(), FailedWithMessage("custom error")); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + EXPECT_EQ(0, Data.Handled); + } + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + EXPECT_EQ(0, Data.Handled); +} + +TEST(VirtualOutputFileTest, keepProxy) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + Proxy.reset(); + EXPECT_THAT_ERROR(F.keep(), Succeeded()); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); +} + +#if GTEST_HAS_DEATH_TEST +TEST(VirtualOutputFileTest, keepProxyStillOpen) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + EXPECT_DEATH(consumeError(F.keep()), "some/file/path: output has open proxy"); +} +#endif + +TEST(VirtualOutputFileTest, keepProxyFlush) { + StringRef Content = "some data"; + MockOutputFileData Data; + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + F.getOS().SetBufferSize(Content.size() * 2); + + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ("", Data.V); + Proxy.reset(); + EXPECT_THAT_ERROR(F.keep(), Succeeded()); + EXPECT_EQ(Content, Data.V); + EXPECT_FALSE(F.isOpen()); + EXPECT_EQ(1, Data.Kept); + EXPECT_EQ(0, Data.Discarded); +} + +TEST(VirtualOutputFileTest, TwoProxies) { + StringRef Content = "some data"; + MockOutputFileData Data; + + OutputFile F("some/file/path", createMockOutput(Data)); + F.discardOnDestroy(Data.getHandler()); + + // Can't have two open proxies at once. + { + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + EXPECT_THAT_ERROR( + F.createProxy().takeError(), + FailedWithMessage("some/file/path: output has open proxy")); + } + EXPECT_EQ(0, Data.Kept); + EXPECT_EQ(0, Data.Discarded); + + // A second proxy after the first closes should work... + { + std::unique_ptr Proxy; + EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded()); + *Proxy << Content; + EXPECT_EQ(Content, Data.V); + } +} + +} // end namespace diff --git a/llvm/unittests/Support/raw_ostream_proxy_test.cpp b/llvm/unittests/Support/raw_ostream_proxy_test.cpp new file mode 100644 index 0000000000000..ee97fe65b6600 --- /dev/null +++ b/llvm/unittests/Support/raw_ostream_proxy_test.cpp @@ -0,0 +1,219 @@ +//===- raw_ostream_proxy_test.cpp - Tests for raw ostream proxies ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/raw_ostream_proxy.h" +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +/// Naive version of raw_svector_ostream that is buffered (by default) and +/// doesn't support pwrite. +class BufferedNoPwriteSmallVectorStream : public raw_ostream { +public: + // Choose a strange buffer size to ensure it doesn't collide with the default + // on \a raw_ostream. + constexpr static const size_t PreferredBufferSize = 63; + + size_t preferred_buffer_size() const override { return PreferredBufferSize; } + uint64_t current_pos() const override { return Vector.size(); } + void write_impl(const char *Ptr, size_t Size) override { + Vector.append(Ptr, Ptr + Size); + } + + bool is_displayed() const override { return IsDisplayed; } + + explicit BufferedNoPwriteSmallVectorStream(SmallVectorImpl &Vector) + : Vector(Vector) {} + ~BufferedNoPwriteSmallVectorStream() override { flush(); } + + SmallVectorImpl &Vector; + bool IsDisplayed = false; +}; + +constexpr const size_t BufferedNoPwriteSmallVectorStream::PreferredBufferSize; + +TEST(raw_ostream_proxyTest, write) { + // Besides confirming that "write" works, this test confirms that the proxy + // takes on the buffer from the stream it's proxying, such that writes to the + // proxy are flushed to the underlying stream as if no proxy were present. + SmallString<128> Dest; + { + // Confirm that BufferedNoPwriteSmallVectorStream is buffered by default, + // and that setting up a proxy effectively transfers a buffer of the same + // size to the proxy. + BufferedNoPwriteSmallVectorStream DestOS(Dest); + EXPECT_EQ(BufferedNoPwriteSmallVectorStream::PreferredBufferSize, + DestOS.GetBufferSize()); + raw_ostream_proxy ProxyOS(DestOS); + EXPECT_EQ(0u, DestOS.GetBufferSize()); + EXPECT_EQ(BufferedNoPwriteSmallVectorStream::PreferredBufferSize, + ProxyOS.GetBufferSize()); + + // Flushing should send through to Dest. + ProxyOS << "abcd"; + EXPECT_EQ("", Dest); + ProxyOS.flush(); + EXPECT_EQ("abcd", Dest); + + // Buffer should still work. + ProxyOS << "e"; + EXPECT_EQ("abcd", Dest); + } + + // Destructing ProxyOS should flush (and not crash). + EXPECT_EQ("abcde", Dest); + + { + // Set up another stream, this time unbuffered. + BufferedNoPwriteSmallVectorStream DestOS(Dest); + DestOS.SetUnbuffered(); + EXPECT_EQ(0u, DestOS.GetBufferSize()); + raw_ostream_proxy ProxyOS(DestOS); + EXPECT_EQ(0u, DestOS.GetBufferSize()); + EXPECT_EQ(0u, ProxyOS.GetBufferSize()); + + // Flushing should not be required. + ProxyOS << "f"; + EXPECT_EQ("abcdef", Dest); + } + EXPECT_EQ("abcdef", Dest); +} + +TEST(raw_ostream_proxyTest, pwrite) { + // This test confirms that the proxy takes on the buffer from the stream it's + // proxying, such that writes to the proxy are flushed to the underlying + // stream as if no proxy were present. + SmallString<128> Dest; + raw_svector_ostream DestOS(Dest); + raw_pwrite_stream_proxy ProxyOS(DestOS); + EXPECT_EQ(0u, ProxyOS.GetBufferSize()); + + // Get some initial data. + ProxyOS << "abcd"; + EXPECT_EQ("abcd", Dest); + + // Confirm that pwrite works. + ProxyOS.pwrite("BC", 2, 1); + EXPECT_EQ("aBCd", Dest); +} + +TEST(raw_ostream_proxyTest, pwriteWithBuffer) { + // This test confirms that when a buffer is configured, pwrite still works. + SmallString<128> Dest; + raw_svector_ostream DestOS(Dest); + DestOS.SetBufferSize(256); + EXPECT_EQ(256u, DestOS.GetBufferSize()); + + // Confirm that the proxy steals the buffer. + raw_pwrite_stream_proxy ProxyOS(DestOS); + EXPECT_EQ(0u, DestOS.GetBufferSize()); + EXPECT_EQ(256u, ProxyOS.GetBufferSize()); + + // Check that the buffer is working. + ProxyOS << "abcd"; + EXPECT_EQ("", Dest); + + // Confirm that pwrite flushes. + ProxyOS.pwrite("BC", 2, 1); + EXPECT_EQ("aBCd", Dest); +} + +class ProxyWithReset : public raw_ostream_proxy_adaptor<> { +public: + ProxyWithReset(raw_ostream &OS) : raw_ostream_proxy_adaptor<>(OS) {} + + // Allow this to be called outside the class. + using raw_ostream_proxy_adaptor<>::hasProxiedOS; + using raw_ostream_proxy_adaptor<>::getProxiedOS; + using raw_ostream_proxy_adaptor<>::resetProxiedOS; +}; + +TEST(raw_ostream_proxyTest, resetProxiedOS) { + // Confirm that base classes can drop the proxied OS before destruction and + // get consistent crashes. + SmallString<128> Dest; + BufferedNoPwriteSmallVectorStream DestOS(Dest); + ProxyWithReset ProxyOS(DestOS); + EXPECT_TRUE(ProxyOS.hasProxiedOS()); + EXPECT_EQ(&DestOS, &ProxyOS.getProxiedOS()); + + // Write some data. + ProxyOS << "abcd"; + EXPECT_EQ("", Dest); + + // Reset the underlying stream. + ProxyOS.resetProxiedOS(); + EXPECT_EQ("abcd", Dest); + EXPECT_EQ(0u, ProxyOS.GetBufferSize()); + EXPECT_FALSE(ProxyOS.hasProxiedOS()); + +#if GTEST_HAS_DEATH_TEST + EXPECT_DEATH(ProxyOS << "e", "use after reset"); + EXPECT_DEATH(ProxyOS.getProxiedOS(), "use after reset"); +#endif +} + +TEST(raw_ostream_proxyTest, ColorMode) { + { + SmallString<128> Dest; + BufferedNoPwriteSmallVectorStream DestOS(Dest); + raw_ostream_proxy ProxyOS(DestOS); + ProxyOS.enable_colors(true); + + WithColor(ProxyOS, HighlightColor::Error, ColorMode::Disable) << "test"; + EXPECT_EQ("", Dest); + ProxyOS.flush(); + EXPECT_EQ("test", Dest); + } + + { + SmallString<128> Dest; + BufferedNoPwriteSmallVectorStream DestOS(Dest); + raw_ostream_proxy ProxyOS(DestOS); + ProxyOS.enable_colors(true); + + WithColor(ProxyOS, HighlightColor::Error, ColorMode::Auto) << "test"; + EXPECT_EQ("", Dest); + ProxyOS.flush(); + EXPECT_EQ("test", Dest); + } + +#ifdef LLVM_ON_UNIX + { + SmallString<128> Dest; + BufferedNoPwriteSmallVectorStream DestOS(Dest); + raw_ostream_proxy ProxyOS(DestOS); + ProxyOS.enable_colors(true); + + WithColor(ProxyOS, HighlightColor::Error, ColorMode::Enable) << "test"; + EXPECT_EQ("", Dest); + ProxyOS.flush(); + EXPECT_EQ("\x1B[0;1;31mtest\x1B[0m", Dest); + } + + { + SmallString<128> Dest; + BufferedNoPwriteSmallVectorStream DestOS(Dest); + DestOS.IsDisplayed = true; + raw_ostream_proxy ProxyOS(DestOS); + ProxyOS.enable_colors(true); + + WithColor(ProxyOS, HighlightColor::Error, ColorMode::Auto) << "test"; + EXPECT_EQ("", Dest); + ProxyOS.flush(); + EXPECT_EQ("\x1B[0;1;31mtest\x1B[0m", Dest); + } +#endif +} + +} // end namespace