diff --git a/llvm/include/llvm/CAS/ActionCache.h b/llvm/include/llvm/CAS/ActionCache.h index 42236c9b5b45f..40c284e9dd574 100644 --- a/llvm/include/llvm/CAS/ActionCache.h +++ b/llvm/include/llvm/CAS/ActionCache.h @@ -48,7 +48,6 @@ using AsyncCASIDValue = AsyncValue; struct AsyncErrorValue { Error take() { return std::move(Value); } - AsyncErrorValue() : Value(Error::success()) {} AsyncErrorValue(Error &&E) : Value(std::move(E)) {} private: diff --git a/llvm/include/llvm/CAS/CASID.h b/llvm/include/llvm/CAS/CASID.h index b6780e3fc6f8e..45a5194127121 100644 --- a/llvm/include/llvm/CAS/CASID.h +++ b/llvm/include/llvm/CAS/CASID.h @@ -131,7 +131,6 @@ class CASID { template struct AsyncValue { Expected> take() { return std::move(Value); } - AsyncValue() : Value(std::nullopt) {} AsyncValue(Error &&E) : Value(std::move(E)) {} AsyncValue(T &&V) : Value(std::move(V)) {} AsyncValue(std::nullopt_t) : Value(std::nullopt) {} diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h index 004f191e13cf4..47c1bb78ddcfd 100644 --- a/llvm/include/llvm/Support/FileSystem.h +++ b/llvm/include/llvm/Support/FileSystem.h @@ -410,6 +410,11 @@ LLVM_ABI std::error_code copy_file(const Twine &From, int ToFD); /// platform-specific error_code. LLVM_ABI std::error_code resize_file(int FD, uint64_t Size); +/// Resize path to size with sparse files explicitly enabled. It uses +/// FSCTL_SET_SPARSE On Windows. This is the same as resize_file on +/// non-Windows +LLVM_ABI std::error_code resize_file_sparse(int FD, uint64_t Size); + /// Resize \p FD to \p Size before mapping \a mapped_file_region::readwrite. On /// non-Windows, this calls \a resize_file(). On Windows, this is a no-op, /// since the subsequent mapping (via \c CreateFileMapping) automatically diff --git a/llvm/lib/CAS/HierarchicalTreeBuilder.cpp b/llvm/lib/CAS/HierarchicalTreeBuilder.cpp index d2cefd43ba057..6ef5bdfb91f53 100644 --- a/llvm/lib/CAS/HierarchicalTreeBuilder.cpp +++ b/llvm/lib/CAS/HierarchicalTreeBuilder.cpp @@ -22,8 +22,10 @@ static StringRef canonicalize(SmallVectorImpl &Path, TreeEntry::EntryKind Kind, sys::path::Style PathStyle) { const char PathSeparatorChar = get_separator(PathStyle)[0]; - // Make absolute. - if (Path.empty() || !is_absolute(Path, PathStyle)) + // Make absolute. Use has_root_directory instead of is_absolute + // because we may not have the drive like C: for abstract tree + // structures on Windows. + if (Path.empty() || !has_root_directory(Path, PathStyle)) Path.insert(Path.begin(), PathSeparatorChar); // FIXME: consider rejecting ".." instead of removing them. @@ -153,7 +155,7 @@ Expected HierarchicalTreeBuilder::create(ObjectStore &CAS) { Tree *Current = &Root; StringRef Path = Entry.getPath(); { - assert(is_absolute(Path, PathStyle) && "Expected absolute paths"); + assert(has_root_directory(Path, PathStyle) && "Expected absolute paths"); StringRef root_path = sys::path::root_path(Path, PathStyle); assert(!root_path.empty() && "Expected canonical POSIX absolute paths"); Path.consume_front(root_path); diff --git a/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp b/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp index ed94387143431..2ed691d03fe76 100644 --- a/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp +++ b/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp @@ -166,10 +166,8 @@ Expected MappedFileRegionBumpPtr::create( // We are initializing the file; it may be empty, or may have been shrunk // during a previous close. // FIXME: Detect a case where someone opened it with a smaller capacity. - // FIXME: On Windows we should use FSCTL_SET_SPARSE and FSCTL_SET_ZERO_DATA - // to make this a sparse region, if supported. assert(InitLock.Locked == FileLockRAII::Exclusive); - if (std::error_code EC = sys::fs::resize_file(FD, Capacity)) + if (std::error_code EC = sys::fs::resize_file_sparse(FD, Capacity)) return createFileError(Result.Path, EC); if (Result.Logger) @@ -230,7 +228,8 @@ void MappedFileRegionBumpPtr::destroyImpl() { assert(Size < Capacity); // sync to file system to make sure all contents are up-to-date. (void)Region.sync(); - (void)sys::fs::resize_file(*FD, size()); + Region.unmap(); + (void)sys::fs::resize_file(*FD, Size); (void)unlockFileThreadSafe(*SharedLockFD); if (Logger) @@ -338,4 +337,4 @@ ErrorOr FileSizeInfo::get(sys::fs::file_t File) { return EC; return FileSizeInfo{Status.getSize(), Status.getSize()}; #endif -} \ No newline at end of file +} diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc index 2326c074c9364..2b3c03df7494a 100644 --- a/llvm/lib/Support/Unix/Path.inc +++ b/llvm/lib/Support/Unix/Path.inc @@ -600,6 +600,10 @@ std::error_code resize_file(int FD, uint64_t Size) { return std::error_code(); } +std::error_code resize_file_sparse(int FD, uint64_t Size) { + return resize_file(FD, Size); +} + static int convertAccessMode(AccessMode Mode) { switch (Mode) { case AccessMode::Exist: diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc index 1f5580d2d5de9..dc3f0f9dd191e 100644 --- a/llvm/lib/Support/Windows/Path.inc +++ b/llvm/lib/Support/Windows/Path.inc @@ -27,6 +27,7 @@ #include "llvm/Support/Windows/WindowsSupport.h" #include #include +#include #undef max @@ -623,6 +624,22 @@ std::error_code resize_file(int FD, uint64_t Size) { return std::error_code(error, std::generic_category()); } +std::error_code resize_file_sparse(int FD, uint64_t Size) { + HANDLE hFile = reinterpret_cast(::_get_osfhandle(FD)); + DWORD temp; + if (!DeviceIoControl(hFile, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &temp, + NULL)) { + return mapWindowsError(GetLastError()); + } + LARGE_INTEGER liSize; + liSize.QuadPart = Size; + if (!SetFilePointerEx(hFile, liSize, NULL, FILE_BEGIN) || + !SetEndOfFile(hFile)) { + return mapWindowsError(GetLastError()); + } + return std::error_code(); +} + std::error_code access(const Twine &Path, AccessMode Mode) { SmallVector PathUtf16; diff --git a/llvm/unittests/CAS/CASConfigurationTest.cpp b/llvm/unittests/CAS/CASConfigurationTest.cpp index 0416ab799e215..d6e584b1a2761 100644 --- a/llvm/unittests/CAS/CASConfigurationTest.cpp +++ b/llvm/unittests/CAS/CASConfigurationTest.cpp @@ -43,21 +43,46 @@ TEST(CASConfigurationTest, roundTrips) { } TEST(CASConfigurationTest, configFileSearch) { + // /a/b/c + SmallString<261> PathToC = sys::path::get_separator(); + sys::path::append(PathToC, "a", "b", "c"); + // /a/b/c/d + SmallString<261> PathToD = PathToC; + sys::path::append(PathToD, "d"); + // /a/b/c/d/e + SmallString<261> PathToE = PathToD; + sys::path::append(PathToE, "e"); + // /a/b/c/.cas-config + SmallString<261> PathToCConfig = PathToC; + sys::path::append(PathToCConfig, ".cas-config"); + // /a/b/c/d/.cas-config + SmallString<261> PathToDConfig = PathToD; + sys::path::append(PathToDConfig, ".cas-config"); + auto VFS = makeIntrusiveRefCnt(); - ASSERT_FALSE(CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS)); + ASSERT_FALSE( + CASConfiguration::createFromSearchConfigFile(PathToE.str(), VFS)); // Add an empty file. - VFS->addFile("/a/b/c/.cas-config", 0, + VFS->addFile(PathToCConfig.str(), 0, llvm::MemoryBuffer::getMemBufferCopy("")); - ASSERT_FALSE(CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS)); + ASSERT_FALSE( + CASConfiguration::createFromSearchConfigFile(PathToE.str(), VFS)); + + VFS->addFile(PathToDConfig.str(), 0, +#ifndef _WIN32 + llvm::MemoryBuffer::getMemBufferCopy("{\"CASPath\": \"/tmp\"}")); +#else + llvm::MemoryBuffer::getMemBufferCopy("{\"CASPath\": \"\\\\tmp\"}")); +#endif - VFS->addFile("/a/b/c/d/.cas-config", 0, - llvm::MemoryBuffer::getMemBufferCopy("{\"CASPath\": \"/tmp\"}")); CASConfiguration Config; - Config.CASPath = "/tmp"; + SmallString<261> CASPath = sys::path::get_separator(); + sys::path::append(CASPath, "tmp"); + Config.CASPath = CASPath.str(); auto NewConfig = - CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS); + CASConfiguration::createFromSearchConfigFile(PathToE.str(), VFS); ASSERT_TRUE(NewConfig); - ASSERT_TRUE(NewConfig->first == "/a/b/c/d/.cas-config"); + ASSERT_TRUE(NewConfig->first == PathToDConfig.str()); ASSERT_TRUE(Config == NewConfig->second); } diff --git a/llvm/unittests/CAS/ObjectStoreTest.cpp b/llvm/unittests/CAS/ObjectStoreTest.cpp index e3564e6ba0ffd..5fd301200c5be 100644 --- a/llvm/unittests/CAS/ObjectStoreTest.cpp +++ b/llvm/unittests/CAS/ObjectStoreTest.cpp @@ -347,6 +347,7 @@ TEST_P(CASTest, BlobsBigParallel) { } #if LLVM_ENABLE_ONDISK_CAS +#ifndef _WIN32 // create_link won't work for directories on Windows TEST(OnDiskCASTest, BlobsParallelMultiCAS) { // This test intentionally uses symlinked paths to the same CAS to subvert the // shared memory mappings that would normally be created within a single @@ -402,8 +403,8 @@ TEST(OnDiskCASTest, BlobsBigParallelMultiCAS) { uint64_t Size = 100ULL * 1024; ASSERT_NO_FATAL_FAILURE(testBlobsParallel(*CAS1, *CAS2, *CAS3, *CAS4, Size)); } +#endif // _WIN32 -#ifndef _WIN32 // FIXME: resize support on Windows. TEST(OnDiskCASTest, DiskSize) { setMaxOnDiskCASMappingSize(); unittest::TempDir Temp("on-disk-cas", /*Unique=*/true); @@ -451,7 +452,6 @@ TEST(OnDiskCASTest, DiskSize) { CAS.reset(); CheckFileSizes(/*Mapped=*/false); } -#endif // _WIN32 #endif // LLVM_ENABLE_ONDISK_CAS #endif // LLVM_ENABLE_THREADS diff --git a/llvm/unittests/CAS/OnDiskCASLoggerTest.cpp b/llvm/unittests/CAS/OnDiskCASLoggerTest.cpp index a5143e5510028..b564aa3f8eb61 100644 --- a/llvm/unittests/CAS/OnDiskCASLoggerTest.cpp +++ b/llvm/unittests/CAS/OnDiskCASLoggerTest.cpp @@ -159,7 +159,12 @@ TEST(OnDiskCASLoggerTest, MultiProcess) { SmallVector PIs; for (int I = 0; I < 5; ++I) { bool ExecutionFailed; + #ifndef _WIN32 auto PI = ExecuteNoWait(Executable, Argv, ArrayRef{}, {}, 0, &Error, + #else + // CreateProcessW will fail with zero-length env on Windows + auto PI = ExecuteNoWait(Executable, Argv, std::nullopt, {}, 0, &Error, + #endif &ExecutionFailed); ASSERT_FALSE(ExecutionFailed) << Error; PIs.push_back(std::move(PI)); diff --git a/llvm/unittests/CAS/PluginCASTest.cpp b/llvm/unittests/CAS/PluginCASTest.cpp index 30361aacb77b7..6584e6694e60a 100644 --- a/llvm/unittests/CAS/PluginCASTest.cpp +++ b/llvm/unittests/CAS/PluginCASTest.cpp @@ -30,8 +30,13 @@ static std::string getCASPluginPath() { sys::fs::getMainExecutable(TestMainArgv0, &TestStringArg1); llvm::SmallString<256> PathBuf(sys::path::parent_path( sys::path::parent_path(sys::path::parent_path(Executable)))); +#ifndef _WIN32 std::string LibName = "libCASPluginTest"; sys::path::append(PathBuf, "lib", LibName + LLVM_PLUGIN_EXT); +#else + std::string LibName = "CASPluginTest"; + sys::path::append(PathBuf, "bin", LibName + LLVM_PLUGIN_EXT); +#endif return std::string(PathBuf); }