From 0998a3ac01fdbd30cbb803eae9d8c6a73b8edb11 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 May 2024 09:39:04 -0400 Subject: [PATCH 1/2] Remove `LocalStore::OptimiseStats::blocksFreed` as it is dead code --- src/libstore/unix/local-store.hh | 1 - src/libstore/unix/optimise-store.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/src/libstore/unix/local-store.hh b/src/libstore/unix/local-store.hh index 2b6e2e25f75..b3d7bd6d0c3 100644 --- a/src/libstore/unix/local-store.hh +++ b/src/libstore/unix/local-store.hh @@ -32,7 +32,6 @@ struct OptimiseStats { unsigned long filesLinked = 0; uint64_t bytesFreed = 0; - uint64_t blocksFreed = 0; }; struct LocalStoreConfig : virtual LocalFSStoreConfig diff --git a/src/libstore/unix/optimise-store.cc b/src/libstore/unix/optimise-store.cc index bedb77849b1..17b8e4d9236 100644 --- a/src/libstore/unix/optimise-store.cc +++ b/src/libstore/unix/optimise-store.cc @@ -256,7 +256,6 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, stats.filesLinked++; stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; if (act) act->result(resFileLinked, st.st_size, st.st_blocks); From e0ff8da9d585d7a3ea6be6ec4b0506dc5d7cc03c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Apr 2024 17:49:17 -0400 Subject: [PATCH 2/2] Build the local store on Windows Fixes #10558 Co-Authored-By: Eugene Butler Co-authored-by: Eelco Dolstra --- maintainers/flake-module.nix | 12 +-- src/libfetchers/cache.cc | 2 +- .../{unix => }/ca-specific-schema.sql | 0 src/libstore/{unix => }/gc.cc | 88 ++++++++++++------- src/libstore/globals.hh | 2 + src/libstore/{unix => }/local-store.cc | 74 ++++++++++------ src/libstore/{unix => }/local-store.hh | 0 src/libstore/{unix => }/local-store.md | 0 src/libstore/local.mk | 4 +- src/libstore/{unix => }/optimise-store.cc | 79 +++++++++-------- .../{unix => }/posix-fs-canonicalise.cc | 40 +++++++-- .../{unix => }/posix-fs-canonicalise.hh | 12 ++- src/libstore/{unix => }/schema.sql | 0 src/libstore/store-api.cc | 18 +--- src/libstore/unix/build/derivation-goal.cc | 26 ------ src/libutil/file-system.cc | 2 +- src/libutil/file-system.hh | 5 ++ src/libutil/unix-domain-socket.hh | 1 + src/libutil/unix/file-system.cc | 10 +++ src/libutil/windows/file-system.cc | 12 +++ 20 files changed, 230 insertions(+), 157 deletions(-) rename src/libstore/{unix => }/ca-specific-schema.sql (100%) rename src/libstore/{unix => }/gc.cc (94%) rename src/libstore/{unix => }/local-store.cc (97%) rename src/libstore/{unix => }/local-store.hh (100%) rename src/libstore/{unix => }/local-store.md (100%) rename src/libstore/{unix => }/optimise-store.cc (80%) rename src/libstore/{unix => }/posix-fs-canonicalise.cc (89%) rename src/libstore/{unix => }/posix-fs-canonicalise.hh (85%) rename src/libstore/{unix => }/schema.sql (100%) create mode 100644 src/libutil/unix/file-system.cc create mode 100644 src/libutil/windows/file-system.cc diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index fdac87be3fa..351a01fcbff 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -241,17 +241,17 @@ ''^src/libstore/unix/build/worker\.hh$'' ''^src/libstore/unix/builtins/fetchurl\.cc$'' ''^src/libstore/unix/builtins/unpack-channel\.cc$'' - ''^src/libstore/unix/gc\.cc$'' + ''^src/libstore/gc\.cc$'' ''^src/libstore/unix/local-overlay-store\.cc$'' ''^src/libstore/unix/local-overlay-store\.hh$'' - ''^src/libstore/unix/local-store\.cc$'' - ''^src/libstore/unix/local-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' ''^src/libstore/unix/lock\.cc$'' ''^src/libstore/unix/lock\.hh$'' - ''^src/libstore/unix/optimise-store\.cc$'' + ''^src/libstore/optimise-store\.cc$'' ''^src/libstore/unix/pathlocks\.cc$'' - ''^src/libstore/unix/posix-fs-canonicalise\.cc$'' - ''^src/libstore/unix/posix-fs-canonicalise\.hh$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' ''^src/libstore/uds-remote-store\.cc$'' ''^src/libstore/uds-remote-store\.hh$'' ''^src/libstore/windows/build\.cc$'' diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 87a8e4702f2..7019b0325d7 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -109,7 +109,7 @@ struct CacheImpl : Cache Key key, Store & store, Attrs value, - const StorePath & storePath) + const StorePath & storePath) override { /* Add the store prefix to the cache key to handle multiple store prefixes. */ diff --git a/src/libstore/unix/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql similarity index 100% rename from src/libstore/unix/ca-specific-schema.sql rename to src/libstore/ca-specific-schema.sql diff --git a/src/libstore/unix/gc.cc b/src/libstore/gc.cc similarity index 94% rename from src/libstore/unix/gc.cc rename to src/libstore/gc.cc index 38cbf12b218..3cd4fb839db 100644 --- a/src/libstore/unix/gc.cc +++ b/src/libstore/gc.cc @@ -7,7 +7,7 @@ #if !defined(__linux__) // For shelling out to lsof -# include "processes.hh" +# include "processes.hh" #endif #include @@ -19,18 +19,20 @@ #include #include #include -#include -#include #include -#include +#if HAVE_STATVFS +# include +#endif +#ifndef _WIN32 +# include +# include +# include +#endif #include -#include #include namespace nix { -using namespace nix::unix; - static std::string gcSocketPath = "/gc-socket/socket"; static std::string gcRootsDir = "gcroots"; @@ -64,7 +66,7 @@ void LocalStore::createTempRootsFile() /* Check whether the garbage collector didn't get in our way. */ struct stat st; - if (fstat(fdTempRoots->get(), &st) == -1) + if (fstat(fromDescriptorReadOnly(fdTempRoots->get()), &st) == -1) throw SysError("statting '%1%'", fnTempRoots); if (st.st_size == 0) break; @@ -108,7 +110,7 @@ void LocalStore::addTempRoot(const StorePath & path) debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { - nix::connect(fdRootsSocket->get(), socketPath); + nix::connect(toSocket(fdRootsSocket->get()), socketPath); } catch (SysError & e) { /* The garbage collector may have exited or not created the socket yet, so we need to restart. */ @@ -166,12 +168,16 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) // those to keep the directory alive. continue; } - Path path = i.path(); + Path path = i.path().string(); pid_t pid = std::stoi(name); debug("reading temporary root file '%1%'", path); - AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); + AutoCloseFD fd(toDescriptor(open(path.c_str(), +#ifndef _WIN32 + O_CLOEXEC | +#endif + O_RDWR, 0666))); if (!fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; @@ -240,8 +246,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R unlink(path.c_str()); } } else { - struct stat st2 = lstat(target); - if (!S_ISLNK(st2.st_mode)) return; + if (!std::filesystem::is_symlink(target)) return; Path target2 = readLink(target); if (isInStore(target2)) foundRoot(target, target2); } @@ -297,24 +302,25 @@ Roots LocalStore::findRoots(bool censor) return roots; } -typedef std::unordered_map> UncheckedRoots; +/** + * Key is a mere string because cannot has path with macOS's libc++ + */ +typedef std::unordered_map> UncheckedRoots; -static void readProcLink(const std::string & file, UncheckedRoots & roots) +static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) { - constexpr auto bufsiz = PATH_MAX; - char buf[bufsiz]; - auto res = readlink(file.c_str(), buf, bufsiz); - if (res == -1) { - if (errno == ENOENT || errno == EACCES || errno == ESRCH) + std::filesystem::path buf; + try { + buf = std::filesystem::read_symlink(file); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::no_such_file_or_directory + || e.code() == std::errc::permission_denied + || e.code() == std::errc::no_such_process) return; - throw SysError("reading symlink"); + throw; } - if (res == bufsiz) { - throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); - } - if (res > 0 && buf[0] == '/') - roots[std::string(static_cast(buf), res)] - .emplace(file); + if (buf.is_absolute()) + roots[buf.string()].emplace(file.string()); } static std::string quoteRegexChars(const std::string & raw) @@ -371,12 +377,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } fdDir.reset(); - auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>(readFile(mapFile), "\n"); + std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace(mapFile); + unchecked[match[1]].emplace(mapFile.string()); } auto envFile = fmt("/proc/%s/environ", ent->d_name); @@ -407,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); + unchecked[match[1].str()].emplace("{lsof}"); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -490,6 +496,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); + // TODO nonblocking socket on windows? +#ifdef _WIN32 + throw UnimplementedError("External GC client not implemented yet"); +#else if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) throw SysError("making socket '%1%' non-blocking", socketPath); @@ -590,6 +600,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (serverThread.joinable()) serverThread.join(); }); +#endif + /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printInfo("finding garbage collector roots..."); @@ -623,8 +635,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); - if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { + AutoCloseFD tmpDirFd = openDirectory(realPath); + if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir '%s'", realPath); return; } @@ -857,7 +869,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; + int64_t overhead = +#ifdef _WIN32 + 0 +#else + st.st_blocks * 512ULL +#endif + ; printInfo("note: currently hard linking saves %.2f MiB", ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); @@ -870,6 +888,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { +#ifdef HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { @@ -946,6 +965,7 @@ void LocalStore::autoGC(bool sync) sync: // Wait for the future outside of the state lock. if (sync) future.get(); +#endif } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0c71a751553..10893342219 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -424,8 +424,10 @@ public: Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; +#ifndef _WIN32 Setting syncBeforeRegistering{this, false, "sync-before-registering", "Whether to call `sync()` before registering a path as valid."}; +#endif Setting useSubstitutes{ this, true, "substitute", diff --git a/src/libstore/unix/local-store.cc b/src/libstore/local-store.cc similarity index 97% rename from src/libstore/unix/local-store.cc rename to src/libstore/local-store.cc index f33d9271780..800e6930971 100644 --- a/src/libstore/unix/local-store.cc +++ b/src/libstore/local-store.cc @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -34,17 +33,20 @@ #include #include #include -#include + +#ifndef _WIN32 +# include +#endif #if __linux__ -#include -#include -#include -#include +# include +# include +# include +# include #endif #ifdef __CYGWIN__ -#include +# include #endif #include @@ -52,8 +54,6 @@ namespace nix { -using namespace nix::unix; - std::string LocalStoreConfig::doc() { return @@ -224,6 +224,7 @@ LocalStore::LocalStore(const Params & params) } } +#ifndef _WIN32 /* Optionally, create directories and set permissions for a multi-user install. */ if (isRootUser() && settings.buildUsersGroup != "") { @@ -245,6 +246,7 @@ LocalStore::LocalStore(const Params & params) } } } +#endif /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { @@ -270,14 +272,25 @@ LocalStore::LocalStore(const Params & params) if (stat(reservedPath.c_str(), &st) == -1 || st.st_size != settings.reservedSize) { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); + AutoCloseFD fd = toDescriptor(open(reservedPath.c_str(), O_WRONLY | O_CREAT +#ifndef _WIN32 + | O_CLOEXEC +#endif + , 0600)); int res = -1; #if HAVE_POSIX_FALLOCATE res = posix_fallocate(fd.get(), 0, settings.reservedSize); #endif if (res == -1) { writeFull(fd.get(), std::string(settings.reservedSize, 'X')); - [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); + [[gnu::unused]] auto res2 = + +#ifdef _WIN32 + SetEndOfFile(fd.get()) +#else + ftruncate(fd.get(), settings.reservedSize) +#endif + ; } } } catch (SystemError & e) { /* don't care about errors */ @@ -460,10 +473,14 @@ LocalStore::LocalStore(std::string scheme, std::string path, const Params & para AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; - auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT +#ifndef _WIN32 + | O_CLOEXEC +#endif + , 0600); if (!fdGCLock) throw SysError("opening global GC lock '%1%'", fnGCLock); - return fdGCLock; + return toDescriptor(fdGCLock); } @@ -491,7 +508,7 @@ LocalStore::~LocalStore() try { auto fdTempRoots(_fdTempRoots.lock()); if (*fdTempRoots) { - *fdTempRoots = -1; + fdTempRoots->close(); unlink(fnTempRoots.c_str()); } } catch (...) { @@ -969,11 +986,13 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { +#ifndef _WIN32 /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. So some may want to fsync them before registering the validity, at the expense of some speed of the path registering operation. */ if (settings.syncBeforeRegistering) sync(); +#endif return retrySQLite([&]() { auto state(_state.lock()); @@ -1155,7 +1174,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, autoGC(); - canonicalisePathMetaData(realPath, {}); + canonicalisePathMetaData(realPath); optimisePath(realPath, repair); // FIXME: combine with hashPath() @@ -1307,7 +1326,7 @@ StorePath LocalStore::addToStoreFromDump( narHash = narSink.finish(); } - canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath + canonicalisePathMetaData(realPath); // FIXME: merge into restorePath optimisePath(realPath, repair); @@ -1340,8 +1359,8 @@ std::pair LocalStore::createTempDirInStore() the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ tmpDirFn = createTempDir(realStoreDir, "tmp"); - tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); - if (tmpDirFd.get() < 0) { + tmpDirFd = openDirectory(tmpDirFn); + if (!tmpDirFd) { continue; } lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); @@ -1390,19 +1409,16 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { auto name = link.path().filename(); printMsg(lvlTalkative, "checking contents of '%s'", name); - Path linkPath = linksDir / name; PosixSourceAccessor accessor; std::string hash = hashPath( - {getFSSourceAccessor(), CanonPath(linkPath)}, + PosixSourceAccessor::createAtRoot(link.path()), FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false); if (hash != name.string()) { printError("link '%s' was modified! expected hash '%s', got '%s'", - linkPath, name, hash); + link.path(), name, hash); if (repair) { - if (unlink(linkPath.c_str()) == 0) - printInfo("removed link '%s'", linkPath); - else - throw SysError("removing corrupt link '%s'", linkPath); + std::filesystem::remove(link.path()); + printInfo("removed link '%s'", link.path()); } else { errors = true; } @@ -1583,8 +1599,12 @@ static void makeMutable(const Path & path) /* The O_NOFOLLOW is important to prevent us from changing the mutable bit on the target of a symlink (which would be a security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd == -1) { + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW +#ifndef _WIN32 + | O_CLOEXEC +#endif + ); + if (fd == INVALID_DESCRIPTOR) { if (errno == ELOOP) return; // it's a symlink throw SysError("opening file '%1%'", path); } diff --git a/src/libstore/unix/local-store.hh b/src/libstore/local-store.hh similarity index 100% rename from src/libstore/unix/local-store.hh rename to src/libstore/local-store.hh diff --git a/src/libstore/unix/local-store.md b/src/libstore/local-store.md similarity index 100% rename from src/libstore/unix/local-store.md rename to src/libstore/local-store.md diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 590a230e577..cc67da78659 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -89,11 +89,11 @@ else endif endif -$(d)/unix/local-store.cc: $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh +$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(d)/unix/build.cc: -clean-files += $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh +clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libstore/unix/optimise-store.cc b/src/libstore/optimise-store.cc similarity index 80% rename from src/libstore/unix/optimise-store.cc rename to src/libstore/optimise-store.cc index 17b8e4d9236..2477cf0c020 100644 --- a/src/libstore/unix/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -155,55 +155,53 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false); + std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ - if (pathExists(linkPath)) { - auto stLink = lstat(linkPath); + if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + auto stLink = lstat(linkPath.string()); if (st.st_size != stLink.st_size || (repair && hash != ({ hashPath( - {make_ref(), CanonPath(linkPath)}, + PosixSourceAccessor::createAtRoot(linkPath), FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; }))) { // XXX: Consider overwriting linkPath with our valid version. - warn("removing corrupted link '%s'", linkPath); + warn("removing corrupted link %s", linkPath); warn("There may be more corrupted paths." "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); - unlink(linkPath.c_str()); + std::filesystem::remove(linkPath); } } - if (!pathExists(linkPath)) { + if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { /* Nope, create a hard link in the links directory. */ - if (link(path.c_str(), linkPath.c_str()) == 0) { + try { + std::filesystem::create_hard_link(path, linkPath); inodeHash.insert(st.st_ino); - return; - } - - switch (errno) { - case EEXIST: - /* Fall through if another process created ‘linkPath’ before - we did. */ - break; - - case ENOSPC: - /* On ext4, that probably means the directory index is - full. When that happens, it's fine to ignore it: we - just effectively disable deduplication of this - file. */ - printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); - return; - - default: - throw SysError("cannot link '%1%' to '%2%'", linkPath, path); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::file_exists) { + /* Fall through if another process created ‘linkPath’ before + we did. */ + } + + else if (e.code() == std::errc::no_space_on_device) { + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); + return; + } + + else throw; } } /* Yes! We've seen a file with the same contents. Replace the current file with a hard link to that file. */ - auto stLink = lstat(linkPath); + auto stLink = lstat(linkPath.string()); if (st.st_ino == stLink.st_ino) { debug("'%1%' is already linked to '%2%'", path, linkPath); @@ -223,10 +221,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - Path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), random()); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); - if (link(linkPath.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { + try { + std::filesystem::create_hard_link(linkPath, tempLink); + inodeHash.insert(st.st_ino); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::too_many_links) { /* Too many links to the same file (>= 32000 on most file systems). This is likely to happen with empty files. Just shrug and ignore. */ @@ -234,16 +235,16 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, printInfo("'%1%' has maximum number of links", linkPath); return; } - throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + throw; } /* Atomically replace the old file with the new hard link. */ try { - renameFile(tempLink, path); - } catch (SystemError & e) { - if (unlink(tempLink.c_str()) == -1) + std::filesystem::rename(tempLink, path); + } catch (std::filesystem::filesystem_error & e) { + std::filesystem::remove(tempLink); printError("unable to unlink '%1%'", tempLink); - if (errno == EMLINK) { + if (e.code() == std::errc::too_many_links) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it temporarily increases the st_nlink field before @@ -258,7 +259,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, stats.bytesFreed += st.st_size; if (act) - act->result(resFileLinked, st.st_size, st.st_blocks); + act->result(resFileLinked, st.st_size +#ifndef _WIN32 + , st.st_blocks +#endif + ); } diff --git a/src/libstore/unix/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc similarity index 89% rename from src/libstore/unix/posix-fs-canonicalise.cc rename to src/libstore/posix-fs-canonicalise.cc index 916db49ac22..d70ef6faefe 100644 --- a/src/libstore/unix/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -31,6 +31,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct } +#ifndef _WIN32 // TODO implement if (st.st_mtime != mtimeStore) { struct timeval times[2]; times[0].tv_sec = st.st_atime; @@ -46,6 +47,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #endif throw SysError("changing modification time of '%1%'", path); } +#endif } @@ -57,7 +59,9 @@ void canonicaliseTimestampAndPermissions(const Path & path) static void canonicalisePathMetaData_( const Path & path, +#ifndef _WIN32 std::optional> uidRange, +#endif InodesSeen & inodesSeen) { checkInterrupt(); @@ -99,6 +103,7 @@ static void canonicalisePathMetaData_( } #endif +#ifndef _WIN32 /* Fail if the file is not owned by the build user. This prevents us from messing up the ownership/permissions of files hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). @@ -112,11 +117,13 @@ static void canonicalisePathMetaData_( assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); return; } +#endif inodesSeen.insert(Inode(st.st_dev, st.st_ino)); canonicaliseTimestampAndPermissions(path, st); +#ifndef _WIN32 /* Change ownership to the current uid. If it's a symlink, use lchown if available, otherwise don't bother. Wrong ownership of a symlink doesn't matter, since the owning user can't change @@ -134,22 +141,36 @@ static void canonicalisePathMetaData_( throw SysError("changing owner of '%1%' to %2%", path, geteuid()); } +#endif if (S_ISDIR(st.st_mode)) { std::vector entries = readDirectory(path); for (auto & i : entries) - canonicalisePathMetaData_(i.path().string(), uidRange, inodesSeen); + canonicalisePathMetaData_( + i.path().string(), +#ifndef _WIN32 + uidRange, +#endif + inodesSeen); } } void canonicalisePathMetaData( const Path & path, +#ifndef _WIN32 std::optional> uidRange, +#endif InodesSeen & inodesSeen) { - canonicalisePathMetaData_(path, uidRange, inodesSeen); + canonicalisePathMetaData_( + path, +#ifndef _WIN32 + uidRange, +#endif + inodesSeen); +#ifndef _WIN32 /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ auto st = lstat(path); @@ -158,14 +179,23 @@ void canonicalisePathMetaData( assert(S_ISLNK(st.st_mode)); throw Error("wrong ownership of top-level store path '%1%'", path); } +#endif } -void canonicalisePathMetaData(const Path & path, - std::optional> uidRange) +void canonicalisePathMetaData(const Path & path +#ifndef _WIN32 + , std::optional> uidRange +#endif + ) { InodesSeen inodesSeen; - canonicalisePathMetaData(path, uidRange, inodesSeen); + canonicalisePathMetaData_( + path, +#ifndef _WIN32 + uidRange, +#endif + inodesSeen); } } diff --git a/src/libstore/unix/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh similarity index 85% rename from src/libstore/unix/posix-fs-canonicalise.hh rename to src/libstore/posix-fs-canonicalise.hh index 35644af125f..45a4f3f2069 100644 --- a/src/libstore/unix/posix-fs-canonicalise.hh +++ b/src/libstore/posix-fs-canonicalise.hh @@ -24,7 +24,7 @@ typedef std::set InodesSeen; * without execute permission; setuid bits etc. are cleared) * * - the owner and group are set to the Nix user and group, if we're - * running as root. + * running as root. (Unix only.) * * If uidRange is not empty, this function will throw an error if it * encounters files owned by a user outside of the closed interval @@ -32,11 +32,17 @@ typedef std::set InodesSeen; */ void canonicalisePathMetaData( const Path & path, +#ifndef _WIN32 std::optional> uidRange, +#endif InodesSeen & inodesSeen); + void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange); + const Path & path +#ifndef _WIN32 + , std::optional> uidRange = std::nullopt +#endif + ); void canonicaliseTimestampAndPermissions(const Path & path); diff --git a/src/libstore/unix/schema.sql b/src/libstore/schema.sql similarity index 100% rename from src/libstore/unix/schema.sql rename to src/libstore/schema.sql diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cefb5befd98..419c55e9239 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -20,10 +20,6 @@ #include "signals.hh" #include "users.hh" -#ifndef _WIN32 -# include "remote-store.hh" -#endif - #include #include @@ -1265,10 +1261,9 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) } -#ifndef _WIN32 -# include "local-store.hh" -# include "uds-remote-store.hh" -#endif + +#include "local-store.hh" +#include "uds-remote-store.hh" namespace nix { @@ -1286,9 +1281,6 @@ std::pair splitUriAndParams(const std::string & uri_ return {uri, params}; } -#ifdef _WIN32 // Unused on Windows because the next `#ifndef` -[[maybe_unused]] -#endif static bool isNonUriPath(const std::string & spec) { return @@ -1303,7 +1295,6 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para { // TODO reenable on Windows once we have `LocalStore` and // `UDSRemoteStore`. - #ifndef _WIN32 if (uri == "" || uri == "auto") { auto stateDir = getOr(params, "state", settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -1348,9 +1339,6 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } else { return nullptr; } - #else - return nullptr; - #endif } // The `parseURL` function supports both IPv6 URIs as defined in diff --git a/src/libstore/unix/build/derivation-goal.cc b/src/libstore/unix/build/derivation-goal.cc index 4d43429965d..8d6e3501516 100644 --- a/src/libstore/unix/build/derivation-goal.cc +++ b/src/libstore/unix/build/derivation-goal.cc @@ -30,32 +30,6 @@ #include #include -#if HAVE_STATVFS -#include -#endif - -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if __APPLE__ -#include -#include -#endif - #include #include diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 47251dbd7a4..39efa19fe40 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -94,7 +94,7 @@ Path canonPath(PathView path, bool resolveSymlinks) path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { - if (resolveSymlinks && std::filesystem::is_symlink(result)) { + if (resolveSymlinks && fs::is_symlink(result)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%0%'", remaining); remaining = (temp = concatStrings(readLink(result), remaining)); diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 5a068810497..4536accc379 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -117,6 +117,11 @@ bool pathAccessible(const Path & path); */ Path readLink(const Path & path); +/** + * Open a `Descriptor` with read-only access to the given directory. + */ +Descriptor openDirectory(const std::filesystem::path & path); + /** * Read the contents of a directory. The entries `.` and `..` are * removed. diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh index a8bbc4b89d3..ba2baeb1334 100644 --- a/src/libutil/unix-domain-socket.hh +++ b/src/libutil/unix-domain-socket.hh @@ -39,6 +39,7 @@ using Socket = * Windows gives this a different name */ # define SHUT_WR SD_SEND +# define SHUT_RDWR SD_BOTH #endif /** diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc new file mode 100644 index 00000000000..bbbbfa5597c --- /dev/null +++ b/src/libutil/unix/file-system.cc @@ -0,0 +1,10 @@ +#include "file-system.hh" + +namespace nix { + +Descriptor openDirectory(const std::filesystem::path & path) +{ + return open(path.c_str(), O_RDONLY | O_DIRECTORY); +} + +} diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc new file mode 100644 index 00000000000..8002dd75eec --- /dev/null +++ b/src/libutil/windows/file-system.cc @@ -0,0 +1,12 @@ +#include "file-system.hh" + +namespace nix { + +Descriptor openDirectory(const std::filesystem::path & path) +{ + return CreateFileW( + path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); +} + +}