Lazy trees v2

This adds a setting 'lazy-trees' that causes flake inputs to be
"mounted" as virtual filesystems on top of /nix/store as random
"virtual" store paths. Only when the store path is actually used as a
dependency of a store derivation do we materialize ("devirtualize")
the input by copying it to its content-addressed location in the
store.

String contexts determine when devirtualization happens. One wrinkle
is that there are cases where we had store paths without proper
contexts, in particular when the user does `toString <path>` (where
<path> is a source tree in the Nix store) and passes the result to a
derivation. This usage was always broken, since it can result in
derivations that lack correct references. But to ensure that we don't
change evaluation results, we introduce a new type of context that
results in devirtualization but not in store references. We also now
print a warning about this.

Author: edolstra

src/libcmd/installable-value.cc

@@ -57,7 +57,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
     else if (v.type() == nString) {
         return {{
             .path = DerivedPath::fromSingle(
-                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
+                state->devirtualize(
+                    state->coerceToSingleDerivedPath(pos, v, errorCtx))),
             .info = make_ref<ExtraPathInfo>(),
         }};
     }

src/libexpr-test-support/include/nix/expr/tests/value/context.hh

@@ -23,6 +23,11 @@ struct Arbitrary<NixStringContextElem::DrvDeep> {
     static Gen<NixStringContextElem::DrvDeep> arbitrary();
 };
 
+template<>
+struct Arbitrary<NixStringContextElem::Path> {
+    static Gen<NixStringContextElem::Path> arbitrary();
+};
+
 template<>
 struct Arbitrary<NixStringContextElem> {
     static Gen<NixStringContextElem> arbitrary();

src/libexpr-test-support/tests/value/context.cc

@@ -15,6 +15,15 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb
     });
 }
 
+Gen<NixStringContextElem::Path> Arbitrary<NixStringContextElem::Path>::arbitrary()
+{
+    return gen::map(gen::arbitrary<StorePath>(), [](StorePath storePath) {
+        return NixStringContextElem::Path{
+            .storePath = storePath,
+        };
+    });
+}
+
 Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
 {
     return gen::mapcat(
@@ -30,6 +39,9 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
             case 2:
                 return gen::map(
                     gen::arbitrary<NixStringContextElem::Built>(), [](NixStringContextElem a) { return a; });
+            case 3:
+                return gen::map(
+                    gen::arbitrary<NixStringContextElem::Path>(), [](NixStringContextElem a) { return a; });
             default:
                 assert(false);
             }

src/libexpr/eval-cache.cc

@@ -618,18 +618,21 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    const StorePath & path = std::visit(overloaded {
-                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
-                            return d.drvPath;
+                    const StorePath * path = std::visit(overloaded {
+                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath * {
+                            return &d.drvPath;
                         },
-                        [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath->getBaseStorePath();
+                        [&](const NixStringContextElem::Built & b) -> const StorePath * {
+                            return &b.drvPath->getBaseStorePath();
                         },
-                        [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
-                            return o.path;
+                        [&](const NixStringContextElem::Opaque & o) -> const StorePath * {
+                            return &o.path;
+                        },
+                        [&](const NixStringContextElem::Path & p) -> const StorePath * {
+                            return nullptr;
                         },
                     }, c.raw);
-                    if (!root->state.store->isValidPath(path)) {
+                    if (!path || !root->state.store->isValidPath(*path)) {
                         valid = false;
                         break;
                     }

src/libexpr/eval.cc

@@ -15,6 +15,7 @@
 #include "nix/expr/print.hh"
 #include "nix/fetchers/filtering-source-accessor.hh"
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/util/mounted-source-accessor.hh"
 #include "nix/expr/gc-small-vector.hh"
 #include "nix/util/url.hh"
 #include "nix/fetchers/fetch-to-store.hh"
@@ -269,7 +270,7 @@ EvalState::EvalState(
                    exception, and make union source accessor
                    catch it, so we don't need to do this hack.
                  */
-                {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
+                {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))}
             }))
     , rootFS(
         ({
@@ -284,12 +285,9 @@ EvalState::EvalState(
                /nix/store while using a chroot store. */
             auto accessor = getFSSourceAccessor();
 
-            auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
-            if (settings.pureEval || store->storeDir != realStoreDir) {
-                accessor = settings.pureEval
-                    ? storeFS
-                    : makeUnionSourceAccessor({accessor, storeFS});
-            }
+            accessor = settings.pureEval
+                ? storeFS.cast<SourceAccessor>()
+                : makeUnionSourceAccessor({accessor, storeFS});
 
             /* Apply access control if needed. */
             if (settings.restrictEval || settings.pureEval)
@@ -968,7 +966,16 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(sFile).mkString(path->path.abs());
+        if (path->accessor == rootFS && store->isInStore(path->path.abs()))
+            // FIXME: only do this for virtual store paths?
+            attrs.alloc(sFile).mkString(path->path.abs(),
+                {
+                    NixStringContextElem::Path{
+                        .storePath = store->toStorePath(path->path.abs()).first
+                    }
+                });
+        else
+            attrs.alloc(sFile).mkString(path->path.abs());
         makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
         v.mkAttrs(attrs);
     } else
@@ -2089,7 +2096,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nFloat)
         v.mkFloat(nf);
     else if (firstType == nPath) {
-        if (!context.empty())
+        if (hasContext(context))
             state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
         v.mkPath(state.rootPath(CanonPath(str())));
     } else
@@ -2288,7 +2295,10 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
 {
     auto s = forceString(v, pos, errorCtx);
     if (v.context()) {
-        error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
+        NixStringContext context;
+        copyContext(v, context);
+        if (hasContext(context))
+            error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
     }
     return s;
 }
@@ -2337,14 +2347,26 @@ BackedStringView EvalState::coerceToString(
     }
 
     if (v.type() == nPath) {
+        // FIXME: instead of copying the path to the store, we could
+        // return a virtual store path that lazily copies the path to
+        // the store in devirtualize().
         return
             !canonicalizePath && !copyToStore
             ? // FIXME: hack to preserve path literals that end in a
               // slash, as in /foo/${x}.
               v.payload.path.path
             : copyToStore
             ? store->printStorePath(copyPathToStore(context, v.path()))
-            : std::string(v.path().path.abs());
+            : ({
+                auto path = v.path();
+                if (path.accessor == rootFS && store->isInStore(path.path.abs())) {
+                    context.insert(
+                        NixStringContextElem::Path{
+                            .storePath = store->toStorePath(path.path.abs()).first
+                        });
+                }
+                std::string(path.path.abs());
+              });
     }
 
     if (v.type() == nAttrs) {
@@ -2426,7 +2448,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
                 *store,
                 path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
-                path.baseName(),
+                computeBaseName(path),
                 ContentAddressMethod::Raw::NixArchive,
                 nullptr,
                 repair);
@@ -2481,7 +2503,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (auto storePath = store->maybeParseStorePath(path))
         return *storePath;
-    error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
+    error<EvalError>("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow();
 }
 
 
@@ -2507,6 +2529,11 @@ std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedP
         [&](NixStringContextElem::Built && b) -> SingleDerivedPath {
             return std::move(b);
         },
+        [&](NixStringContextElem::Path && p) -> SingleDerivedPath {
+            error<EvalError>(
+                "string '%s' has no context",
+                s).withTrace(pos, errorCtx).debugThrow();
+        },
     }, ((NixStringContextElem &&) *context.begin()).raw);
     return {
         std::move(derivedPath),
@@ -3090,6 +3117,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
 
         auto res = (r / CanonPath(suffix)).resolveSymlinks();
         if (res.pathExists()) return res;
+
+        // Backward compatibility hack: throw an exception if access
+        // to this path is not allowed.
+        if (auto accessor = res.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
+            accessor->checkAccess(res.path);
     }
 
     if (hasPrefix(path, "nix/"))
@@ -3160,6 +3192,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
         if (path.resolveSymlinks().pathExists())
             return finish(std::move(path));
         else {
+            // Backward compatibility hack: throw an exception if access
+            // to this path is not allowed.
+            if (auto accessor = path.accessor.dynamic_pointer_cast<FilteringSourceAccessor>())
+                accessor->checkAccess(path.path);
+
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)
             });

src/libexpr/include/nix/expr/eval-settings.hh

@@ -247,6 +247,11 @@ struct EvalSettings : Config
 
           This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
         )"};
+
+    Setting<bool> lazyTrees{this, false, "lazy-trees",
+        R"(
+          If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily.
+        )"};
 };
 
 /**

src/libexpr/include/nix/expr/eval.hh

@@ -36,13 +36,15 @@ class Store;
 namespace fetchers {
 struct Settings;
 struct InputCache;
+struct Input;
 }
 struct EvalSettings;
 class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct MountedSourceAccessor;
 namespace eval_cache {
     class EvalCache;
 }
@@ -271,7 +273,7 @@ public:
     /**
      * The accessor corresponding to `store`.
      */
-    const ref<SourceAccessor> storeFS;
+    const ref<MountedSourceAccessor> storeFS;
 
     /**
      * The accessor for the root filesystem.
@@ -449,6 +451,15 @@ public:
 
     void checkURI(const std::string & uri);
 
+    /**
+     * Mount an input on the Nix store.
+     */
+    StorePath mountInput(
+        fetchers::Input & input,
+        const fetchers::Input & originalInput,
+        ref<SourceAccessor> accessor,
+        bool requireLockable);
+
     /**
      * Parse a Nix expression from the specified file.
      */
@@ -558,6 +569,18 @@ public:
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
         NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
 
+    StorePath devirtualize(
+        const StorePath & path,
+        StringMap * rewrites = nullptr);
+
+    SingleDerivedPath devirtualize(
+        const SingleDerivedPath & path,
+        StringMap * rewrites = nullptr);
+
+    std::string devirtualize(
+        std::string_view s,
+        const NixStringContext & context);
+
     /**
      * String coercion.
      *
@@ -573,6 +596,19 @@ public:
 
     StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
 
+
+    /**
+     * Compute the base name for a `SourcePath`. For non-store paths,
+     * this is just `SourcePath::baseName()`. But for store paths, for
+     * backwards compatibility, it needs to be `<hash>-source`,
+     * i.e. as if the path were copied to the Nix store. This results
+     * in a "double-copied" store path like
+     * `/nix/store/<hash1>-<hash2>-source`. We don't need to
+     * materialize /nix/store/<hash2>-source though. Still, this
+     * requires reading/hashing the path twice.
+     */
+    std::string computeBaseName(const SourcePath & path);
+
     /**
      * Path coercion.
      *

src/libexpr/include/nix/expr/print-ambiguous.hh

@@ -15,10 +15,10 @@ namespace nix {
  * See: https://github.com/NixOS/nix/issues/9730
  */
 void printAmbiguous(
-    Value &v,
-    const SymbolTable &symbols,
-    std::ostream &str,
-    std::set<const void *> *seen,
+    EvalState & state,
+    Value & v,
+    std::ostream & str,
+    std::set<const void *> * seen,
     int depth);
 
 }

src/libexpr/include/nix/expr/value/context.hh

@@ -54,10 +54,35 @@ struct NixStringContextElem {
      */
     using Built = SingleDerivedPath::Built;
 
+    /**
+     * A store path that will not result in a store reference when
+     * used in a derivation or toFile.
+     *
+     * When you apply `builtins.toString` to a path value representing
+     * a path in the Nix store (as is the case with flake inputs),
+     * historically you got a string without context
+     * (e.g. `/nix/store/...-source`). This is broken, since it allows
+     * you to pass a store path to a derivation/toFile without a
+     * proper store reference. This is especially a problem with lazy
+     * trees, since the store path is a virtual path that doesn't
+     * exist.
+     *
+     * For backwards compatibility, and to warn users about this
+     * unsafe use of `toString`, we keep track of such strings as a
+     * special type of context.
+     */
+    struct Path
+    {
+        StorePath storePath;
+
+        GENERATE_CMP(Path, me->storePath);
+    };
+
     using Raw = std::variant<
         Opaque,
         DrvDeep,
-        Built
+        Built,
+        Path
     >;
 
     Raw raw;
@@ -82,4 +107,10 @@ struct NixStringContextElem {
 
 typedef std::set<NixStringContextElem> NixStringContext;
 
+/**
+ * Returns false if `context` has no elements other than
+ * `NixStringContextElem::Path`.
+ */
+bool hasContext(const NixStringContext & context);
+
 }