Track the dependencies of derivation outputs

flake.nix

@@ -115,6 +115,11 @@
       # 'nix.perl-bindings' packages.
       overlay = final: prev: {
 
+        # An older version of Nix to test against when using the daemon.
+        # Currently using `nixUnstable` as the stable one doesn't respect
+        # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
+        mainstream-nix = prev.nixUnstable;
+
         nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
@@ -123,6 +128,8 @@
 
           VERSION_SUFFIX = versionSuffix;
 
+          OUTER_NIX = mainstream-nix;
+
           outputs = [ "out" "dev" "doc" ];
 
           nativeBuildInputs = nativeBuildDeps;
@@ -484,6 +491,8 @@
         stdenv.mkDerivation {
           name = "nix";
 
+          OUTER_NIX = mainstream-nix;
+
           outputs = [ "out" "dev" "doc" ];
 
           nativeBuildInputs = nativeBuildDeps;

mk/run_test.sh

@@ -14,7 +14,7 @@ if [ -t 1 ]; then
     yellow=""
     normal=""
 fi
-(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
 log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
 status=$?
 if [ $status -eq 0 ]; then

mk/tests.mk

@@ -8,7 +8,7 @@ define run-install-test
 
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
+	@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
 
 endef
 

src/libstore/binary-cache-store.cc

@@ -434,7 +434,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     if (!repair && isValidPath(path))
         return path;
 
-    auto source = StringSource { s };
+    StringSink sink;
+    dumpString(s, sink);
+    auto source = StringSource { *sink.s };
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info { path, nar.first };
         info.narSize = nar.second;
@@ -444,6 +446,33 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     })->path;
 }
 
+std::optional<const DrvOutputInfo> BinaryCacheStore::queryDrvOutputInfo(const DrvOutputId & id)
+{
+    auto outputInfoFilePath =
+        "/drvOutputs/" + std::string(id.drvPath.hashPart()) + "!" + id.outputName + ".doi";
+    auto rawOutputInfo = getFile(outputInfoFilePath);
+
+    if (rawOutputInfo) {
+        return { DrvOutputInfo::parse(*rawOutputInfo, outputInfoFilePath) };
+    } else {
+        return std::nullopt;
+    }
+}
+
+std::optional<StorePath> BinaryCacheStore::queryOutputPathOf(
+    const StorePath& drvPath,
+    const std::string& outputName) {
+    if(auto outputInfo = queryDrvOutputInfo(DrvOutputId{drvPath, outputName}))
+        return {outputInfo->outPath};
+    return std::nullopt;
+}
+
+void BinaryCacheStore::registerDrvOutput(const DrvOutputId & id, const DrvOutputInfo & info)
+{
+    auto filePath = "/drvOutputs/" + std::string(id.drvPath.hashPart()) + "!" + id.outputName + ".doi";
+    upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath");
+}
+
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
 {
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);

src/libstore/binary-cache-store.hh

@@ -99,6 +99,12 @@ public:
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
 
+    void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override;
+
+    std::optional<StorePath> queryOutputPathOf(const StorePath & drvPath, const std::string & outputName) override;
+
+    std::optional<const DrvOutputInfo> queryDrvOutputInfo(const DrvOutputId &) override;
+
     void narFromPath(const StorePath & path, Sink & sink) override;
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

src/libstore/build/derivation-goal.cc

@@ -13,6 +13,7 @@
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
+#include "drv-output-info.hh"
 
 #include <regex>
 #include <queue>
@@ -231,7 +232,7 @@ void DerivationGoal::getDerivation()
         return;
     }
 
-    addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath)));
+    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
 
     state = &DerivationGoal::loadDerivation;
 }
@@ -297,17 +298,22 @@ void DerivationGoal::haveDerivation()
        through substitutes.  If that doesn't work, we'll build
        them. */
     if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
-        for (auto & [_, status] : initialOutputs) {
+        for (auto & [outputName, status] : initialOutputs) {
             if (!status.wanted) continue;
-            if (!status.known) {
-                warn("do not know how to query for unknown floating content-addressed derivation output yet");
-                /* Nothing to wait for; tail call */
-                return DerivationGoal::gaveUpOnSubstitution();
-            }
-            addWaitee(upcast_goal(worker.makeSubstitutionGoal(
-                status.known->path,
-                buildMode == bmRepair ? Repair : NoRepair,
-                getDerivationCA(*drv))));
+            if (!status.known)
+                addWaitee(
+                    upcast_goal(
+                        worker.makeDrvOutputSubstitutionGoal(
+                            DrvOutputId{drvPath, outputName},
+                            buildMode == bmRepair ? Repair : NoRepair
+                        )
+                    )
+                );
+            else
+                addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
+                    status.known->path,
+                    buildMode == bmRepair ? Repair : NoRepair,
+                    getDerivationCA(*drv))));
         }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -388,7 +394,7 @@ void DerivationGoal::gaveUpOnSubstitution()
         if (!settings.useSubstitutes)
             throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
                 worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        addWaitee(upcast_goal(worker.makeSubstitutionGoal(i)));
+        addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i)));
     }
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -442,7 +448,7 @@ void DerivationGoal::repairClosure()
         });
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
-            addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
+            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
         else
             addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
     }
@@ -493,31 +499,32 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if ((!fullDrv.inputDrvs.empty() &&
-             fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) {
+        if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+            ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
+            || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
             /* We are be able to resolve this derivation based on the
                now-known results of dependencies. If so, we become a stub goal
                aliasing that resolved derivation goal */
             std::optional attempt = fullDrv.tryResolve(worker.store);
             assert(attempt);
             Derivation drvResolved { *std::move(attempt) };
 
-            auto pathResolved = writeDerivation(worker.store, drvResolved);
+            resolvedDrvPath = writeDerivation(worker.store, drvResolved);
             /* Add to memotable to speed up downstream goal's queries with the
                original derivation. */
-            drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
+            drvPathResolutions.lock()->insert_or_assign(drvPath, resolvedDrvPath);
 
             auto msg = fmt("Resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
-                worker.store.printStorePath(pathResolved));
+                worker.store.printStorePath(*resolvedDrvPath));
             act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
                 Logger::Fields {
                        worker.store.printStorePath(drvPath),
-                       worker.store.printStorePath(pathResolved),
+                       worker.store.printStorePath(*resolvedDrvPath),
                    });
 
             auto resolvedGoal = worker.makeDerivationGoal(
-                pathResolved, wantedOutputs, buildMode);
+                *resolvedDrvPath, wantedOutputs, buildMode);
             addWaitee(resolvedGoal);
 
             state = &DerivationGoal::resolvedFinished;
@@ -904,8 +911,14 @@ void DerivationGoal::buildDone()
             }
             std::map<std::string, std::string> hookEnvironment = getEnv();
 
+            std::set<std::string> rawDrvOutputs;
+            for (auto [outputName, _] : drv->outputs) {
+                rawDrvOutputs.insert((worker.store.storeDir + "/").append(DrvOutputId{ drvPath, outputName}.to_string()));
+            }
+
             hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
             hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
+            hookEnvironment.emplace("DRV_OUTPUTS", chomp(concatStringsSep(" ", rawDrvOutputs)));
 
             RunOptions opts(settings.postBuildHook, {});
             opts.environment = hookEnvironment;
@@ -1004,6 +1017,18 @@ void DerivationGoal::buildDone()
 }
 
 void DerivationGoal::resolvedFinished() {
+    debug("Resolving the original derivation");
+    assert(resolvedDrvPath);
+    auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
+
+    std::map<std::string, StorePath> outputPaths;
+    auto partialOutputMap = worker.store.queryPartialDerivationOutputMap(*resolvedDrvPath);
+    for (auto& [name, maybeOutPath]: partialOutputMap)
+        if (maybeOutPath)
+            outputPaths.insert({name, *maybeOutPath});
+
+    nix::registerOutputs(worker.store, drvPath, fullDrv, outputPaths);
+
     done(BuildResult::Built);
 }
 
@@ -2096,6 +2121,20 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
         /* Nothing to be done; 'path' must already be valid. */
     }
 
+    void registerDrvOutput(const DrvOutputId & id, const DrvOutputInfo & output) override
+    {
+        if (!goal.isAllowed(id.drvPath))
+            throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(id.drvPath));
+        next->registerDrvOutput(id, output);
+    }
+
+    std::optional<const DrvOutputInfo> queryDrvOutputInfo(const DrvOutputId & id) override
+    {
+        if (!goal.isAllowed(id.drvPath))
+            throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath));
+        return next->queryDrvOutputInfo(id);
+    }
+
     void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
@@ -2859,7 +2898,6 @@ void DerivationGoal::runChild()
     }
 }
 
-
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
@@ -3121,6 +3159,20 @@ void DerivationGoal::registerOutputs()
             newInfo0.references = refs.second;
             if (refs.first)
                 newInfo0.references.insert(newInfo0.path);
+            if (scratchPath != newInfo0.path) {
+                // Also rewrite the output path
+                auto source = sinkToSource([&](Sink & nextSink) {
+                    StringSink sink;
+                    dumpPath(actualPath, sink);
+                    RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
+                    rsink2((unsigned char *) sink.s->data(), sink.s->size());
+                    rsink2.flush();
+                });
+                Path tmpPath = actualPath + ".tmp";
+                restorePath(tmpPath, *source);
+                deletePath(actualPath);
+                movePath(tmpPath, actualPath);
+            }
 
             assert(newInfo0.ca);
             return newInfo0;
@@ -3377,9 +3429,19 @@ void DerivationGoal::registerOutputs()
         drvPathResolved = writeDerivation(worker.store, drv2);
     }
 
-    if (useDerivation || isCaFloating)
-        for (auto & [outputName, newInfo] : infos)
-            worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
+    if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+        useDerivation) {
+        auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
+        std::map<std::string, StorePath> outputPaths;
+        for (auto& [outputName, info] : infos)
+            outputPaths.insert({outputName, info.path});
+        nix::registerOutputs(
+            worker.store,
+            drvPath,
+            fullDrv,
+            outputPaths
+        );
+    }
 }
 
 

src/libstore/build/derivation-goal.hh

@@ -64,6 +64,11 @@ struct DerivationGoal : public Goal
 
     std::unique_ptr<ParsedDerivation> parsedDrv;
 
+    /* Path to the corresponding resolved derivation,
+     *  if we're not already resolved
+     */
+    std::optional<StorePath> resolvedDrvPath;
+
     /* The remainder is state held during the build. */
 
     /* Locks on (fixed) output paths. */