New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
CA derivations that depend on other CA derivations #3958
Changes from 1 commit
b0b59fd
59979e7
e0b0e18
8eb73a8
48e8828
421ed52
02e0001
4db0010
975a47f
b836662
4409530
aad4abc
075d399
98dfd75
c5cceba
7fdbb37
2741fff
b7df353
3786a80
67cc356
10202bb
80e335b
00135e1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -984,6 +984,8 @@ class DerivationGoal : public Goal | |
void tryLocalBuild(); | ||
void buildDone(); | ||
|
||
void resolvedFinished(); | ||
|
||
/* Is the build hook willing to perform the build? */ | ||
HookReply tryBuildHook(); | ||
|
||
|
@@ -1451,8 +1453,39 @@ void DerivationGoal::inputsRealised() | |
/* Determine the full set of input paths. */ | ||
|
||
/* First, the input derivations. */ | ||
if (useDerivation) | ||
for (auto & [depDrvPath, wantedDepOutputs] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { | ||
if (useDerivation) { | ||
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); | ||
|
||
if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { | ||
/* We are be able to resolve this derivation based on the | ||
now-known results of dependencies. If so, we become a stub goal | ||
aliasing that resolved derivation goal */ | ||
Derivation drvResolved { fullDrv.resolve(worker.store) }; | ||
|
||
auto pathResolved = writeDerivation(worker.store, drvResolved); | ||
/* Add to memotable to speed up downstream goal's queries with the | ||
original derivation. */ | ||
drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); | ||
|
||
auto msg = fmt("Resolved derivation: '%s' -> '%s'", | ||
worker.store.printStorePath(drvPath), | ||
worker.store.printStorePath(pathResolved)); | ||
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg, | ||
Logger::Fields { | ||
worker.store.printStorePath(drvPath), | ||
worker.store.printStorePath(pathResolved), | ||
}); | ||
|
||
auto resolvedGoal = worker.makeDerivationGoal( | ||
pathResolved, wantedOutputs, | ||
buildMode == bmRepair ? bmRepair : bmNormal); | ||
Ericson2314 marked this conversation as resolved.
Show resolved
Hide resolved
Ericson2314 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
addWaitee(resolvedGoal); | ||
|
||
state = &DerivationGoal::resolvedFinished; | ||
return; | ||
} | ||
|
||
for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { | ||
/* Add the relevant output closures of the input derivation | ||
`i' as input paths. Only add the closures of output paths | ||
that are specified as inputs. */ | ||
|
@@ -1472,6 +1505,7 @@ void DerivationGoal::inputsRealised() | |
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); | ||
} | ||
} | ||
} | ||
|
||
/* Second, the input sources. */ | ||
worker.store.computeFSClosure(drv->inputSrcs, inputPaths); | ||
|
@@ -1893,6 +1927,9 @@ void DerivationGoal::buildDone() | |
done(BuildResult::Built); | ||
} | ||
|
||
void DerivationGoal::resolvedFinished() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This function seems a bit unnecessary. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Well many derivation goals could resolve to the same derivation, Trying to make the resolve derivations track all those to call |
||
done(BuildResult::Built); | ||
} | ||
|
||
HookReply DerivationGoal::tryBuildHook() | ||
{ | ||
|
@@ -2065,7 +2102,7 @@ void linkOrCopy(const Path & from, const Path & to) | |
file (e.g. 32000 of ext3), which is quite possible after a | ||
'nix-store --optimise'. FIXME: actually, why don't we just | ||
bind-mount in this case? | ||
|
||
It can also fail with EPERM in BeegFS v7 and earlier versions | ||
which don't allow hard-links to other directories */ | ||
if (errno != EMLINK && errno != EPERM) | ||
|
@@ -4248,10 +4285,14 @@ void DerivationGoal::registerOutputs() | |
{ | ||
ValidPathInfos infos2; | ||
for (auto & [outputName, newInfo] : infos) { | ||
/* FIXME: we will want to track this mapping in the DB whether or | ||
not we have a drv file. */ | ||
if (useDerivation) | ||
worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path); | ||
else { | ||
/* Once a floating CA derivations reaches this point, it must | ||
already be resolved, drvPath the basic derivation path, and | ||
a file existsing at that path for sake of the DB's foreign key. */ | ||
Ericson2314 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
assert(drv->type() != DerivationType::CAFloating); | ||
} | ||
infos2.push_back(newInfo); | ||
} | ||
worker.store.registerValidPaths(infos2); | ||
|
Original file line number | Diff line number | Diff line change | ||||||
---|---|---|---|---|---|---|---|---|
|
@@ -672,4 +672,57 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath | |||||||
return "/" + hashString(htSHA256, clearText).to_string(Base32, false); | ||||||||
} | ||||||||
|
||||||||
|
||||||||
// N.B. Outputs are left unchanged | ||||||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { | ||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Style nitpick. |
||||||||
|
||||||||
debug("Rewriting the derivation"); | ||||||||
|
||||||||
for (auto &rewrite: rewrites) { | ||||||||
debug("rewriting %s as %s", rewrite.first, rewrite.second); | ||||||||
} | ||||||||
|
||||||||
drv.builder = rewriteStrings(drv.builder, rewrites); | ||||||||
for (auto & arg: drv.args) { | ||||||||
arg = rewriteStrings(arg, rewrites); | ||||||||
} | ||||||||
|
||||||||
StringPairs newEnv; | ||||||||
for (auto & envVar: drv.env) { | ||||||||
auto envName = rewriteStrings(envVar.first, rewrites); | ||||||||
auto envValue = rewriteStrings(envVar.second, rewrites); | ||||||||
newEnv.emplace(envName, envValue); | ||||||||
} | ||||||||
drv.env = newEnv; | ||||||||
} | ||||||||
|
||||||||
|
||||||||
Sync<DrvPathResolutions> drvPathResolutions; | ||||||||
|
||||||||
BasicDerivation Derivation::resolve(Store & store) { | ||||||||
BasicDerivation resolved { *this }; | ||||||||
|
||||||||
// Input paths that we'll want to rewrite in the derivation | ||||||||
StringMap inputRewrites; | ||||||||
|
||||||||
for (auto & input : inputDrvs) { | ||||||||
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); | ||||||||
StringSet newOutputNames; | ||||||||
for (auto & outputName : input.second) { | ||||||||
auto actualPathOpt = inputDrvOutputs.at(outputName); | ||||||||
if (!actualPathOpt) | ||||||||
throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first)); | ||||||||
auto actualPath = *actualPathOpt; | ||||||||
inputRewrites.emplace( | ||||||||
downstreamPlaceholder(store, input.first, outputName), | ||||||||
store.printStorePath(actualPath)); | ||||||||
resolved.inputSrcs.insert(std::move(actualPath)); | ||||||||
} | ||||||||
} | ||||||||
|
||||||||
rewriteDerivation(store, resolved, inputRewrites); | ||||||||
|
||||||||
return resolved; | ||||||||
} | ||||||||
|
||||||||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -803,13 +803,36 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) | |
} | ||
|
||
|
||
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path) | ||
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) | ||
{ | ||
auto path = path_; | ||
std::map<std::string, std::optional<StorePath>> outputs; | ||
BasicDerivation drv = readDerivation(path); | ||
Derivation drv = readDerivation(path); | ||
for (auto & [outName, _] : drv.outputs) { | ||
outputs.insert_or_assign(outName, std::nullopt); | ||
} | ||
bool haveCached = false; | ||
{ | ||
auto resolutions = drvPathResolutions.lock(); | ||
auto resolvedPathOptIter = resolutions->find(path); | ||
if (resolvedPathOptIter != resolutions->end()) { | ||
auto & [_, resolvedPathOpt] = *resolvedPathOptIter; | ||
if (resolvedPathOpt) | ||
path = *resolvedPathOpt; | ||
haveCached = true; | ||
} | ||
} | ||
/* can't just use else-if instead of `!haveCached` because we need to unlock | ||
`drvPathResolutions` before it is locked in `Derivation::resolve`. */ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe a silly question, but what is preventing us from reusing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This function and There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe there should be internal variants that take a reference to the locked |
||
if (!haveCached && drv.type() == DerivationType::CAFloating) { | ||
/* Resolve drv and use that path instead. */ | ||
auto pathResolved = writeDerivation(*this, drv.resolve(*this)); | ||
Ericson2314 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
/* Store in memo table. */ | ||
/* FIXME: memo logic should not be local-store specific, should have | ||
wrapper-method instead. */ | ||
drvPathResolutions.lock()->insert_or_assign(path, pathResolved); | ||
path = std::move(pathResolved); | ||
} | ||
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() { | ||
auto state(_state.lock()); | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Note we are writing down the resolved derivation so we can link the deriver to the CA output in the sqlite db.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we need that? It doubles the number of .drv files on disk, while we'd like to get rid of them eventually...
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Well it only doubles them for floating CA derivations. Not doing this means changing the SQL schema, and I think there's a few such change's will end up wanting to make, so I rather save that for later to think about them all at once.