Skip to content

Commit

Permalink
Improve support for subflakes
Browse files Browse the repository at this point in the history 
Subflakes are flakes in the same tree, accessed in flake inputs via
relative paths (e.g. `inputs.foo.url = "path:./subdir"`). Previously
these didn't work very well because they would be separately copied to
the store, which is inefficient and makes references to parent
directories tricky or impossible. Furthermore, they had their own NAR
hash in the lock file, which is superfluous since the parent is
already locked.

Now subflakes are accessed via the accessor of the calling flake. This
avoids the unnecessary copy and makes it possible for subflakes to
depend on flakes in a parent directory (so long as they're in the same
tree).

Lock file nodes for relative flake inputs now have a new `parent` field:

  {
    "locked": {
      "path": "./subdir",
      "type": "path"
    },
    "original": {
      "path": "./subdir",
      "type": "path"
    },
    "parent": [
      "foo",
      "bar"
    ]
  }

which denotes that `./subdir` is to be interpreted relative to the
directory of the `bar` input of the `foo` input of the root flake.

Extracted from the lazy-trees branch.
  • Loading branch information
@edolstra
edolstra committed Apr 23, 2024
1 parent aa16530 commit 690281f
Show file tree
Hide file tree
Showing 14 changed files with 332 additions and 120 deletions.
7 changes: 7 additions & 0 deletions src/libexpr/flake/call-flake.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,17 @@ let
(key: node:
let

parentNode = allNodes.${getInputByPath lockFile.root node.parent};

sourceInfo =
if overrides ? ${key}
then
overrides.${key}.sourceInfo
else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/"
then
parentNode.sourceInfo // {
outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path);
}
else
# FIXME: remove obsolete node.info.
fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
Expand Down
160 changes: 114 additions & 46 deletions src/libexpr/flake/flake.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,17 @@ static void expectType(EvalState & state, ValueType type,
}

static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const PosIdx pos,
const std::optional<Path> & baseDir, InputPath lockRootPath);
EvalState & state,
Value * value,
const PosIdx pos,
InputPath lockRootPath);

static FlakeInput parseFlakeInput(EvalState & state,
const std::string & inputName, Value * value, const PosIdx pos,
const std::optional<Path> & baseDir, InputPath lockRootPath)
static FlakeInput parseFlakeInput(
EvalState & state,
const std::string & inputName,
Value * value,
const PosIdx pos,
InputPath lockRootPath)
{
expectType(state, nAttrs, *value, pos);

Expand All @@ -122,7 +127,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
expectType(state, nBool, *attr.value, attr.pos);
input.isFlake = attr.value->boolean();
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath);
input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath);
} else if (attr.name == sFollows) {
expectType(state, nString, *attr.value, attr.pos);
auto follows(parseInputPath(attr.value->c_str()));
Expand Down Expand Up @@ -173,7 +178,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
if (!attrs.empty())
throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]);
if (url)
input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake);
input.ref = parseFlakeRef(*url, {}, true, input.isFlake, true);
}

if (!input.follows && !input.ref)
Expand All @@ -183,8 +188,10 @@ static FlakeInput parseFlakeInput(EvalState & state,
}

static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const PosIdx pos,
const std::optional<Path> & baseDir, InputPath lockRootPath)
EvalState & state,
Value * value,
const PosIdx pos,
InputPath lockRootPath)
{
std::map<FlakeId, FlakeInput> inputs;

Expand All @@ -196,7 +203,6 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
state.symbols[inputAttr.name],
inputAttr.value,
inputAttr.pos,
baseDir,
lockRootPath));
}

Expand Down Expand Up @@ -232,7 +238,7 @@ static Flake readFlake(
auto sInputs = state.symbols.create("inputs");

if (auto inputs = vInfo.attrs()->get(sInputs))
flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME
flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath);

auto sOutputs = state.symbols.create("outputs");

Expand Down Expand Up @@ -366,13 +372,29 @@ LockedFlake lockFlake(

debug("old lock file: %s", oldLockFile);

std::map<InputPath, FlakeInput> overrides;
struct Override
{
FlakeInput input;
SourcePath sourcePath;
std::optional<InputPath> parentPath; // FIXME: rename to inputPathPrefix?
};

std::map<InputPath, Override> overrides;
std::set<InputPath> explicitCliOverrides;
std::set<InputPath> overridesUsed, updatesUsed;
std::map<ref<Node>, SourcePath> nodePaths;

for (auto & i : lockFlags.inputOverrides) {
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
overrides.emplace(
i.first,
Override {
.input = FlakeInput { .ref = i.second },
/* Note: any relative overrides
(e.g. `--override-input B/C "path:./foo/bar"`)
are interpreted relative to the top-level
flake. */
.sourcePath = flake.path,
});
explicitCliOverrides.insert(i.first);
}

Expand All @@ -386,7 +408,7 @@ LockedFlake lockFlake(
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode,
const InputPath & lockRootPath,
const Path & parentPath,
const SourcePath & sourcePath,
bool trustLock)>
computeLocks;

Expand All @@ -402,7 +424,8 @@ LockedFlake lockFlake(
copied. */
std::shared_ptr<const Node> oldNode,
const InputPath & lockRootPath,
const Path & parentPath,
/* The source path of this node's flake. */
const SourcePath & sourcePath,
bool trustLock)
{
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
Expand All @@ -414,7 +437,12 @@ LockedFlake lockFlake(
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
inputPath.push_back(idOverride);
overrides.insert_or_assign(inputPath, inputOverride);
overrides.emplace(inputPath,
Override {
.input = inputOverride,
.sourcePath = sourcePath,
.parentPath = inputPathPrefix // FIXME: should this be inputPath?
});
}
}

Expand Down Expand Up @@ -446,13 +474,18 @@ LockedFlake lockFlake(
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
bool hasCliOverride = explicitCliOverrides.contains(inputPath);
if (hasOverride) {
if (hasOverride)
overridesUsed.insert(inputPath);
// Respect the “flakeness” of the input even if we
// override it
i->second.isFlake = input2.isFlake;
}
auto & input = hasOverride ? i->second : input2;
auto input = hasOverride ? i->second.input : input2;

/* Resolve relative 'path:' inputs relative to
the source path of the overrider. */
auto overridenSourcePath = hasOverride ? i->second.sourcePath : sourcePath;

/* Respect the "flakeness" of the input even if we
override it. */
if (hasOverride)
input.isFlake = input2.isFlake;

/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
Expand All @@ -468,6 +501,33 @@ LockedFlake lockFlake(

assert(input.ref);

auto overridenParentPath =
input.ref->input.isRelative()
? std::optional<InputPath>(hasOverride ? i->second.parentPath : inputPathPrefix)
: std::nullopt;

auto resolveRelativePath = [&]() -> std::optional<SourcePath>
{
if (auto relativePath = input.ref->input.isRelative()) {
return SourcePath {
overridenSourcePath.accessor,
CanonPath(*relativePath, overridenSourcePath.path.parent().value())
};
} else
return std::nullopt;
};

/* Get the input flake, resolve 'path:./...'
flakerefs relative to the parent flake. */
auto getInputFlake = [&]()
{
if (auto resolvedPath = resolveRelativePath()) {
return readFlake(state, *input.ref, *input.ref, *input.ref, *resolvedPath, inputPath);
} else {
return getFlake(state, *input.ref, useRegistries, flakeCache, inputPath);
}
};

/* Do we have an entry in the existing lock file?
And the input is not in updateInputs? */
std::shared_ptr<LockedNode> oldLock;
Expand All @@ -481,6 +541,7 @@ LockedFlake lockFlake(

if (oldLock
&& oldLock->originalRef == *input.ref
&& oldLock->parentPath == overridenParentPath
&& !hasCliOverride)
{
debug("keeping existing input '%s'", inputPathS);
Expand All @@ -489,7 +550,10 @@ LockedFlake lockFlake(
didn't change and there is no override from a
higher level flake. */
auto childNode = make_ref<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
oldLock->lockedRef,
oldLock->originalRef,
oldLock->isFlake,
oldLock->parentPath);

node->inputs.insert_or_assign(id, childNode);

Expand Down Expand Up @@ -541,19 +605,25 @@ LockedFlake lockFlake(
}

if (mustRefetch) {
auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath);
auto inputFlake = getInputFlake();
nodePaths.emplace(childNode, inputFlake.path.parent());
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false);
} else {
computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true);
// FIXME: sourcePath is wrong here, we
// should pass a lambda that lazily
// fetches the parent flake if needed
// (i.e. getInputFlake()).
computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true);
}

} else {
/* We need to create a new lock file entry. So fetch
this input. */
debug("creating new input '%s'", inputPathS);

if (!lockFlags.allowUnlocked && !input.ref->input.isLocked())
if (!lockFlags.allowUnlocked
&& !input.ref->input.isLocked()
&& !input.ref->input.isRelative())
throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS);

/* Note: in case of an --override-input, we use
Expand All @@ -566,17 +636,9 @@ LockedFlake lockFlake(
auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;

if (input.isFlake) {
Path localPath = parentPath;
FlakeRef localRef = *input.ref;

// If this input is a path, recurse it down.
// This allows us to resolve path inputs relative to the current flake.
if (localRef.input.getType() == "path")
localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
auto inputFlake = getInputFlake();

auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath);

auto childNode = make_ref<LockedNode>(inputFlake.lockedRef, ref);
auto childNode = make_ref<LockedNode>(inputFlake.lockedRef, ref, true, overridenParentPath);

node->inputs.insert_or_assign(id, childNode);

Expand All @@ -598,17 +660,26 @@ LockedFlake lockFlake(
? std::dynamic_pointer_cast<const Node>(oldLock)
: readLockFile(inputFlake.lockFilePath()).root.get_ptr(),
oldLock ? lockRootPath : inputPath,
localPath,
inputFlake.path,
false);
}

else {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistries, flakeCache);
auto [path, lockedRef] = [&]() -> std::tuple<SourcePath, FlakeRef>
{
// Handle non-flake 'path:./...' inputs.
if (auto resolvedPath = resolveRelativePath()) {
return {*resolvedPath, *input.ref};
} else {
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistries, flakeCache);
return {state.rootPath(state.store->toRealPath(storePath)), lockedRef};
}
}();

auto childNode = make_ref<LockedNode>(lockedRef, ref, false);
auto childNode = make_ref<LockedNode>(lockedRef, ref, false, overridenParentPath);

nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath)));
nodePaths.emplace(childNode, path);

node->inputs.insert_or_assign(id, childNode);
}
Expand All @@ -621,9 +692,6 @@ LockedFlake lockFlake(
}
};

// Bring in the current ref for relative path resolution if we have it
auto parentPath = flake.path.parent().path.abs();

nodePaths.emplace(newLockFile.root, flake.path.parent());

computeLocks(
Expand All @@ -632,7 +700,7 @@ LockedFlake lockFlake(
{},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(),
{},
parentPath,
flake.path,
false);

for (auto & i : lockFlags.inputOverrides)
Expand Down

0 comments on commit 690281f

Please sign in to comment.