Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add fetchSubmodules to builtins.fetchGit #3166

Merged
merged 17 commits into from
Apr 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions doc/manual/expressions/builtins.xml
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>fetchSubmodules</term>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fetchSubmodules was renamed to submodules.

<listitem>
<para>
A boolean parameter that specifies whether submodules
should be checked out. Defaults to
<literal>false</literal>.
</para>
</listitem>
</varlistentry>
</variablelist>

<example>
Expand Down
1 change: 1 addition & 0 deletions src/libexpr/local.mk
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe
libexpr_LIBS = libutil libstore libnixrust

libexpr_LDFLAGS =

ifneq ($(OS), FreeBSD)
libexpr_LDFLAGS += -ldl
endif
Expand Down
68 changes: 54 additions & 14 deletions src/libexpr/primops/fetchGit.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,25 @@ struct GitInfo
std::string rev;
std::string shortRev;
uint64_t revCount = 0;
bool submodules = false;
};

std::regex revRegex("^[0-9a-fA-F]{40}$");

static bool isNotDotGitDirectory(const Path & path)
{
static const std::regex gitDirRegex("^(?:.*/)?\\.git$");

return not std::regex_match(path, gitDirRegex);
}

GitInfo exportGit(ref<Store> store, const std::string & uri,
std::optional<std::string> ref, std::string rev,
const std::string & name)
const std::string & name, bool fetchSubmodules)
{
GitInfo gitInfo;
gitInfo.submodules = fetchSubmodules;

if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");

Expand All @@ -47,12 +58,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
if (!clean) {

/* This is an unclean working tree. So copy all tracked files. */
GitInfo gitInfo;
gitInfo.rev = "0000000000000000000000000000000000000000";
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);

auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" });
if (fetchSubmodules) {
gitOpts.emplace_back("--recurse-submodules");
}
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
runProgram("git", true, gitOpts), "\0"s);

PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, uri));
Expand Down Expand Up @@ -139,13 +153,13 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
}

// FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo;
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);

printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);

std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev
+ (fetchSubmodules ? "submodules" : "")).to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken

Expand All @@ -165,18 +179,39 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
if (e.errNo != ENOENT) throw;
}

auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});

Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;

// Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root
// repo is cached.
if (fetchSubmodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);

runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
blitz marked this conversation as resolved.
Show resolved Hide resolved
// TODO: the cacheDir repo might lack the ref (it only checks if rev
// exists, see FIXME above) so use a big hammer and fetch everything to
// ensure we get the rev.
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", cacheDir, "refs/*:refs/*" });

runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why add a remote? Is it to resolve submodule urls that are relative paths? Add a comment?

runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });

filter = isNotDotGitDirectory;
} else {
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});

unpackTarfile(*source, tmpDir);
unpackTarfile(*source, tmpDir);
}

gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter));

gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));

Expand All @@ -186,6 +221,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
json["name"] = name;
json["rev"] = gitInfo.rev;
json["revCount"] = gitInfo.revCount;
json["submodules"] = gitInfo.submodules;

writeFile(storeLink, json.dump());

Expand All @@ -198,6 +234,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
std::optional<std::string> ref;
std::string rev;
std::string name = "source";
bool fetchSubmodules = false;
PathSet context;

state.forceValue(*args[0]);
Expand All @@ -216,6 +253,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
rev = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "submodules")
fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
}
Expand All @@ -230,13 +269,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
// whitelist. Ah well.
state.checkURI(url);

auto gitInfo = exportGit(state.store, url, ref, rev, name);
auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules);

state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev);
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev);
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount);
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules);
v.attrs->sort();

if (state.allowedPaths)
Expand Down
97 changes: 97 additions & 0 deletions tests/fetchGitSubmodules.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
source common.sh

set -u

if [[ -z $(type -p git) ]]; then
echo "Git not installed; skipping Git submodule tests"
exit 99
fi

clearStore

rootRepo=$TEST_ROOT/gitSubmodulesRoot
subRepo=$TEST_ROOT/gitSubmodulesSub

rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2

initGitRepo() {
git init $1
git -C $1 config user.email "foobar@example.com"
git -C $1 config user.name "Foobar"
}

addGitContent() {
echo "lorem ipsum" > $1/content
git -C $1 add content
git -C $1 commit -m "Initial commit"
}

initGitRepo $subRepo
addGitContent $subRepo

initGitRepo $rootRepo

git -C $rootRepo submodule init
git -C $rootRepo submodule add $subRepo sub
git -C $rootRepo add sub
git -C $rootRepo commit -m "Add submodule"

rev=$(git -C $rootRepo rev-parse HEAD)

r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")

[[ $r1 == $r2 ]]
[[ $r2 != $r3 ]]

r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath")
r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath")
r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")
r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")

[[ $r1 == $r4 ]]
[[ $r4 == $r5 ]]
[[ $r3 == $r6 ]]
[[ $r6 == $r7 ]]
[[ $r7 == $r8 ]]

have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
[[ $have_submodules == false ]]

have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
[[ $have_submodules == false ]]

have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
[[ $have_submodules == true ]]

pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test blows up if passing ref = "refs/heads/master" to fetchGit with submodules = true:

++ nix eval --raw '(builtins.fetchGit { url = file:///run/user/1000/nix-test/gitSubmodulesRoot; ref = "refs/heads/master"; rev = "96b77ed66ad52090eebe12ff631e79d3c30d6ba1"; submodules = true; }).outPath'
fatal: couldn't find remote ref refs/heads/master
fatal: the remote end hung up unexpectedly
error: program 'git' failed with exit code 128

If using url = /path/to/repo (instead of url = file:///path/to/repo) the error becomes what I see in real life:

++ nix eval --raw '(builtins.fetchGit { url = /run/user/1000/nix-test/gitSubmodulesRoot; ref = "refs/heads/master"; rev = "90ccb6a7d2a59893f0c7283dee98396d3149ea62"; submodules = true; }).outPath'
fatal: Refusing to fetch into current branch refs/heads/master of non-bare repository
error: program 'git' failed with exit code 128
fatal: the remote end hung up unexpectedly

I wonder why file://... makes a difference. But anyway, I experience the error also with git ssh url, so it has nothing to do with local file access.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this fixes it:

--- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -179,7 +179,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
         runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                                  "--", cacheDir, fmt("%s:%s", *ref, *ref) });
+                                  "--", cacheDir, fmt("%s", *ref) });
 
         runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" });
         runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri });

pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath")

# The resulting store path cannot be the same.
[[ $pathWithoutSubmodules != $pathWithSubmodules ]]

# Checking out the same repo with submodules returns in the same store path.
[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]

# Checking out the same repo with submodules returns in the same store path.
[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]]

# The submodules flag is actually honored.
[[ ! -e $pathWithoutSubmodules/sub/content ]]
[[ -e $pathWithSubmodules/sub/content ]]

[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]]

# No .git directory or submodule reference files must be left
test "$(find "$pathWithSubmodules" -name .git)" = ""

# Git repos without submodules can be fetched with submodules = true.
subRev=$(git -C $subRepo rev-parse HEAD)
noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")

blitz marked this conversation as resolved.
Show resolved Hide resolved
[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]
1 change: 1 addition & 0 deletions tests/local.mk
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ nix_tests = \
nar-access.sh \
structured-attrs.sh \
fetchGit.sh \
fetchGitSubmodules.sh \
fetchMercurial.sh \
signing.sh \
run.sh \
Expand Down