-
-
Notifications
You must be signed in to change notification settings - Fork 1.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add fetchSubmodules to builtins.fetchGit #3166
Changes from all commits
f58604a
c2a24c2
ea861be
c8d33de
c846abb
435366e
6c00a95
cc4fe97
002a3a9
587e259
6864ad7
369fffd
be84049
b306b70
cc522d0
f686efe
40c023e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,14 +22,25 @@ struct GitInfo | |
std::string rev; | ||
std::string shortRev; | ||
uint64_t revCount = 0; | ||
bool submodules = false; | ||
}; | ||
|
||
std::regex revRegex("^[0-9a-fA-F]{40}$"); | ||
|
||
static bool isNotDotGitDirectory(const Path & path) | ||
{ | ||
static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); | ||
|
||
return not std::regex_match(path, gitDirRegex); | ||
} | ||
|
||
GitInfo exportGit(ref<Store> store, const std::string & uri, | ||
std::optional<std::string> ref, std::string rev, | ||
const std::string & name) | ||
const std::string & name, bool fetchSubmodules) | ||
{ | ||
GitInfo gitInfo; | ||
gitInfo.submodules = fetchSubmodules; | ||
|
||
if (evalSettings.pureEval && rev == "") | ||
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); | ||
|
||
|
@@ -47,12 +58,15 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, | |
if (!clean) { | ||
|
||
/* This is an unclean working tree. So copy all tracked files. */ | ||
GitInfo gitInfo; | ||
gitInfo.rev = "0000000000000000000000000000000000000000"; | ||
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); | ||
|
||
auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" }); | ||
if (fetchSubmodules) { | ||
gitOpts.emplace_back("--recurse-submodules"); | ||
} | ||
auto files = tokenizeString<std::set<std::string>>( | ||
runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); | ||
runProgram("git", true, gitOpts), "\0"s); | ||
|
||
PathFilter filter = [&](const Path & p) -> bool { | ||
assert(hasPrefix(p, uri)); | ||
|
@@ -139,13 +153,13 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, | |
} | ||
|
||
// FIXME: check whether rev is an ancestor of ref. | ||
GitInfo gitInfo; | ||
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); | ||
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); | ||
|
||
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); | ||
|
||
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); | ||
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev | ||
+ (fetchSubmodules ? "submodules" : "")).to_string(Base32, false); | ||
Path storeLink = cacheDir + "/" + storeLinkName + ".link"; | ||
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken | ||
|
||
|
@@ -165,18 +179,39 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, | |
if (e.errNo != ENOENT) throw; | ||
} | ||
|
||
auto source = sinkToSource([&](Sink & sink) { | ||
RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); | ||
gitOptions.standardOut = &sink; | ||
runProgram2(gitOptions); | ||
}); | ||
|
||
Path tmpDir = createTempDir(); | ||
AutoDelete delTmpDir(tmpDir, true); | ||
PathFilter filter = defaultPathFilter; | ||
|
||
// Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root | ||
// repo is cached. | ||
if (fetchSubmodules) { | ||
Path tmpGitDir = createTempDir(); | ||
AutoDelete delTmpGitDir(tmpGitDir, true); | ||
|
||
runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); | ||
blitz marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// TODO: the cacheDir repo might lack the ref (it only checks if rev | ||
// exists, see FIXME above) so use a big hammer and fetch everything to | ||
// ensure we get the rev. | ||
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", | ||
"--update-head-ok", "--", cacheDir, "refs/*:refs/*" }); | ||
|
||
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); | ||
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why add a remote? Is it to resolve submodule urls that are relative paths? Add a comment? |
||
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); | ||
|
||
filter = isNotDotGitDirectory; | ||
} else { | ||
auto source = sinkToSource([&](Sink & sink) { | ||
RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); | ||
gitOptions.standardOut = &sink; | ||
runProgram2(gitOptions); | ||
}); | ||
|
||
unpackTarfile(*source, tmpDir); | ||
unpackTarfile(*source, tmpDir); | ||
} | ||
|
||
gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); | ||
gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter)); | ||
|
||
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); | ||
|
||
|
@@ -186,6 +221,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, | |
json["name"] = name; | ||
json["rev"] = gitInfo.rev; | ||
json["revCount"] = gitInfo.revCount; | ||
json["submodules"] = gitInfo.submodules; | ||
|
||
writeFile(storeLink, json.dump()); | ||
|
||
|
@@ -198,6 +234,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va | |
std::optional<std::string> ref; | ||
std::string rev; | ||
std::string name = "source"; | ||
bool fetchSubmodules = false; | ||
PathSet context; | ||
|
||
state.forceValue(*args[0]); | ||
|
@@ -216,6 +253,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va | |
rev = state.forceStringNoCtx(*attr.value, *attr.pos); | ||
else if (n == "name") | ||
name = state.forceStringNoCtx(*attr.value, *attr.pos); | ||
else if (n == "submodules") | ||
fetchSubmodules = state.forceBool(*attr.value, *attr.pos); | ||
else | ||
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); | ||
} | ||
|
@@ -230,13 +269,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va | |
// whitelist. Ah well. | ||
state.checkURI(url); | ||
|
||
auto gitInfo = exportGit(state.store, url, ref, rev, name); | ||
auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules); | ||
|
||
state.mkAttrs(v, 8); | ||
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); | ||
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); | ||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); | ||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); | ||
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules); | ||
v.attrs->sort(); | ||
|
||
if (state.allowedPaths) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
source common.sh | ||
|
||
set -u | ||
|
||
if [[ -z $(type -p git) ]]; then | ||
echo "Git not installed; skipping Git submodule tests" | ||
exit 99 | ||
fi | ||
|
||
clearStore | ||
|
||
rootRepo=$TEST_ROOT/gitSubmodulesRoot | ||
subRepo=$TEST_ROOT/gitSubmodulesSub | ||
|
||
rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2 | ||
|
||
initGitRepo() { | ||
git init $1 | ||
git -C $1 config user.email "foobar@example.com" | ||
git -C $1 config user.name "Foobar" | ||
} | ||
|
||
addGitContent() { | ||
echo "lorem ipsum" > $1/content | ||
git -C $1 add content | ||
git -C $1 commit -m "Initial commit" | ||
} | ||
|
||
initGitRepo $subRepo | ||
addGitContent $subRepo | ||
|
||
initGitRepo $rootRepo | ||
|
||
git -C $rootRepo submodule init | ||
git -C $rootRepo submodule add $subRepo sub | ||
git -C $rootRepo add sub | ||
git -C $rootRepo commit -m "Add submodule" | ||
|
||
rev=$(git -C $rootRepo rev-parse HEAD) | ||
|
||
r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") | ||
r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") | ||
r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") | ||
|
||
[[ $r1 == $r2 ]] | ||
[[ $r2 != $r3 ]] | ||
|
||
r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath") | ||
r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath") | ||
r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") | ||
r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") | ||
r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") | ||
|
||
[[ $r1 == $r4 ]] | ||
[[ $r4 == $r5 ]] | ||
[[ $r3 == $r6 ]] | ||
[[ $r6 == $r7 ]] | ||
[[ $r7 == $r8 ]] | ||
|
||
have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") | ||
[[ $have_submodules == false ]] | ||
|
||
have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") | ||
[[ $have_submodules == false ]] | ||
|
||
have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") | ||
[[ $have_submodules == true ]] | ||
|
||
pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") | ||
pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The test blows up if passing
If using
I wonder why There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this fixes it: --- a/src/libexpr/primops/fetchGit.cc
+++ b/src/libexpr/primops/fetchGit.cc
@@ -179,7 +179,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
- "--", cacheDir, fmt("%s:%s", *ref, *ref) });
+ "--", cacheDir, fmt("%s", *ref) });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); |
||
pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") | ||
pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") | ||
|
||
# The resulting store path cannot be the same. | ||
[[ $pathWithoutSubmodules != $pathWithSubmodules ]] | ||
|
||
# Checking out the same repo with submodules returns in the same store path. | ||
[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] | ||
|
||
# Checking out the same repo with submodules returns in the same store path. | ||
[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] | ||
|
||
# The submodules flag is actually honored. | ||
[[ ! -e $pathWithoutSubmodules/sub/content ]] | ||
[[ -e $pathWithSubmodules/sub/content ]] | ||
|
||
[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] | ||
|
||
# No .git directory or submodule reference files must be left | ||
test "$(find "$pathWithSubmodules" -name .git)" = "" | ||
|
||
# Git repos without submodules can be fetched with submodules = true. | ||
subRev=$(git -C $subRepo rev-parse HEAD) | ||
noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") | ||
noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") | ||
|
||
blitz marked this conversation as resolved.
Show resolved
Hide resolved
|
||
[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
fetchSubmodules was renamed to submodules.