@@ -0,0 +1,45 @@
#pragma once

#include "util.hh"
#include "path.hh"
#include "eval.hh"

#include <optional>

namespace nix {

struct Buildable
{
std::optional<StorePath> drvPath;
std::map<std::string, StorePath> outputs;
};

typedef std::vector<Buildable> Buildables;

struct Installable
{
virtual ~Installable() { }

virtual std::string what() = 0;

virtual Buildables toBuildables()
{
throw Error("argument '%s' cannot be built", what());
}

Buildable toBuildable();

virtual std::pair<Value *, Pos> toValue(EvalState & state)
{
throw Error("argument '%s' cannot be evaluated", what());
}

/* Return a value only if this installable is a store path or a
symlink to it. */
virtual std::optional<StorePath> getStorePath()
{
return {};
}
};

}
@@ -15,9 +15,9 @@ nix_SOURCES := \
$(wildcard src/nix-prefetch-url/*.cc) \
$(wildcard src/nix-store/*.cc) \

nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain
nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain

nix_LIBS = libexpr libmain libstore libutil libnixrust
nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust

nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system

@@ -8,6 +8,7 @@
#include "fs-accessor.hh"
#include "progress-bar.hh"
#include "affinity.hh"
#include "eval.hh"

#if __linux__
#include <sys/mount.h>
@@ -19,11 +20,46 @@ using namespace nix;

std::string chrootHelperName = "__run_in_chroot";

struct CmdRun : InstallablesCommand
struct RunCommon : virtual Command
{
void runProgram(ref<Store> store,
const std::string & program,
const Strings & args)
{
stopProgressBar();

restoreSignals();

restoreAffinity();

/* If this is a diverted store (i.e. its "logical" location
(typically /nix/store) differs from its "physical" location
(e.g. /home/eelco/nix/store), then run the command in a
chroot. For non-root users, this requires running it in new
mount and user namespaces. Unfortunately,
unshare(CLONE_NEWUSER) doesn't work in a multithreaded
program (which "nix" is), so we exec() a single-threaded
helper program (chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>();

if (store2 && store->storeDir != store2->realStoreDir) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
for (auto & arg : args) helperArgs.push_back(arg);

execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());

throw SysError("could not execute chroot helper");
}

execvp(program.c_str(), stringsToCharPtrs(args).data());

throw SysError("unable to execute '%s'", program);
}
};

struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
{
std::vector<std::string> command = { "bash" };
StringSet keep, unset;
bool ignoreEnvironment = false;

CmdRun()
{
@@ -37,28 +73,6 @@ struct CmdRun : InstallablesCommand
if (ss.empty()) throw UsageError("--command requires at least one argument");
command = ss;
});

mkFlag()
.longName("ignore-environment")
.shortName('i')
.description("clear the entire environment (except those specified with --keep)")
.set(&ignoreEnvironment, true);

mkFlag()
.longName("keep")
.shortName('k')
.description("keep specified environment variable")
.arity(1)
.labels({"name"})
.handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });

mkFlag()
.longName("unset")
.shortName('u')
.description("unset specified environment variable")
.arity(1)
.labels({"name"})
.handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
}

std::string description() override
@@ -94,35 +108,13 @@ struct CmdRun : InstallablesCommand

auto accessor = store->getFSAccessor();

if (ignoreEnvironment) {

if (!unset.empty())
throw UsageError("--unset does not make sense with --ignore-environment");

std::map<std::string, std::string> kept;
for (auto & var : keep) {
auto s = getenv(var.c_str());
if (s) kept[var] = s;
}

clearEnv();

for (auto & var : kept)
setenv(var.first.c_str(), var.second.c_str(), 1);

} else {

if (!keep.empty())
throw UsageError("--keep does not make sense without --ignore-environment");

for (auto & var : unset)
unsetenv(var.c_str());
}

std::unordered_set<StorePath> done;
std::queue<StorePath> todo;
for (auto & path : outPaths) todo.push(path.clone());

setEnviron();

auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");

while (!todo.empty()) {
@@ -142,38 +134,10 @@ struct CmdRun : InstallablesCommand

setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);

std::string cmd = *command.begin();
Strings args;
for (auto & arg : command) args.push_back(arg);

stopProgressBar();

restoreSignals();

restoreAffinity();

/* If this is a diverted store (i.e. its "logical" location
(typically /nix/store) differs from its "physical" location
(e.g. /home/eelco/nix/store), then run the command in a
chroot. For non-root users, this requires running it in new
mount and user namespaces. Unfortunately,
unshare(CLONE_NEWUSER) doesn't work in a multithreaded
program (which "nix" is), so we exec() a single-threaded
helper program (chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>();

if (store2 && store->storeDir != store2->realStoreDir) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd };
for (auto & arg : args) helperArgs.push_back(arg);

execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());

throw SysError("could not execute chroot helper");
}

execvp(cmd.c_str(), stringsToCharPtrs(args).data());

throw SysError("unable to exec '%s'", cmd);
runProgram(store, *command.begin(), args);
}
};

@@ -0,0 +1,319 @@
#include "eval.hh"
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
#include "derivations.hh"
#include "affinity.hh"
#include "progress-bar.hh"

#include <regex>

using namespace nix;

struct Var
{
bool exported;
std::string value; // quoted string or array
};

struct BuildEnvironment
{
std::map<std::string, Var> env;
std::string bashFunctions;
};

BuildEnvironment readEnvironment(const Path & path)
{
BuildEnvironment res;

std::set<std::string> exported;

debug("reading environment file '%s'", path);

auto file = readFile(path);

auto pos = file.cbegin();

static std::string varNameRegex =
R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";

static std::regex declareRegex(
"^declare -x (" + varNameRegex + ")" +
R"re((?:="((?:[^"\\]|\\.)*)")?\n)re");

static std::string simpleStringRegex =
R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";

static std::string quotedStringRegex =
R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";

static std::string arrayRegex =
R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re";

static std::regex varRegex(
"^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n");

static std::regex functionRegex(
"^" + varNameRegex + " \\(\\) *\n");

while (pos != file.end()) {

std::smatch match;

if (std::regex_search(pos, file.cend(), match, declareRegex)) {
pos = match[0].second;
exported.insert(match[1]);
}

else if (std::regex_search(pos, file.cend(), match, varRegex)) {
pos = match[0].second;
res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }});
}

else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
res.bashFunctions = std::string(pos, file.cend());
break;
}

else throw Error("shell environment '%s' has unexpected line '%s'",
path, file.substr(pos - file.cbegin(), 60));
}

return res;
}

/* Given an existing derivation, return the shell environment as
initialised by stdenv's setup script. We do this by building a
modified derivation with the same dependencies and nearly the same
initial environment variables, that just writes the resulting
environment to a file and exits. */
StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
{
auto builder = baseNameOf(drv.builder);
if (builder != "bash")
throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder");

drv.args = {
"-c",
"set -e; "
"export IN_NIX_SHELL=impure; "
"export dontAddDisableDepTrack=1; "
"if [[ -n $stdenv ]]; then "
" source $stdenv/setup; "
"fi; "
"export > $out; "
"set >> $out "};

/* Remove derivation checks. */
drv.env.erase("allowedReferences");
drv.env.erase("allowedRequisites");
drv.env.erase("disallowedReferences");
drv.env.erase("disallowedRequisites");

// FIXME: handle structured attrs

/* Rehash and write the derivation. FIXME: would be nice to use
'buildDerivation', but that's privileged. */
auto drvName = drv.env["name"] + "-env";
for (auto & output : drv.outputs)
drv.env.erase(output.first);
drv.env["out"] = "";
drv.env["outputs"] = "out";
Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", ""));
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName);

/* Build the derivation. */
store->buildPaths({shellDrvPath2});

assert(store->isValidPath(shellOutPath));

return shellOutPath;
}

struct Common : InstallableCommand, MixProfile
{
std::set<string> ignoreVars{
"BASHOPTS",
"EUID",
"HOME", // FIXME: don't ignore in pure mode?
"NIX_BUILD_TOP",
"NIX_ENFORCE_PURITY",
"NIX_LOG_FD",
"PPID",
"PWD",
"SHELLOPTS",
"SHLVL",
"SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
"TEMP",
"TEMPDIR",
"TERM",
"TMP",
"TMPDIR",
"TZ",
"UID",
};

void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
{
out << "nix_saved_PATH=\"$PATH\"\n";

for (auto & i : buildEnvironment.env) {
if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
out << fmt("%s=%s\n", i.first, i.second.value);
if (i.second.exported)
out << fmt("export %s\n", i.first);
}
}

out << "PATH=\"$PATH:$nix_saved_PATH\"\n";

out << buildEnvironment.bashFunctions << "\n";

// FIXME: set outputs

out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);

out << "eval \"$shellHook\"\n";
}

StorePath getShellOutPath(ref<Store> store)
{
auto path = installable->getStorePath();
if (path && hasSuffix(path->to_string(), "-env"))
return path->clone();
else {
auto drvs = toDerivations(store, {installable});

if (drvs.size() != 1)
throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations",
installable->what(), drvs.size());

auto & drvPath = *drvs.begin();

return getDerivationEnvironment(store, store->derivationFromPath(drvPath));
}
}

BuildEnvironment getBuildEnvironment(ref<Store> store)
{
auto shellOutPath = getShellOutPath(store);

updateProfile(shellOutPath);

return readEnvironment(store->printStorePath(shellOutPath));
}
};

struct CmdDevShell : Common, MixEnvironment
{
std::vector<std::string> command;

CmdDevShell()
{
mkFlag()
.longName("command")
.shortName('c')
.description("command and arguments to be executed insted of an interactive shell")
.labels({"command", "args"})
.arity(ArityAny)
.handler([&](std::vector<std::string> ss) {
if (ss.empty()) throw UsageError("--command requires at least one argument");
command = ss;
});
}

std::string description() override
{
return "run a bash shell that provides the build environment of a derivation";
}

Examples examples() override
{
return {
Example{
"To get the build environment of GNU hello:",
"nix dev-shell nixpkgs.hello"
},
Example{
"To store the build environment in a profile:",
"nix dev-shell --profile /tmp/my-shell nixpkgs.hello"
},
Example{
"To use a build environment previously recorded in a profile:",
"nix dev-shell /tmp/my-shell"
},
};
}

void run(ref<Store> store) override
{
auto buildEnvironment = getBuildEnvironment(store);

auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");

std::ostringstream ss;
makeRcScript(buildEnvironment, ss);

ss << fmt("rm -f '%s'\n", rcFilePath);

if (!command.empty()) {
std::vector<std::string> args;
for (auto s : command)
args.push_back(shellEscape(s));
ss << fmt("exec %s\n", concatStringsSep(" ", args));
}

writeFull(rcFileFd.get(), ss.str());

stopProgressBar();

auto shell = getEnv("SHELL").value_or("bash");

setEnviron();

auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};

restoreAffinity();
restoreSignals();

execvp(shell.c_str(), stringsToCharPtrs(args).data());

throw SysError("executing shell '%s'", shell);
}
};

struct CmdPrintDevEnv : Common
{
std::string description() override
{
return "print shell code that can be sourced by bash to reproduce the build environment of a derivation";
}

Examples examples() override
{
return {
Example{
"To apply the build environment of GNU hello to the current shell:",
". <(nix print-dev-env nixpkgs.hello)"
},
};
}

void run(ref<Store> store) override
{
auto buildEnvironment = getBuildEnvironment(store);

stopProgressBar();

makeRcScript(buildEnvironment, std::cout);
}
};

static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
static auto r2 = registerCommand<CmdDevShell>("dev-shell");
@@ -11,7 +11,7 @@ repo=$TEST_ROOT/git

export _NIX_FORCE_HTTP=1

rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow

git init $repo
git -C $repo config user.email "foobar@example.com"
@@ -25,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD)

echo world > $repo/hello
git -C $repo commit -m 'Bla2' -a
git -C $repo worktree add $TEST_ROOT/worktree
echo hello >> $TEST_ROOT/worktree/hello
rev2=$(git -C $repo rev-parse HEAD)

# Fetch a worktree
unset _NIX_FORCE_HTTP
path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
export _NIX_FORCE_HTTP=1
[[ $(tail -n 1 $path0/hello) = "hello" ]]

# Fetch the default branch.
path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
[[ $(cat $path/hello) = world ]]
@@ -50,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]]
[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]]

# But with TTL 0, it should fail.
(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv)

# Fetching with a explicit hash should succeed.
path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
@@ -74,6 +79,7 @@ echo bar > $repo/dir2/bar
git -C $repo add dir1/foo
git -C $repo rm hello

unset _NIX_FORCE_HTTP
path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
[ ! -e $path2/hello ]
[ ! -e $path2/bar ]
@@ -110,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
git -C $repo checkout $rev2 -b dev
echo dev > $repo/hello

# File URI uses 'master' unless specified otherwise
# File URI uses dirty tree unless specified otherwise
path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
[[ $path = $path2 ]]
[ $(cat $path2/hello) = dev ]

# Using local path with branch other than 'master' should work when clean or dirty
path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
@@ -131,13 +137,23 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP


# Nuke the cache
rm -rf $TEST_HOME/.cache/nix/gitv2
rm -rf $TEST_HOME/.cache/nix

# Try again, but without 'git' on PATH
# Try again, but without 'git' on PATH. This should fail.
NIX=$(command -v nix)
# This should fail
(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )

# Try again, with 'git' available. This should work.
path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
[[ $path3 = $path5 ]]

# Fetching a shallow repo shouldn't work by default, because we can't
# return a revCount.
git clone --depth 1 file://$repo $TEST_ROOT/shallow
(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath")

# But you can request a shallow clone, which won't return a revCount.
path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
[[ $path3 = $path6 ]]
[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
@@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
build-users-group =
keep-derivations = false
sandbox = false
experimental-features = nix-command
experimental-features = nix-command flakes
include nix.conf.extra
EOF

@@ -10,6 +10,8 @@ mkdir -p $tarroot
cp dependencies.nix $tarroot/default.nix
cp config.nix dependencies.builder*.sh $tarroot/

hash=$(nix hash-path $tarroot)

test_tarball() {
local ext="$1"
local compressor="$2"
@@ -25,6 +27,11 @@ test_tarball() {

nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"

nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'

nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)