Permalink
Browse files

Pass lists/attrsets to bash as (associative) arrays

  • Loading branch information...
edolstra committed Oct 25, 2017
1 parent ac12517 commit 2d5b1b24bf70a498e4c0b378704cfdb6471cc699
@@ -713,7 +713,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
drv.env["out"] = outPath;
if (!jsonObject) drv.env["out"] = outPath;
drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash);
}
@@ -724,7 +724,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
an empty value. This ensures that changes in the set of
output names do get reflected in the hash. */
for (auto & i : outputs) {
drv.env[i] = "";
if (!jsonObject) drv.env[i] = "";
drv.outputs[i] = DerivationOutput("", "", "");
}
@@ -735,7 +735,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : drv.outputs)
if (i.second.path == "") {
Path outPath = state.store->makeOutputPath(i.first, h, drvName);
drv.env[i.first] = outPath;
if (!jsonObject) drv.env[i.first] = outPath;
i.second.path = outPath;
}
}
@@ -18,6 +18,7 @@
#include <thread>
#include <future>
#include <chrono>
#include <regex>
#include <limits.h>
#include <sys/time.h>
@@ -55,6 +56,8 @@
#include <sys/statvfs.h>
#endif
#include <nlohmann/json.hpp>
namespace nix {
@@ -2286,12 +2289,99 @@ void DerivationGoal::initEnv()
}
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
void DerivationGoal::writeStructuredAttrs()
{
auto json = drv->env.find("__json");
if (json == drv->env.end()) return;
auto jsonAttr = drv->env.find("__json");
if (jsonAttr == drv->env.end()) return;
try {
auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites);
auto json = nlohmann::json::parse(jsonStr);
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs)
outputs[i.first] = rewriteStrings(i.second.path, inputRewrites);
json["outputs"] = outputs;
writeFile(tmpDir + "/.attrs.json", json.dump());
/* As a convenience to bash scripts, write a shell file that
maps all attributes that are representable in bash -
namely, strings, integers, nulls, Booleans, and arrays and
objects consisting entirely of those values. (So nested
arrays or objects are not supported.) */
auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> {
if (value.is_string())
return shellEscape(value);
if (value.is_number()) {
auto f = value.get<float>();
if (std::ceil(f) == f)
return std::to_string(value.get<int>());
}
if (value.is_null())
return "''";
if (value.is_boolean())
return value.get<bool>() ? "1" : "";
return {};
};
std::string jsonSh;
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites));
for (auto i = json.begin(); i != json.end(); ++i) {
if (!std::regex_match(i.key(), shVarName)) continue;
auto & value = i.value();
auto s = handleSimpleType(value);
if (s)
jsonSh += fmt("declare %s=%s\n", i.key(), *s);
else if (value.is_array()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += *s3; s2 += ' ';
}
if (good)
jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
}
else if (value.is_object()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
}
if (good)
jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
}
}
writeFile(tmpDir + "/.attrs.sh", jsonSh);
} catch (std::exception & e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
}
}
@@ -1142,6 +1142,16 @@ std::string toLower(const std::string & s)
}
std::string shellEscape(const std::string & s)
{
std::string r = "'";
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
r += '\'';
return r;
}
void ignoreException()
{
try {
@@ -352,10 +352,8 @@ bool hasSuffix(const string & s, const string & suffix);
std::string toLower(const std::string & s);
/* Escape a string that contains octal-encoded escape codes such as
used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to
"foo bar"). */
string decodeOctalEscaped(const string & s);
/* Escape a string as a shell word. */
std::string shellEscape(const std::string & s);
/* Exception handling in destructors: print an error message, then
@@ -196,10 +196,6 @@ void mainWrapped(int argc, char * * argv)
interactive = false;
auto execArgs = "";
auto shellEscape = [](const string & s) {
return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'";
};
// Überhack to support Perl. Perl examines the shebang and
// executes it unless it contains the string "perl" or "indir",
// or (undocumented) argv[0] does not contain "perl". Exploit
@@ -440,15 +440,6 @@ static void opQuery(Strings opFlags, Strings opArgs)
}
static string shellEscape(const string & s)
{
string r;
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
return r;
}
static void opPrintEnv(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -460,7 +451,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs)
/* Print each environment variable in the derivation in a format
that can be sourced by the shell. */
for (auto & i : drv.env)
cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second);
cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second);
/* Also output the arguments. This doesn't preserve whitespace in
arguments. */
@@ -13,7 +13,7 @@ rec {
derivation ({
inherit system;
builder = shell;
args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")];
args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
PATH = path;
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
@@ -14,7 +14,8 @@ nix_tests = \
placeholders.sh nix-shell.sh \
linux-sandbox.sh \
build-remote.sh \
nar-index.sh
nar-index.sh \
structured-attrs.sh
# parallel.sh
install-tests += $(foreach x, $(nix_tests), tests/$(x))
@@ -0,0 +1,47 @@
with import ./config.nix;
mkDerivation {
name = "structured";
__structuredAttrs = true;
buildCommand = ''
set -x
[[ $int = 123456789 ]]
[[ -z $float ]]
[[ -n $boolTrue ]]
[[ -z $boolFalse ]]
[[ -n ''${hardening[format]} ]]
[[ -z ''${hardening[fortify]} ]]
[[ ''${#buildInputs[@]} = 7 ]]
[[ ''${buildInputs[2]} = c ]]
[[ -v nothing ]]
[[ -z $nothing ]]
mkdir ''${outputs[out]}
echo bar > $dest
'';
buildInputs = [ "a" "b" "c" 123 "'" "\"" null ];
hardening.format = true;
hardening.fortify = false;
outer.inner = [ 1 2 3 ];
int = 123456789;
float = 123.456;
boolTrue = true;
boolFalse = false;
nothing = null;
dest = "${placeholder "out"}/foo";
"foo bar" = "BAD";
"1foobar" = "BAD";
"foo$" = "BAD";
}
@@ -0,0 +1,7 @@
source common.sh
clearStore
outPath=$(nix-build structured-attrs.nix --no-out-link)
[[ $(cat $outPath/foo) = bar ]]

2 comments on commit 2d5b1b2

@copumpkin

This comment has been minimized.

Member

copumpkin replied Oct 26, 2017

w00t

@globin

This comment has been minimized.

Member

globin replied Oct 26, 2017

🎉

Please sign in to comment.