Skip to content

Commit

Permalink
EvalCache: Store string contexts
Browse files Browse the repository at this point in the history
  • Loading branch information
@edolstra
edolstra committed Jun 29, 2020
1 parent b681408 commit 50f13b0
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 27 deletions.
75 changes: 62 additions & 13 deletions src/libexpr/eval-cache.cc
View file
Expand Up @@ -12,6 +12,7 @@ create table if not exists Attributes (
name text,
type integer not null,
value text,
context text,
primary key (parent, name)
);
)sql";
Expand All @@ -22,6 +23,7 @@ struct AttrDb
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt insertAttributeWithContext;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
Expand All @@ -34,7 +36,7 @@ struct AttrDb
{
auto state(_state->lock());

Path cacheDir = getCacheDir() + "/nix/eval-cache-v1";
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
createDirs(cacheDir);

Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
Expand All @@ -46,8 +48,11 @@ struct AttrDb
state->insertAttribute.create(state->db,
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");

state->insertAttributeWithContext.create(state->db,
"insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");

state->queryAttribute.create(state->db,
"select rowid, type, value from Attributes where parent = ? and name = ?");
"select rowid, type, value, context from Attributes where parent = ? and name = ?");

state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
Expand Down Expand Up @@ -93,15 +98,30 @@ struct AttrDb

AttrId setString(
AttrKey key,
std::string_view s)
std::string_view s,
const char * * context = nullptr)
{
auto state(_state->lock());

state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
if (context) {
std::string ctx;
for (const char * * p = context; *p; ++p) {
if (p != context) ctx.push_back(' ');
ctx.append(*p);
}
state->insertAttributeWithContext.use()
(key.first)
(key.second)
(AttrType::String)
(s)
(ctx).exec();
} else {
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
}

return state->db.getLastInsertedRowId();
}
Expand Down Expand Up @@ -196,8 +216,13 @@ struct AttrDb
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
}
case AttrType::String:
return {{rowId, queryAttribute.getStr(2)}};
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::Missing:
Expand Down Expand Up @@ -320,7 +345,7 @@ Value & AttrCursor::forceValue()

if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getKey(), v.string.s), v.string.s};
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
else if (v.type == tPath)
cachedValue = {root->db->setString(getKey(), v.path), v.path};
else if (v.type == tBool)
Expand Down Expand Up @@ -427,9 +452,9 @@ std::string AttrCursor::getString()
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
return s->first;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
Expand All @@ -443,6 +468,30 @@ std::string AttrCursor::getString()
return v.type == tString ? v.string.s : v.path;
}

string_t AttrCursor::getStringWithContext()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}

auto & v = forceValue();

if (v.type == tString)
return {v.string.s, v.getContext()};
else if (v.type == tPath)
return {v.path, {}};
else
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
}

bool AttrCursor::getBool()
{
if (root->db) {
Expand Down
14 changes: 13 additions & 1 deletion src/libexpr/eval-cache.hh
View file
Expand Up @@ -50,7 +50,17 @@ struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t, bool> AttrValue;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;

typedef std::variant<
std::vector<Symbol>,
string_t,
placeholder_t,
missing_t,
misc_t,
failed_t,
bool
> AttrValue;

class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
Expand Down Expand Up @@ -94,6 +104,8 @@ public:

std::string getString();

string_t getStringWithContext();

bool getBool();

std::vector<Symbol> getAttrs();
Expand Down
23 changes: 23 additions & 0 deletions src/libexpr/eval.cc
View file
Expand Up @@ -1607,6 +1607,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
}


/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}


void copyContext(const Value & v, PathSet & context)
{
if (v.string.context)
Expand All @@ -1615,6 +1627,17 @@ void copyContext(const Value & v, PathSet & context)
}


std::vector<std::pair<Path, std::string>> Value::getContext()
{
std::vector<std::pair<Path, std::string>> res;
assert(type == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(*p));
return res;
}


string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
{
string s = forceString(v, pos);
Expand Down
2 changes: 1 addition & 1 deletion src/libexpr/eval.hh
View file
Expand Up @@ -333,7 +333,7 @@ string showType(const Value & v);

/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(const string & s);
std::pair<string, string> decodeContext(std::string_view s);

/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);
Expand Down
12 changes: 0 additions & 12 deletions src/libexpr/primops.cc
View file
Expand Up @@ -30,18 +30,6 @@ namespace nix {
*************************************************************/


/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(const string & s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
} else
return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
}


InvalidPathError::InvalidPathError(const Path & path) :
EvalError("path '%s' is not valid", path), path(path) {}

Expand Down
2 changes: 2 additions & 0 deletions src/libexpr/value.hh
View file
Expand Up @@ -171,6 +171,8 @@ struct Value
computation. In particular, function applications are
non-trivial. */
bool isTrivial() const;

std::vector<std::pair<Path, std::string>> getContext();
};


Expand Down

0 comments on commit 50f13b0

Please sign in to comment.