Skip to content

/ nix

Permalink
Browse files

Remove failed build caching 

This feature was implemented for Hydra, but Hydra no longer uses it.
  • Loading branch information
@edolstra
edolstra committed Apr 8, 2016
1 parent f398949 commit 8cffec84859cec8b610a2a22ab0c4d462a9351ff
15 doc/manual/command-ref/conf-file.xml
View file
@@ -306,21 +306,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
</varlistentry>


<varlistentry><term><literal>build-cache-failure</literal></term>

<listitem><para>If set to <literal>true</literal>, Nix will
“cache” build failures, meaning that it will remember (in its
database) that a derivation previously failed. If you then try to
build the derivation again, Nix will immediately fail rather than
perform the build again. Failures in fixed-output derivations
(such as <function>fetchurl</function> calls) are never cached.
The “failed” status of a derivation can be cleared using
<command>nix-store --clear-failed-paths</command>. By default,
failure caching is disabled.</para></listitem>

</varlistentry>


<varlistentry><term><literal>build-keep-log</literal></term>

<listitem><para>If set to <literal>true</literal> (the default),
76 doc/manual/command-ref/nix-store.xml
View file
@@ -1348,82 +1348,6 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil
</refsection>


<!--######################################################################-->

<refsection><title>Operation <option>--query-failed-paths</option></title>

<refsection>
<title>Synopsis</title>
<cmdsynopsis>
<command>nix-store</command>
<arg choice='plain'><option>--query-failed-paths</option></arg>
</cmdsynopsis>
</refsection>

<refsection><title>Description</title>

<para>If build failure caching is enabled through the
<literal>build-cache-failure</literal> configuration option, the
operation <option>--query-failed-paths</option> will print out all
store paths that have failed to build.</para>

</refsection>

<refsection><title>Example</title>

<screen>
$ nix-store --query-failed-paths
/nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59
/nix/store/0011iy7sfwbc1qj5a1f6ifjnbcdail8a-haskell-gitit-ghc7.0.4-0.8.1
/nix/store/001c0yn1hkh86gprvrb46cxnz3pki7q3-gamin-0.1.10
<replaceable>…</replaceable>
</screen>

</refsection>

</refsection>


<!--######################################################################-->

<refsection><title>Operation <option>--clear-failed-paths</option></title>

<refsection>
<title>Synopsis</title>
<cmdsynopsis>
<command>nix-store</command>
<arg choice='plain'><option>--clear-failed-paths</option></arg>
<arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
</cmdsynopsis>
</refsection>

<refsection><title>Description</title>

<para>If build failure caching is enabled through the
<literal>build-cache-failure</literal> configuration option, the
operation <option>--clear-failed-paths</option> clears the “failed”
state of the given store paths, allowing them to be built again. This
is useful if the failure was actually transient (e.g. because the disk
was full).</para>

<para>If a path denotes a derivation, its output paths are cleared.
You can provide the argument <literal>*</literal> to clear all store
paths.</para>

</refsection>

<refsection><title>Example</title>

<screen>
$ nix-store --clear-failed-paths /nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59
$ nix-store --clear-failed-paths *
</screen>

</refsection>

</refsection>


<!--######################################################################-->

<refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title>
6 src/libstore/binary-cache-store.hh
View file
@@ -156,12 +156,6 @@ public:
void collectGarbage(const GCOptions & options, GCResults & results) override
{ notImpl(); }

PathSet queryFailedPaths() override
{ return {}; }

void clearFailedPaths(const PathSet & paths) override
{ }

void optimiseStore() override
{ }

41 src/libstore/build.cc
View file
@@ -1047,11 +1047,6 @@ void DerivationGoal::haveDerivation()
return;
}

/* Check whether any output previously failed to build. If so,
don't bother. */
for (auto & i : invalidOutputs)
if (pathFailed(i)) return;

/* Reject doing a hash build of anything other than a fixed-output
derivation. */
if (buildMode == bmHash) {
@@ -1322,12 +1317,6 @@ void DerivationGoal::tryToBuild()
deletePath(path);
}

/* Check again whether any output previously failed to build,
because some other process may have tried and failed before we
acquired the lock. */
for (auto & i : drv->outputs)
if (pathFailed(i.second.path)) return;

/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
@@ -1549,17 +1538,6 @@ void DerivationGoal::buildDone()
statusOk(status) ? BuildResult::OutputRejected :
fixedOutput || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;

/* Register the outputs of this build as "failed" so we
won't try to build them again (negative caching).
However, don't do this for fixed-output derivations,
since they're likely to fail for transient reasons
(e.g., fetchurl not being able to access the network).
Hook errors (like communication problems with the
remote machine) shouldn't be cached either. */
if (settings.cacheFailure && !fixedOutput && !diskFull)
for (auto & i : drv->outputs)
worker.store.registerFailedPath(i.second.path);
}

done(st, e.msg());
@@ -2993,23 +2971,6 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
}


bool DerivationGoal::pathFailed(const Path & path)
{
if (!settings.cacheFailure) return false;

if (!worker.store.hasPathFailed(path)) return false;

printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path);

if (settings.printBuildTrace)
printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath);

done(BuildResult::CachedFailure);

return true;
}


Path DerivationGoal::addHashRewrite(const Path & path)
{
string h1 = string(path, settings.nixStore.size() + 1, 32);
@@ -3031,7 +2992,7 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg)
amDone(result.success() ? ecSuccess : ecFailed);
if (result.status == BuildResult::TimedOut)
worker.timedOut = true;
if (result.status == BuildResult::PermanentFailure || result.status == BuildResult::CachedFailure)
if (result.status == BuildResult::PermanentFailure)
worker.permanentFailure = true;
}

2 src/libstore/globals.cc
View file
@@ -52,7 +52,6 @@ Settings::Settings()
keepLog = true;
compressLog = true;
maxLogSize = 0;
cacheFailure = false;
pollInterval = 5;
checkRootReachability = false;
gcKeepOutputs = false;
@@ -175,7 +174,6 @@ void Settings::update()
_get(keepLog, "build-keep-log");
_get(compressLog, "build-compress-log");
_get(maxLogSize, "build-max-log-size");
_get(cacheFailure, "build-cache-failure");
_get(pollInterval, "build-poll-interval");
_get(checkRootReachability, "gc-check-reachability");
_get(gcKeepOutputs, "gc-keep-outputs");
3 src/libstore/globals.hh
View file
@@ -168,9 +168,6 @@ struct Settings {
before being killed (0 means no limit). */
unsigned long maxLogSize;

/* Whether to cache build failures. */
bool cacheFailure;

/* How often (in seconds) to poll for locks. */
unsigned int pollInterval;

66 src/libstore/local-store.cc
View file
@@ -198,6 +198,13 @@ LocalStore::LocalStore()
txn.commit();
}

if (curSchema < 9) {
SQLiteTxn txn(state->db);
if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "upgrading database schema");
txn.commit();
}

writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());

lockFile(globalLock, ltRead, true);
@@ -327,16 +334,6 @@ void LocalStore::openDB(State & state, bool create)
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
state.stmtInvalidatePath.create(db,
"delete from ValidPaths where path = ?;");
state.stmtRegisterFailedPath.create(db,
"insert or ignore into FailedPaths (path, time) values (?, ?);");
state.stmtHasPathFailed.create(db,
"select time from FailedPaths where path = ?;");
state.stmtQueryFailedPaths.create(db,
"select path from FailedPaths;");
// If the path is a derivation, then clear its outputs.
state.stmtClearFailedPath.create(db,
"delete from FailedPaths where ?1 = '*' or path = ?1 "
"or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
state.stmtAddDerivationOutput.create(db,
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
state.stmtQueryValidDerivers.create(db,
@@ -583,55 +580,6 @@ uint64_t LocalStore::addValidPath(State & state,
}


void LocalStore::registerFailedPath(const Path & path)
{
retrySQLite<void>([&]() {
auto state(_state.lock());
state->stmtRegisterFailedPath.use()(path)(time(0)).step();
});
}


bool LocalStore::hasPathFailed(const Path & path)
{
return retrySQLite<bool>([&]() {
auto state(_state.lock());
return state->stmtHasPathFailed.use()(path).next();
});
}


PathSet LocalStore::queryFailedPaths()
{
return retrySQLite<PathSet>([&]() {
auto state(_state.lock());

auto useQueryFailedPaths(state->stmtQueryFailedPaths.use());

PathSet res;
while (useQueryFailedPaths.next())
res.insert(useQueryFailedPaths.getStr(0));

return res;
});
}


void LocalStore::clearFailedPaths(const PathSet & paths)
{
retrySQLite<void>([&]() {
auto state(_state.lock());

SQLiteTxn txn(state->db);

for (auto & path : paths)
state->stmtClearFailedPath.use()(path).exec();

txn.commit();
});
}


Hash parseHashField(const Path & path, const string & s)
{
string::size_type colon = s.find(':');
19 src/libstore/local-store.hh
View file
@@ -17,8 +17,8 @@ namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
Nix 1.0. Version 7 is Nix 1.3. Version 8 is 1.12. */
const int nixSchemaVersion = 8;
Nix 1.0. Version 7 is Nix 1.3. Version 9 is 1.12. */
const int nixSchemaVersion = 9;


extern string drvsLogDir;
@@ -71,10 +71,6 @@ private:
SQLiteStmt stmtQueryReferences;
SQLiteStmt stmtQueryReferrers;
SQLiteStmt stmtInvalidatePath;
SQLiteStmt stmtRegisterFailedPath;
SQLiteStmt stmtHasPathFailed;
SQLiteStmt stmtQueryFailedPaths;
SQLiteStmt stmtClearFailedPath;
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
@@ -194,17 +190,6 @@ public:

void registerValidPaths(const ValidPathInfos & infos);

/* Register that the build of a derivation with output `path' has
failed. */
void registerFailedPath(const Path & path);

/* Query whether `path' previously failed to build. */
bool hasPathFailed(const Path & path);

PathSet queryFailedPaths() override;

void clearFailedPaths(const PathSet & paths) override;

void vacuumDB();

/* Repair the contents of the given path by redownloading it using
18 src/libstore/remote-store.cc
View file
@@ -520,23 +520,6 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
}


PathSet RemoteStore::queryFailedPaths()
{
auto conn(connections->get());
conn->to << wopQueryFailedPaths;
conn->processStderr();
return readStorePaths<PathSet>(conn->from);
}


void RemoteStore::clearFailedPaths(const PathSet & paths)
{
auto conn(connections->get());
conn->to << wopClearFailedPaths << paths;
conn->processStderr();
readInt(conn->from);
}

void RemoteStore::optimiseStore()
{
auto conn(connections->get());
@@ -545,6 +528,7 @@ void RemoteStore::optimiseStore()
readInt(conn->from);
}


bool RemoteStore::verifyStore(bool checkContents, bool repair)
{
auto conn(connections->get());
4 src/libstore/remote-store.hh
View file
@@ -85,10 +85,6 @@ public:

void collectGarbage(const GCOptions & options, GCResults & results) override;

PathSet queryFailedPaths() override;

void clearFailedPaths(const PathSet & paths) override;

void optimiseStore() override;

bool verifyStore(bool checkContents, bool repair) override;
5 src/libstore/schema.sql
View file
@@ -39,8 +39,3 @@ create table if not exists DerivationOutputs (
);

create index if not exists IndexDerivationOutputs on DerivationOutputs(path);

create table if not exists FailedPaths (
path text primary key not null,
time integer not null
);

3 comments on commit 8cffec8

@michalrus

This comment has been minimized.

Sign in to view
Copy link
Member

@michalrus michalrus replied Jul 13, 2018

I used it locally, quite extensively, to lazily see which derivation failed in big updates.

RIP. 🕯️
@nixos-discourse

This comment has been minimized.

Sign in to view
Copy link

@nixos-discourse nixos-discourse replied Feb 13, 2019

This commit has been mentioned on Nix community. There might be relevant details there:

https://discourse.nixos.org/t/behavior-of-undeclared-broken-packages-is-confusing/2123/2
@nixos-discourse

This comment has been minimized.

Sign in to view
Copy link

@nixos-discourse nixos-discourse replied Feb 26, 2020

This commit has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/why-do-nix-try-to-realise-a-derivation-that-already-failed/6026/4
Please sign in to comment.