Remove failed build caching

This feature was implemented for Hydra, but Hydra no longer uses it.

doc/manual/command-ref/conf-file.xml

@@ -306,21 +306,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
  </varlistentry>


  <varlistentry><term><literal>build-cache-failure</literal></term>

    <listitem><para>If set to <literal>true</literal>, Nix will
    “cache” build failures, meaning that it will remember (in its
    database) that a derivation previously failed.  If you then try to
    build the derivation again, Nix will immediately fail rather than
    perform the build again.  Failures in fixed-output derivations
    (such as <function>fetchurl</function> calls) are never cached.
    The “failed” status of a derivation can be cleared using
    <command>nix-store --clear-failed-paths</command>.  By default,
    failure caching is disabled.</para></listitem>

  </varlistentry>


  <varlistentry><term><literal>build-keep-log</literal></term>

    <listitem><para>If set to <literal>true</literal> (the default),

doc/manual/command-ref/nix-store.xml

@@ -1348,82 +1348,6 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil
</refsection>


<!--######################################################################-->

<refsection><title>Operation <option>--query-failed-paths</option></title>

<refsection>
  <title>Synopsis</title>
  <cmdsynopsis>
    <command>nix-store</command>
    <arg choice='plain'><option>--query-failed-paths</option></arg>
  </cmdsynopsis>
</refsection>

<refsection><title>Description</title>

<para>If build failure caching is enabled through the
<literal>build-cache-failure</literal> configuration option, the
operation <option>--query-failed-paths</option> will print out all
store paths that have failed to build.</para>

</refsection>

<refsection><title>Example</title>

<screen>
$ nix-store --query-failed-paths
/nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59
/nix/store/0011iy7sfwbc1qj5a1f6ifjnbcdail8a-haskell-gitit-ghc7.0.4-0.8.1
/nix/store/001c0yn1hkh86gprvrb46cxnz3pki7q3-gamin-0.1.10
<replaceable>…</replaceable>
</screen>

</refsection>

</refsection>


<!--######################################################################-->

<refsection><title>Operation <option>--clear-failed-paths</option></title>

<refsection>
  <title>Synopsis</title>
  <cmdsynopsis>
    <command>nix-store</command>
    <arg choice='plain'><option>--clear-failed-paths</option></arg>
    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
  </cmdsynopsis>
</refsection>

<refsection><title>Description</title>

<para>If build failure caching is enabled through the
<literal>build-cache-failure</literal> configuration option, the
operation <option>--clear-failed-paths</option> clears the “failed”
state of the given store paths, allowing them to be built again.  This
is useful if the failure was actually transient (e.g. because the disk
was full).</para>

<para>If a path denotes a derivation, its output paths are cleared.
You can provide the argument <literal>*</literal> to clear all store
paths.</para>

</refsection>

<refsection><title>Example</title>

<screen>
$ nix-store --clear-failed-paths /nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59
$ nix-store --clear-failed-paths *
</screen>

</refsection>

</refsection>


<!--######################################################################-->

<refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title>

src/libstore/binary-cache-store.hh

@@ -156,12 +156,6 @@ public:
    void collectGarbage(const GCOptions & options, GCResults & results) override
    { notImpl(); }

    PathSet queryFailedPaths() override
    { return {}; }

    void clearFailedPaths(const PathSet & paths) override
    { }

    void optimiseStore() override
    { }

src/libstore/build.cc

@@ -1047,11 +1047,6 @@ void DerivationGoal::haveDerivation()
        return;
    }

    /* Check whether any output previously failed to build.  If so,
       don't bother. */
    for (auto & i : invalidOutputs)
        if (pathFailed(i)) return;

    /* Reject doing a hash build of anything other than a fixed-output
       derivation. */
    if (buildMode == bmHash) {
@@ -1322,12 +1317,6 @@ void DerivationGoal::tryToBuild()
        deletePath(path);
    }

    /* Check again whether any output previously failed to build,
       because some other process may have tried and failed before we
       acquired the lock. */
    for (auto & i : drv->outputs)
        if (pathFailed(i.second.path)) return;

    /* Don't do a remote build if the derivation has the attribute
       `preferLocalBuild' set.  Also, check and repair modes are only
       supported for local builds. */
@@ -1549,17 +1538,6 @@ void DerivationGoal::buildDone()
                statusOk(status) ? BuildResult::OutputRejected :
                fixedOutput || diskFull ? BuildResult::TransientFailure :
                BuildResult::PermanentFailure;

            /* Register the outputs of this build as "failed" so we
               won't try to build them again (negative caching).
               However, don't do this for fixed-output derivations,
               since they're likely to fail for transient reasons
               (e.g., fetchurl not being able to access the network).
               Hook errors (like communication problems with the
               remote machine) shouldn't be cached either. */
            if (settings.cacheFailure && !fixedOutput && !diskFull)
                for (auto & i : drv->outputs)
                    worker.store.registerFailedPath(i.second.path);
        }

        done(st, e.msg());
@@ -2993,23 +2971,6 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
}


bool DerivationGoal::pathFailed(const Path & path)
{
    if (!settings.cacheFailure) return false;

    if (!worker.store.hasPathFailed(path)) return false;

    printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path);

    if (settings.printBuildTrace)
        printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath);

    done(BuildResult::CachedFailure);

    return true;
}


Path DerivationGoal::addHashRewrite(const Path & path)
{
    string h1 = string(path, settings.nixStore.size() + 1, 32);
@@ -3031,7 +2992,7 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg)
    amDone(result.success() ? ecSuccess : ecFailed);
    if (result.status == BuildResult::TimedOut)
        worker.timedOut = true;
    if (result.status == BuildResult::PermanentFailure || result.status == BuildResult::CachedFailure)
    if (result.status == BuildResult::PermanentFailure)
        worker.permanentFailure = true;
}

src/libstore/globals.cc

@@ -52,7 +52,6 @@ Settings::Settings()
    keepLog = true;
    compressLog = true;
    maxLogSize = 0;
    cacheFailure = false;
    pollInterval = 5;
    checkRootReachability = false;
    gcKeepOutputs = false;
@@ -175,7 +174,6 @@ void Settings::update()
    _get(keepLog, "build-keep-log");
    _get(compressLog, "build-compress-log");
    _get(maxLogSize, "build-max-log-size");
    _get(cacheFailure, "build-cache-failure");
    _get(pollInterval, "build-poll-interval");
    _get(checkRootReachability, "gc-check-reachability");
    _get(gcKeepOutputs, "gc-keep-outputs");

src/libstore/globals.hh

@@ -168,9 +168,6 @@ struct Settings {
       before being killed (0 means no limit). */
    unsigned long maxLogSize;

    /* Whether to cache build failures. */
    bool cacheFailure;

    /* How often (in seconds) to poll for locks. */
    unsigned int pollInterval;

src/libstore/local-store.cc

@@ -198,6 +198,13 @@ LocalStore::LocalStore()
            txn.commit();
        }

        if (curSchema < 9) {
            SQLiteTxn txn(state->db);
            if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK)
                throwSQLiteError(state->db, "upgrading database schema");
            txn.commit();
        }

        writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());

        lockFile(globalLock, ltRead, true);
@@ -327,16 +334,6 @@ void LocalStore::openDB(State & state, bool create)
        "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
    state.stmtInvalidatePath.create(db,
        "delete from ValidPaths where path = ?;");
    state.stmtRegisterFailedPath.create(db,
        "insert or ignore into FailedPaths (path, time) values (?, ?);");
    state.stmtHasPathFailed.create(db,
        "select time from FailedPaths where path = ?;");
    state.stmtQueryFailedPaths.create(db,
        "select path from FailedPaths;");
    // If the path is a derivation, then clear its outputs.
    state.stmtClearFailedPath.create(db,
        "delete from FailedPaths where ?1 = '*' or path = ?1 "
        "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
    state.stmtAddDerivationOutput.create(db,
        "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
    state.stmtQueryValidDerivers.create(db,
@@ -583,55 +580,6 @@ uint64_t LocalStore::addValidPath(State & state,
}


void LocalStore::registerFailedPath(const Path & path)
{
    retrySQLite<void>([&]() {
        auto state(_state.lock());
        state->stmtRegisterFailedPath.use()(path)(time(0)).step();
    });
}


bool LocalStore::hasPathFailed(const Path & path)
{
    return retrySQLite<bool>([&]() {
        auto state(_state.lock());
        return state->stmtHasPathFailed.use()(path).next();
    });
}


PathSet LocalStore::queryFailedPaths()
{
    return retrySQLite<PathSet>([&]() {
        auto state(_state.lock());

        auto useQueryFailedPaths(state->stmtQueryFailedPaths.use());

        PathSet res;
        while (useQueryFailedPaths.next())
            res.insert(useQueryFailedPaths.getStr(0));

        return res;
    });
}


void LocalStore::clearFailedPaths(const PathSet & paths)
{
    retrySQLite<void>([&]() {
        auto state(_state.lock());

        SQLiteTxn txn(state->db);

        for (auto & path : paths)
            state->stmtClearFailedPath.use()(path).exec();

        txn.commit();
    });
}


Hash parseHashField(const Path & path, const string & s)
{
    string::size_type colon = s.find(':');

src/libstore/local-store.hh

@@ -17,8 +17,8 @@ namespace nix {
/* Nix store and database schema version.  Version 1 (or 0) was Nix <=
   0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
   Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
   Nix 1.0.  Version 7 is Nix 1.3. Version 8 is 1.12. */
const int nixSchemaVersion = 8;
   Nix 1.0.  Version 7 is Nix 1.3. Version 9 is 1.12. */
const int nixSchemaVersion = 9;


extern string drvsLogDir;
@@ -71,10 +71,6 @@ private:
        SQLiteStmt stmtQueryReferences;
        SQLiteStmt stmtQueryReferrers;
        SQLiteStmt stmtInvalidatePath;
        SQLiteStmt stmtRegisterFailedPath;
        SQLiteStmt stmtHasPathFailed;
        SQLiteStmt stmtQueryFailedPaths;
        SQLiteStmt stmtClearFailedPath;
        SQLiteStmt stmtAddDerivationOutput;
        SQLiteStmt stmtQueryValidDerivers;
        SQLiteStmt stmtQueryDerivationOutputs;
@@ -194,17 +190,6 @@ public:

    void registerValidPaths(const ValidPathInfos & infos);

    /* Register that the build of a derivation with output `path' has
       failed. */
    void registerFailedPath(const Path & path);

    /* Query whether `path' previously failed to build. */
    bool hasPathFailed(const Path & path);

    PathSet queryFailedPaths() override;

    void clearFailedPaths(const PathSet & paths) override;

    void vacuumDB();

    /* Repair the contents of the given path by redownloading it using

src/libstore/remote-store.cc

@@ -520,23 +520,6 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
}


PathSet RemoteStore::queryFailedPaths()
{
    auto conn(connections->get());
    conn->to << wopQueryFailedPaths;
    conn->processStderr();
    return readStorePaths<PathSet>(conn->from);
}


void RemoteStore::clearFailedPaths(const PathSet & paths)
{
    auto conn(connections->get());
    conn->to << wopClearFailedPaths << paths;
    conn->processStderr();
    readInt(conn->from);
}

void RemoteStore::optimiseStore()
{
    auto conn(connections->get());
@@ -545,6 +528,7 @@ void RemoteStore::optimiseStore()
    readInt(conn->from);
}


bool RemoteStore::verifyStore(bool checkContents, bool repair)
{
    auto conn(connections->get());

src/libstore/remote-store.hh

@@ -85,10 +85,6 @@ public:

    void collectGarbage(const GCOptions & options, GCResults & results) override;

    PathSet queryFailedPaths() override;

    void clearFailedPaths(const PathSet & paths) override;

    void optimiseStore() override;

    bool verifyStore(bool checkContents, bool repair) override;

src/libstore/schema.sql

@@ -39,8 +39,3 @@ create table if not exists DerivationOutputs (
);

create index if not exists IndexDerivationOutputs on DerivationOutputs(path);

create table if not exists FailedPaths (
    path text primary key not null,
    time integer not null
);