Replace hasSubstitutes() with querySubstitutablePaths()

querySubstitutablePaths() takes a set of paths, so this greatly
reduces daemon <-> client latency.

Author: edolstra

src/libstore/local-store.cc

@@ -932,16 +932,24 @@ template<class T> T getIntLine(int fd)
 }
 
 
-bool LocalStore::hasSubstitutes(const Path & path)
+PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 {
+    PathSet res;
     foreach (Paths::iterator, i, substituters) {
+        if (res.size() == paths.size()) break;
         RunningSubstituter & run(runningSubstituters[*i]);
         startSubstituter(*i, run);
-        writeLine(run.to, "have\n" + path);
-        if (getIntLine<int>(run.from)) return true;
+        string s = "have ";
+        foreach (PathSet::const_iterator, i, paths)
+            if (res.find(*i) == res.end()) { s += *i; s += " "; }
+        writeLine(run.to, s);
+        while (true) {
+            Path path = readLine(run.from);
+            if (path == "") break;
+            res.insert(path);
+        }
     }
-
-    return false;
+    return res;
 }
 
 

src/libstore/local-store.hh

@@ -123,9 +123,7 @@ public:
 
     StringSet queryDerivationOutputNames(const Path & path);
 
-    PathSet querySubstitutablePaths();
-    
-    bool hasSubstitutes(const Path & path);
+    PathSet querySubstitutablePaths(const PathSet & paths);
 
     void querySubstitutablePathInfos(const Path & substituter,
         PathSet & paths, SubstitutablePathInfos & infos);

src/libstore/remote-store.cc

@@ -219,13 +219,13 @@ bool RemoteStore::isValidPath(const Path & path)
 
 PathSet RemoteStore::queryValidPaths(const PathSet & paths)
 {
+    openConnection();
     if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
         PathSet res;
         foreach (PathSet::const_iterator, i, paths)
             if (isValidPath(*i)) res.insert(*i);
         return res;
     } else {
-        openConnection();
         writeInt(wopQueryValidPaths, to);
         writeStrings(paths, to);
         processStderr();
@@ -243,14 +243,24 @@ PathSet RemoteStore::queryAllValidPaths()
 }
 
 
-bool RemoteStore::hasSubstitutes(const Path & path)
+PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
 {
     openConnection();
-    writeInt(wopHasSubstitutes, to);
-    writeString(path, to);
-    processStderr();
-    unsigned int reply = readInt(from);
-    return reply != 0;
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+        PathSet res;
+        foreach (PathSet::const_iterator, i, paths) {
+            writeInt(wopHasSubstitutes, to);
+            writeString(*i, to);
+            processStderr();
+            if (readInt(from)) res.insert(*i);
+        }
+        return res;
+    } else {
+        writeInt(wopQuerySubstitutablePaths, to);
+        writeStrings(paths, to);
+        processStderr();
+        return readStorePaths<PathSet>(from);
+    }
 }
 
 

src/libstore/remote-store.hh

@@ -45,7 +45,7 @@ public:
 
     StringSet queryDerivationOutputNames(const Path & path);
 
-    bool hasSubstitutes(const Path & path);
+    PathSet querySubstitutablePaths(const PathSet & paths);
 
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos);

src/libstore/store-api.hh

@@ -145,8 +145,8 @@ public:
     /* Query the output names of the derivation denoted by `path'. */
     virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
 
-    /* Query whether a path has substitutes. */
-    virtual bool hasSubstitutes(const Path & path) = 0;
+    /* Query which of the given paths have substitutes. */
+    virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
 
     /* Query substitute info (i.e. references, derivers and download
        sizes) of a set of paths.  If a path does not have substitute

src/libstore/worker-protocol.hh

@@ -42,6 +42,7 @@ typedef enum {
     wopQueryDerivationOutputNames = 28,
     wopQuerySubstitutablePathInfos = 29,
     wopQueryValidPaths = 30,
+    wopQuerySubstitutablePaths = 31,
 } WorkerOp;
 
 

src/nix-env/nix-env.cc

@@ -211,9 +211,12 @@ static int comparePriorities(EvalState & state,
 
 static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
 {
+    assert(false);
+#if 0
     return
         store->isValidPath(elem.queryOutPath(state)) ||
         store->hasSubstitutes(elem.queryOutPath(state));
+#endif
 }
 
 
@@ -931,8 +934,7 @@ static void opQuery(Globals & globals,
 
 
     /* Query which paths have substitutes. */
-    SubstitutablePathInfos subs;
-    PathSet validPaths;
+    PathSet validPaths, substitutablePaths;
     if (printStatus) {
         PathSet paths;
         foreach (vector<DrvInfo>::iterator, i, elems2)
@@ -943,7 +945,7 @@ static void opQuery(Globals & globals,
                 i->setFailed();
             }
         validPaths = store->queryValidPaths(paths);
-        store->querySubstitutablePathInfos(paths, subs);
+        substitutablePaths = store->querySubstitutablePaths(paths);
     }
 
 
@@ -969,7 +971,7 @@ static void opQuery(Globals & globals,
 
             if (printStatus) {
                 Path outPath = i->queryOutPath(globals.state);
-                bool hasSubs = subs.find(outPath) != subs.end();
+                bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
                 bool isInstalled = installed.find(outPath) != installed.end();
                 bool isValid = validPaths.find(outPath) != validPaths.end();
                 if (xmlOutput) {

src/nix-worker/nix-worker.cc

@@ -309,12 +309,21 @@ static void performOp(unsigned int clientVersion,
     case wopHasSubstitutes: {
         Path path = readStorePath(from);
         startWork();
-        bool result = store->hasSubstitutes(path);
+        PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path));
         stopWork();
-        writeInt(result, to);
+        writeInt(res.find(path) != res.end(), to);
         break;
     }
 
+    case wopQuerySubstitutablePaths: {
+        PathSet paths = readStorePaths<PathSet>(from);
+        startWork();
+        PathSet res = store->querySubstitutablePaths(paths);
+        stopWork();
+        writeStrings(res, to);
+        break;
+    }
+        
     case wopQueryPathHash: {
         Path path = readStorePath(from);
         startWork();