Merge pull request #19 from obsidiansystems/ipfs-add-body-to-network-…

…errors

Add body to network errors

src/libstore/daemon.cc

@@ -721,9 +721,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
             source = std::make_unique<TunnelSource>(from, to);
         else {
-            TeeSink tee(from);
-            parseDump(tee, tee.source);
-            saved = std::move(*tee.source.data);
+            TeeSource tee(from);
+            ParseSink sink;
+            parseDump(sink, tee);
+            saved = std::move(*tee.data);
             source = std::make_unique<StringSource>(saved);
         }
 

src/libstore/export-import.cc

@@ -77,8 +77,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
 
         /* Extract the NAR from the source. */
-        TeeSink tee(source);
-        parseDump(tee, tee.source);
+        TeeSource tee(source);
+        ParseSink sink;
+        parseDump(sink, tee);
 
         uint32_t magic = readInt(source);
         if (magic != exportMagic)
@@ -94,15 +95,15 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
 
-        info.narHash = hashString(htSHA256, *tee.source.data);
-        info.narSize = tee.source.data->size();
+        info.narHash = hashString(htSHA256, *tee.data);
+        info.narSize = tee.data->size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
         // Can't use underlying source, which would have been exhausted
-        auto source = StringSource { *tee.source.data };
+        auto source = StringSource { *tee.data };
         addToStore(info, source, NoRepair, checkSigs, accessor);
 
         res.push_back(info.path);

src/libstore/filetransfer.cc

@@ -58,7 +58,6 @@ struct curlFileTransfer : public FileTransfer
         Callback<FileTransferResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
-        std::string status;
 
         unsigned int attempt = 0;
 
@@ -125,7 +124,7 @@ struct curlFileTransfer : public FileTransfer
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
+                    fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -155,8 +154,18 @@ struct curlFileTransfer : public FileTransfer
                 size_t realSize = size * nmemb;
                 result.bodySize += realSize;
 
-                if (!decompressionSink)
+                if (!decompressionSink) {
                     decompressionSink = makeDecompressionSink(encoding, finalSink);
+                    if (! successfulStatuses.count(getHTTPStatus())) {
+                        // In this case we want to construct a TeeSink, to keep
+                        // the response around (which we figure won't be big
+                        // like an actual download should be) to improve error
+                        // messages.
+                        decompressionSink = std::make_shared<TeeSink<ref<CompressionSink>>>(
+                            ref<CompressionSink>{ decompressionSink }
+                        );
+                    }
+                }
 
                 (*decompressionSink)((unsigned char *) contents, realSize);
 
@@ -177,6 +186,7 @@ struct curlFileTransfer : public FileTransfer
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
             printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
+            std::string status;
             if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
                 result.etag = "";
                 auto ss = tokenizeString<vector<string>>(line, " ");
@@ -420,14 +430,18 @@ struct curlFileTransfer : public FileTransfer
 
                 attempt++;
 
+                std::shared_ptr<std::string> response;
+                if (decompressionSink)
+                    if (auto teeSink = std::dynamic_pointer_cast<TeeSink<std::shared_ptr<CompressionSink>>>(decompressionSink))
+                        response = teeSink->data;
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? FileTransferError(Interrupted, "%s of '%s' was interrupted", request.verb(), request.uri)
+                    ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
                     : httpStatus != 0
-                    ? FileTransferError(err, "unable to %s '%s': HTTP error %d%s",
+                    ? FileTransferError(err, response, "unable to %s '%s': HTTP error %d%s",
                         request.verb(), request.uri, httpStatus,
                         code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
-                    : FileTransferError(err, "unable to %s '%s': %s (%d)",
+                    : FileTransferError(err, response, "unable to %s '%s': %s (%d)",
                         request.verb(), request.uri, curl_easy_strerror(code), code);
 
                 /* If this is a transient error, then maybe retry the
@@ -684,7 +698,7 @@ struct curlFileTransfer : public FileTransfer
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
-                    throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri);
+                    throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
                 res.data = s3Res.data;
                 callback(std::move(res));
 #else
@@ -849,6 +863,18 @@ std::string FileTransfer::urlEncode(const std::string & param) {
     throw URLEncodeError("not implemented");
 }
 
+template<typename... Args>
+FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
+    : Error(args...), error(error), response(response)
+{
+    const auto hf = hintfmt(args...);
+    if (response) {
+        err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), response);
+    } else {
+        err.hint = hf;
+    }
+}
+
 bool isUri(const string & s)
 {
     if (s.compare(0, 8, "channel:") == 0) return true;

src/libstore/filetransfer.hh

@@ -106,10 +106,12 @@ class FileTransferError : public Error
 {
 public:
     FileTransfer::Error error;
+    std::shared_ptr<string> response; // intentionally optional
+
     template<typename... Args>
-    FileTransferError(FileTransfer::Error error, const Args & ... args)
-        : Error(args...), error(error)
-    { }
+    FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
+
+    virtual const char* sname() const override { return "FileTransferError"; }
 };
 
 bool isUri(const string & s);

src/libutil/archive.hh

@@ -63,13 +63,6 @@ struct ParseSink
     virtual void createSymlink(const Path & path, const string & target) { };
 };
 
-struct TeeSink : ParseSink
-{
-    TeeSource source;
-
-    TeeSink(Source & source) : source(source) { }
-};
-
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);

src/libutil/compression.hh

@@ -25,4 +25,16 @@ MakeError(UnknownCompressionMethod, Error);
 
 MakeError(CompressionError, Error);
 
+template<>
+struct TeeSink<ref<CompressionSink>> : CompressionSink
+{
+    MAKE_TEE_SINK(ref<CompressionSink>);
+    void finish() override {
+        orig->finish();
+    }
+    void write(const unsigned char * data, size_t len) override {
+        return orig->write(data, len);
+    }
+};
+
 }

src/libutil/error.hh

@@ -173,9 +173,8 @@ public:
 
     template<typename... Args>
     SysError(const Args & ... args)
-      :Error("")
+        : Error(""), errNo(errno)
     {
-        errNo = errno;
         auto hf = hintfmt(args...);
         err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
     }

src/libutil/serialise.hh

@@ -181,6 +181,28 @@ struct TeeSource : Source
     }
 };
 
+#define MAKE_TEE_SINK(T) \
+	T orig; \
+    ref<std::string> data; \
+    TeeSink(T && orig) \
+        : orig(std::move(orig)), data(make_ref<std::string>()) { } \
+    void operator () (const unsigned char * data, size_t len) { \
+        this->data->append((const char *) data, len); \
+        (*this->orig)(data, len); \
+    } \
+    void operator () (const std::string & s) \
+    { \
+        *data += s; \
+        (*this->orig)(s); \
+    }
+
+template<typename T>
+struct TeeSink : Sink
+{
+    MAKE_TEE_SINK(T);
+};
+
+
 /* A reader that consumes the original Source until 'size'. */
 struct SizedSource : Source
 {