tag nars containing only lib/debug as hint=debug when uploading to s3

this allows setting a different retention policy for debug symbols in s3
backed binary caches.

doc/manual/src/package-management/s3-substituter.md

@@ -117,6 +117,7 @@ Your account will need the following IAM policy to upload to the cache:
         "s3:ListBucket",
         "s3:ListBucketMultipartUploads",
         "s3:ListMultipartUploadParts",
+        "s3:PutObjectTagging",
         "s3:PutObject"
       ],
       "Resource": [
@@ -128,6 +129,11 @@ Your account will need the following IAM policy to upload to the cache:
 }
 ```
 
+The `s3:PutObjectTagging` is only needed if the `index-debug-info` option is
+set to true and `multipart-upload` is set to false. In this case, nars
+containing only debug symbols will be tagged as `hint=debug`. You can then
+select a different retention policy for debug symbols.
+
 ## Examples
 
 To upload with a specific credential profile for Amazon S3:

src/libstore/binary-cache-store.cc

@@ -39,7 +39,7 @@ void BinaryCacheStore::init()
 
     auto cacheInfo = getFile(cacheInfoFile);
     if (!cacheInfo) {
-        upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
+        upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info", "");
     } else {
         for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
             size_t colon= line.find(':');
@@ -61,9 +61,10 @@ void BinaryCacheStore::init()
 
 void BinaryCacheStore::upsertFile(const std::string & path,
     std::string && data,
-    const std::string & mimeType)
+    const std::string & mimeType,
+    const std::string & hint)
 {
-    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+    upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType, hint);
 }
 
 void BinaryCacheStore::getFile(const std::string & path,
@@ -104,11 +105,11 @@ std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
     return std::string(storePath.hashPart()) + ".narinfo";
 }
 
-void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
+void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo, const std::string & hint)
 {
     auto narInfoFile = narInfoFileFor(narInfo->path);
 
-    upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
+    upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo", hint);
 
     {
         auto state_(state.lock());
@@ -190,16 +191,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
                 printStorePath(info.path), printStorePath(ref));
         }
 
-    /* Optionally write a JSON file containing a listing of the
-       contents of the NAR. */
-    if (writeNARListing) {
-        nlohmann::json j = {
-            {"version", 1},
-            {"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
-        };
-
-        upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
-    }
+    std::string hint = "";
 
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
@@ -209,6 +201,12 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
         std::string buildIdDir = "/lib/debug/.build-id";
 
         if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
+            // if this nar only contains lib/debug then mark it as debug
+            if (narAccessor->readDirectory("") == std::set<std::string> {"lib"}
+                    && narAccessor->readDirectory("/lib") == std::set<std::string> {"debug"}) {
+                hint = "debug";
+                printMsg(lvlDebug, "hinting nar %s as debug", printStorePath(narInfo->path));
+            }
 
             ThreadPool threadPool(25);
 
@@ -225,7 +223,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
 
                 printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
 
-                upsertFile(key, json.dump(), "application/json");
+                upsertFile(key, json.dump(), "application/json", hint);
             };
 
             std::regex regex1("^[0-9a-f]{2}$");
@@ -258,12 +256,24 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
         }
     }
 
+    /* Optionally write a JSON file containing a listing of the
+       contents of the NAR. */
+    if (writeNARListing) {
+        nlohmann::json j = {
+            {"version", 1},
+            {"root", listNar(ref<FSAccessor>(narAccessor), "", true)},
+        };
+
+        upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json", hint);
+    }
+
+
     /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
         upsertFile(narInfo->url,
             std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
-            "application/x-nix-nar");
+            "application/x-nix-nar", hint);
     } else
         stats.narWriteAverted++;
 
@@ -274,7 +284,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     /* Atomically write the NAR info file.*/
     if (secretKey) narInfo->sign(*this, *secretKey);
 
-    writeNarInfo(narInfo);
+    writeNarInfo(narInfo, hint);
 
     stats.narInfoWrite++;
 
@@ -499,7 +509,8 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
 
     narInfo->sigs.insert(sigs.begin(), sigs.end());
 
-    writeNarInfo(narInfo);
+    // FIXME: removes the hint
+    writeNarInfo(narInfo, "");
 }
 
 std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath & path)

src/libstore/binary-cache-store.hh

@@ -51,12 +51,16 @@ public:
 
     virtual void upsertFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType) = 0;
+        const std::string & mimeType,
+        // "debug" for debug info
+        const std::string & hint = "") = 0;
 
     void upsertFile(const std::string & path,
         // FIXME: use std::string_view
         std::string && data,
-        const std::string & mimeType);
+        const std::string & mimeType,
+        // "debug" for debug info
+        const std::string & hint = "");
 
     /* Note: subclasses must implement at least one of the two
        following getFile() methods. */
@@ -82,7 +86,7 @@ private:
 
     std::string narInfoFileFor(const StorePath & storePath);
 
-    void writeNarInfo(ref<NarInfo> narInfo);
+    void writeNarInfo(ref<NarInfo> narInfo, const std::string & hint);
 
     ref<const ValidPathInfo> addToStoreCommon(
         Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,

src/libstore/http-binary-cache-store.cc

@@ -123,8 +123,10 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v
 
     void upsertFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType) override
+        const std::string & mimeType,
+        const std::string & hint) override
     {
+        (void)hint;
         auto req = makeRequest(path);
         req.data = StreamToSourceAdapter(istream).drain();
         req.mimeType = mimeType;

src/libstore/local-binary-cache-store.cc

@@ -49,8 +49,10 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public
 
     void upsertFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType) override
+        const std::string & mimeType,
+        const std::string & hint) override
     {
+        (void)hint;
         auto path2 = binaryCacheDir + "/" + path;
         static std::atomic<int> counter{0};
         Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);

src/libstore/s3-binary-cache-store.cc

@@ -7,6 +7,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
+#include "url.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -294,12 +295,19 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
     void uploadFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
         const std::string & mimeType,
-        const std::string & contentEncoding)
+        const std::string & contentEncoding,
+        const std::string & hint)
     {
         istream->seekg(0, istream->end);
         auto size = istream->tellg();
         istream->seekg(0, istream->beg);
 
+        std::string tag;
+        if (!hint.empty()) {
+            std::map<std::string, std::string> params { {"hint", hint} };
+            tag = encodeQuery(params);
+        }
+
         auto maxThreads = std::thread::hardware_concurrency();
 
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@@ -365,6 +373,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
             if (contentEncoding != "")
                 request.SetContentEncoding(contentEncoding);
 
+            if (tag != "")
+                request.SetTagging(tag);
+
             request.SetBody(istream);
 
             auto result = checkAws(fmt("AWS error uploading '%s'", path),
@@ -387,7 +398,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
 
     void upsertFile(const std::string & path,
         std::shared_ptr<std::basic_iostream<char>> istream,
-        const std::string & mimeType) override
+        const std::string & mimeType,
+        const std::string & hint) override
     {
         auto compress = [&](std::string compression)
         {
@@ -396,13 +408,13 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
         };
 
         if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
-            uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
+            uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression, hint);
         else if (lsCompression != "" && hasSuffix(path, ".ls"))
-            uploadFile(path, compress(lsCompression), mimeType, lsCompression);
+            uploadFile(path, compress(lsCompression), mimeType, lsCompression, hint);
         else if (logCompression != "" && hasPrefix(path, "log/"))
-            uploadFile(path, compress(logCompression), mimeType, logCompression);
+            uploadFile(path, compress(logCompression), mimeType, logCompression, hint);
         else
-            uploadFile(path, istream, mimeType, "");
+            uploadFile(path, istream, mimeType, "", hint);
     }
 
     void getFile(const std::string & path, Sink & sink) override

src/libutil/url.hh

@@ -25,6 +25,7 @@ std::string percentDecode(std::string_view in);
 std::string percentEncode(std::string_view s, std::string_view keep="");
 
 std::map<std::string, std::string> decodeQuery(const std::string & query);
+std::string encodeQuery(const std::map<std::string, std::string> & params);
 
 ParsedURL parseURL(const std::string & url);