diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ae483c95efc..a47d3fd5761 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -1,5 +1,6 @@ #include "archive.hh" #include "binary-cache-store.hh" +#include "canon-path.hh" #include "compression.hh" #include "derivations.hh" #include "source-accessor.hh" @@ -62,9 +63,10 @@ void BinaryCacheStore::init() void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, - const std::string & mimeType) + const std::string & mimeType, + std::map tags) { - upsertFile(path, std::make_shared(std::move(data)), mimeType); + upsertFile(path, std::make_shared(std::move(data)), mimeType, tags); } void BinaryCacheStore::getFile(const std::string & path, @@ -105,11 +107,11 @@ std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) return std::string(storePath.hashPart()) + ".narinfo"; } -void BinaryCacheStore::writeNarInfo(ref narInfo) +void BinaryCacheStore::writeNarInfo(ref narInfo, std::map tags) { auto narInfoFile = narInfoFileFor(narInfo->path); - upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); + upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo", tags); { auto state_(state.lock()); @@ -138,6 +140,8 @@ ref BinaryCacheStore::addToStoreCommon( AutoDelete autoDelete(fnTemp); + const CanonPath tagsFile { "/nix-support/tags.json" }; + auto now1 = std::chrono::steady_clock::now(); /* Read the NAR simultaneously into a CompressionSink+FileSink (to @@ -152,7 +156,8 @@ ref BinaryCacheStore::addToStoreCommon( auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; - narAccessor = makeNarAccessor(teeSource); + // index the nar, and keep the content of tagsFile in memory in case it is present. + narAccessor = makeNarAccessor(teeSource, [&tagsFile](CanonPath& path) { return path == tagsFile; }); compressionSink->finish(); fileSink.flush(); } @@ -191,6 +196,23 @@ ref BinaryCacheStore::addToStoreCommon( printStorePath(info.path), printStorePath(ref)); } + /* load tags if applicable */ + std::map tags; + auto lstat = narAccessor->maybeLstat(tagsFile); + if (lstat.has_value() && lstat->type == SourceAccessor::Type::tRegular) { + std::string tags_str = narAccessor->readFile(tagsFile); + try { + nlohmann::json tags_json = nlohmann::json::parse(tags_str); + for (const auto& kv: tags_json.items()) { + std::string key = kv.key(); + std::string value = kv.value().template get(); + tags[key] = value; + } + } catch (const nlohmann::json::exception& e) { + printMsg(lvlWarn, "could not read nar tags from %s in %s: %s", tagsFile, printStorePath(info.path), e.what()); + } + } + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -199,7 +221,7 @@ ref BinaryCacheStore::addToStoreCommon( {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; - upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json", tags); } /* Optionally maintain an index of DWARF debug info files @@ -226,7 +248,7 @@ ref BinaryCacheStore::addToStoreCommon( printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); - upsertFile(key, json.dump(), "application/json"); + upsertFile(key, json.dump(), "application/json", tags); }; std::regex regex1("^[0-9a-f]{2}$"); @@ -264,7 +286,7 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWrite++; upsertFile(narInfo->url, std::make_shared(fnTemp, std::ios_base::in | std::ios_base::binary), - "application/x-nix-nar"); + "application/x-nix-nar", tags); } else stats.narWriteAverted++; @@ -275,7 +297,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Atomically write the NAR info file.*/ if (secretKey) narInfo->sign(*this, *secretKey); - writeNarInfo(narInfo); + writeNarInfo(narInfo, tags); stats.narInfoWrite++; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index cea2a571f13..76d6f978441 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -73,12 +73,14 @@ public: virtual void upsertFile(const std::string & path, std::shared_ptr> istream, - const std::string & mimeType) = 0; + const std::string & mimeType, + std::map tags = {}) = 0; void upsertFile(const std::string & path, // FIXME: use std::string_view std::string && data, - const std::string & mimeType); + const std::string & mimeType, + std::map tags = {}); /** * Dump the contents of the specified file to a sink. @@ -105,7 +107,7 @@ private: std::string narInfoFileFor(const StorePath & storePath); - void writeNarInfo(ref narInfo); + void writeNarInfo(ref narInfo, std::map tags={}); ref addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 85c5eed4c01..c1b527e7b8e 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -130,8 +130,11 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v void upsertFile(const std::string & path, std::shared_ptr> istream, - const std::string & mimeType) override + const std::string & mimeType, + std::map tags) override { + (void)tags; + auto req = makeRequest(path); req.data = StreamToSourceAdapter(istream).drain(); req.mimeType = mimeType; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 5481dd762e2..b6c3682465e 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -56,8 +56,11 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public void upsertFile(const std::string & path, std::shared_ptr> istream, - const std::string & mimeType) override + const std::string & mimeType, + std::map tags) override { + (void)tags; + auto path2 = binaryCacheDir + "/" + path; static std::atomic counter{0}; Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 15b05fe25fe..07430183e42 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -17,6 +17,8 @@ struct NarMember /* If this is a directory, all the children of the directory. */ std::map children; + + std::optional content; }; struct NarAccessor : public SourceAccessor @@ -27,6 +29,8 @@ struct NarAccessor : public SourceAccessor NarMember root; + std::optional> loadContentsFilter; + struct NarIndexer : ParseSink, Source { NarAccessor & acc; @@ -38,6 +42,8 @@ struct NarAccessor : public SourceAccessor uint64_t pos = 0; + std::optional currentRegularFile; + NarIndexer(NarAccessor & acc, Source & source) : acc(acc), source(source) { } @@ -76,10 +82,13 @@ struct NarAccessor : public SourceAccessor .isExecutable = false, .narOffset = 0 } }); + currentRegularFile = CanonPath(path); } void closeRegularFile() override - { } + { + currentRegularFile = std::nullopt; + } void isExecutable() override { @@ -94,7 +103,13 @@ struct NarAccessor : public SourceAccessor } void receiveContents(std::string_view data) override - { } + { + if (acc.loadContentsFilter != std::nullopt) { + if (acc.loadContentsFilter.value()(currentRegularFile.value())) { + parents.top()->content = std::string(data); + } + } + } void createSymlink(const Path & path, const std::string & target) override { @@ -119,7 +134,7 @@ struct NarAccessor : public SourceAccessor parseDump(indexer, indexer); } - NarAccessor(Source & source) + NarAccessor(Source & source, std::optional> _loadContentsFilter): loadContentsFilter(_loadContentsFilter) { NarIndexer indexer(*this, source); parseDump(indexer, indexer); @@ -208,8 +223,13 @@ struct NarAccessor : public SourceAccessor if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); - assert(nar); - return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); + if (nar) { + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); + } + + if (i.content != std::nullopt) return i.content.value(); + + throw Error("content of path %1% is not available with this NarAccessor", path); } std::string readLink(const CanonPath & path) override @@ -226,9 +246,9 @@ ref makeNarAccessor(std::string && nar) return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source, std::optional> loadContentsFilter) { - return make_ref(source); + return make_ref(source, loadContentsFilter); } ref makeLazyNarAccessor(const std::string & listing, diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 0043897c658..b2281f67e3e 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -4,6 +4,7 @@ #include "source-accessor.hh" #include +#include #include @@ -17,7 +18,16 @@ struct Source; */ ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +/** + * Return an object that provides access to the contents of a NAR + * file. + * + * the readFile() member function only works for files where loadContentsFilter + * return true. The content of those files is stored in memory. For + * other files, and if loadContentsFilter is nullopt, readFile() throws + * an error. + */ +ref makeNarAccessor(Source & source, std::optional> loadContentsFilter = std::nullopt); /** * Create a NAR accessor from a NAR listing (in the format produced by diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1a62d92d44a..54381cb9303 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "url.hh" #include #include @@ -347,12 +348,27 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void uploadFile(const std::string & path, std::shared_ptr> istream, const std::string & mimeType, - const std::string & contentEncoding) + const std::string & contentEncoding, + std::map tags) { istream->seekg(0, istream->end); auto size = istream->tellg(); istream->seekg(0, istream->beg); + std::map actual_tags; + const std::string prefix = "nix:"; + for (const auto& kv: tags) { + if (kv.first.size() > 128 - prefix.size()) { + printMsg(lvlWarn, "tag %s in %s is too long for s3 upload", kv.first, path); + continue; + } + if (kv.second.size() > 256) { + printMsg(lvlWarn, "tag value %s in %s is too long for s3 upload", kv.second, path); + continue; + } + actual_tags[prefix + kv.first] = kv.second; + } + auto maxThreads = std::thread::hardware_concurrency(); static std::shared_ptr @@ -418,6 +434,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual if (contentEncoding != "") request.SetContentEncoding(contentEncoding); + if (!actual_tags.empty()) { + request.SetTagging(encodeQuery(actual_tags)); + } + + request.SetBody(istream); auto result = checkAws(fmt("AWS error uploading '%s'", path), @@ -440,7 +461,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void upsertFile(const std::string & path, std::shared_ptr> istream, - const std::string & mimeType) override + const std::string & mimeType, + std::map tags) override { auto compress = [&](std::string compression) { @@ -449,13 +471,13 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual }; if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); + uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression, tags); else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, compress(lsCompression), mimeType, lsCompression); + uploadFile(path, compress(lsCompression), mimeType, lsCompression, tags); else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, compress(logCompression), mimeType, logCompression); + uploadFile(path, compress(logCompression), mimeType, logCompression, tags); else - uploadFile(path, istream, mimeType, ""); + uploadFile(path, istream, mimeType, "", tags); } void getFile(const std::string & path, Sink & sink) override diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 833f546787b..1ce661883a2 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -32,6 +32,7 @@ std::string percentDecode(std::string_view in); std::string percentEncode(std::string_view s, std::string_view keep=""); std::map decodeQuery(const std::string & query); +std::string encodeQuery(const std::map & params); ParsedURL parseURL(const std::string & url);