Use azcopy for uploading large artifacts (#1679)

dg0yt · web-flow · commit 59c905a8aa64 · 2025-06-16T21:54:37.000-07:00
diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h
@@ -116,6 +116,11 @@ namespace vcpkg
                               const Path& file_to_put,
                               StringView sha512);
 
+    bool azcopy_to_asset_cache(DiagnosticContext& context,
+                               StringView raw_url,
+                               const SanitizedUrl& sanitized_url,
+                               const Path& file);
+
     Optional<unsigned long long> try_parse_curl_max5_size(StringView sv);
 
     struct CurlProgressData
diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h
@@ -327,6 +327,10 @@ DECLARE_MESSAGE(AVersionDatabaseEntry, (), "", "a version database entry")
 DECLARE_MESSAGE(AVersionObject, (), "", "a version object")
 DECLARE_MESSAGE(AVersionOfAnyType, (), "", "a version of any type")
 DECLARE_MESSAGE(AVersionConstraint, (), "", "a version constraint")
+DECLARE_MESSAGE(AzcopyFailedToPutBlob,
+                (msg::exit_code, msg::url, msg::value),
+                "azcopy is the name of a program. {value} is an HTTP status code.",
+                "azcopy failed to upload a file to {url} with exit code {exit_code} and http code {value}.")
 DECLARE_MESSAGE(AzUrlAssetCacheRequiresBaseUrl,
                 (),
                 "",
diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h
@@ -148,6 +148,8 @@ namespace vcpkg
         std::vector<UrlTemplate> url_templates_to_get;
         std::vector<UrlTemplate> url_templates_to_put;
 
+        std::vector<UrlTemplate> azblob_templates_to_put;
+
         std::vector<std::string> gcs_read_prefixes;
         std::vector<std::string> gcs_write_prefixes;
 
diff --git a/locales/messages.json b/locales/messages.json
@@ -211,6 +211,8 @@
   "AvailableHelpTopics": "Available help topics:",
   "AzUrlAssetCacheRequiresBaseUrl": "unexpected arguments: asset config 'azurl' requires a base url",
   "AzUrlAssetCacheRequiresLessThanFour": "unexpected arguments: asset config 'azurl' requires fewer than 4 arguments",
+  "AzcopyFailedToPutBlob": "azcopy failed to upload a file to {url} with exit code {exit_code} and http code {value}.",
+  "_AzcopyFailedToPutBlob.comment": "azcopy is the name of a program. {value} is an HTTP status code. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
   "BaselineConflict": "Specifying vcpkg-configuration.default-registry in a manifest file conflicts with built-in baseline.\nPlease remove one of these conflicting settings.",
   "BaselineGitShowFailed": "while checking out baseline from commit '{commit_sha}', failed to `git show` versions/baseline.json. This may be fixed by fetching commits with `git fetch`.",
   "_BaselineGitShowFailed.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
diff --git a/src/vcpkg-test/configparser.cpp b/src/vcpkg-test/configparser.cpp
@@ -459,8 +459,9 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]")
 
         REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azblob"}, {"default"}});
         CHECK(state.url_templates_to_get.empty());
-        CHECK(state.url_templates_to_put.size() == 1);
-        CHECK(state.url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas");
+        CHECK(state.url_templates_to_put.empty());
+        CHECK(state.azblob_templates_to_put.size() == 1);
+        CHECK(state.azblob_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas");
         REQUIRE(state.secrets == std::vector<std::string>{"sas"});
         REQUIRE(!state.archives_to_write.empty());
     }
@@ -471,8 +472,9 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]")
         REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azblob"}, {"default"}});
         CHECK(state.url_templates_to_get.size() == 1);
         CHECK(state.url_templates_to_get.front().url_template == "https://azure/container/{sha}.zip?sas");
-        CHECK(state.url_templates_to_put.size() == 1);
-        CHECK(state.url_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas");
+        CHECK(state.url_templates_to_put.empty());
+        CHECK(state.azblob_templates_to_put.size() == 1);
+        CHECK(state.azblob_templates_to_put.front().url_template == "https://azure/container/{sha}.zip?sas");
         REQUIRE(state.secrets == std::vector<std::string>{"sas"});
         REQUIRE(!state.archives_to_read.empty());
         REQUIRE(!state.archives_to_write.empty());
diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp
@@ -2,9 +2,22 @@
 
 #include <vcpkg/base/downloads.h>
 #include <vcpkg/base/expected.h>
+#include <vcpkg/base/system.h>
+#include <vcpkg/base/util.h>
+
+#include <random>
 
 using namespace vcpkg;
 
+#define CHECK_EC_ON_FILE(file, ec)                                                                                     \
+    do                                                                                                                 \
+    {                                                                                                                  \
+        if (ec)                                                                                                        \
+        {                                                                                                              \
+            FAIL((file).native() << ": " << (ec).message());                                                           \
+        }                                                                                                              \
+    } while (0)
+
 TEST_CASE ("parse_split_url_view", "[downloads]")
 {
     {
@@ -313,3 +326,83 @@ TEST_CASE ("url_encode_spaces", "[downloads]")
     REQUIRE(url_encode_spaces("https://example.com/a  space/b?query=value&query2=value2") ==
             "https://example.com/a%20%20space/b?query=value&query2=value2");
 }
+
+/*
+ * To run this test:
+ * - Set environment variables VCPKG_TEST_AZBLOB_URL and VCPKG_TEST_AZBLOB_SAS.
+ *   (Use Azurite for creating a local test environment, and
+ *   Azure Storage Explorer for getting a suitable Shared Access Signature.)
+ * - Run 'vcpkg-test azblob [-s]'.
+ */
+TEST_CASE ("azblob", "[.][azblob]")
+{
+    auto maybe_url = vcpkg::get_environment_variable("VCPKG_TEST_AZBLOB_URL");
+    REQUIRE(maybe_url.has_value());
+    std::string url = maybe_url.value_or_exit(VCPKG_LINE_INFO);
+    REQUIRE(!url.empty());
+
+    if (url.back() != '/') url += '/';
+
+    auto maybe_sas = vcpkg::get_environment_variable("VCPKG_TEST_AZBLOB_SAS");
+    REQUIRE(maybe_sas.has_value());
+    std::string query_string = maybe_sas.value_or_exit(VCPKG_LINE_INFO);
+    REQUIRE(!query_string.empty());
+
+    if (query_string.front() != '?') query_string += '?' + query_string;
+
+    auto& fs = real_filesystem;
+    auto temp_dir = Test::base_temporary_directory() / "azblob";
+    fs.remove_all(temp_dir, VCPKG_LINE_INFO);
+
+    std::error_code ec;
+    fs.create_directories(temp_dir, ec);
+    CHECK_EC_ON_FILE(temp_dir, ec);
+
+    const char* data = "(blob content)";
+    auto data_filepath = temp_dir / "data";
+    CAPTURE(data_filepath);
+    fs.write_contents(data_filepath, data, ec);
+    CHECK_EC_ON_FILE(data_filepath, ec);
+
+    auto rnd = Strings::b32_encode(std::mt19937_64()());
+    std::vector<std::pair<std::string, Path>> url_pairs;
+    {
+        auto plain_put_filename = "plain_put_" + rnd;
+        auto plain_put_url = url + plain_put_filename + query_string;
+        url_pairs.emplace_back(plain_put_url, temp_dir / plain_put_filename);
+
+        FullyBufferedDiagnosticContext diagnostics{};
+        auto plain_put_success = store_to_asset_cache(
+            diagnostics, plain_put_url, SanitizedUrl{url, {}}, "PUT", azure_blob_headers(), data_filepath);
+        INFO(diagnostics.to_string());
+        CHECK(plain_put_success);
+    }
+
+    {
+        auto azcopy_put_filename = "azcopy_put_" + rnd;
+        auto azcopy_put_url = url + azcopy_put_filename + query_string;
+        url_pairs.emplace_back(azcopy_put_url, temp_dir / azcopy_put_filename);
+
+        FullyBufferedDiagnosticContext diagnostics{};
+        auto azcopy_put_success =
+            azcopy_to_asset_cache(diagnostics, azcopy_put_url, SanitizedUrl{url, {}}, data_filepath);
+        INFO(diagnostics.to_string());
+        CHECK(azcopy_put_success);
+    }
+
+    {
+        FullyBufferedDiagnosticContext diagnostics{};
+        auto results = download_files_no_cache(diagnostics, url_pairs, azure_blob_headers(), {});
+        INFO(diagnostics.to_string());
+        CHECK(results == std::vector<int>{200, 200});
+    }
+
+    for (auto& download : url_pairs)
+    {
+        auto download_filepath = download.second;
+        CAPTURE(download_filepath);
+        CHECK(fs.read_contents(download_filepath, VCPKG_LINE_INFO) == data);
+    }
+
+    fs.remove_all(temp_dir, VCPKG_LINE_INFO);
+}
diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp
@@ -890,6 +890,43 @@ namespace vcpkg
         return true;
     }
 
+    bool azcopy_to_asset_cache(DiagnosticContext& context,
+                               StringView raw_url,
+                               const SanitizedUrl& sanitized_url,
+                               const Path& file)
+    {
+        auto azcopy_cmd = Command{"azcopy"};
+        azcopy_cmd.string_arg("copy");
+        azcopy_cmd.string_arg("--from-to").string_arg("LocalBlob");
+        azcopy_cmd.string_arg("--log-level").string_arg("NONE");
+        azcopy_cmd.string_arg(file);
+        azcopy_cmd.string_arg(raw_url.to_string());
+
+        int code = 0;
+        auto res = cmd_execute_and_stream_lines(context, azcopy_cmd, [&code](StringView line) {
+            static constexpr StringLiteral response_marker = "RESPONSE ";
+            if (line.starts_with(response_marker))
+            {
+                code = std::strtol(line.data() + response_marker.size(), nullptr, 10);
+            }
+        });
+
+        auto pres = res.get();
+        if (!pres)
+        {
+            return false;
+        }
+
+        if (*pres != 0)
+        {
+            context.report_error(msg::format(
+                msgAzcopyFailedToPutBlob, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code));
+            return false;
+        }
+
+        return true;
+    }
+
     std::string format_url_query(StringView base_url, View<std::string> query_params)
     {
         if (query_params.empty())
diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp
@@ -506,6 +506,57 @@ namespace
         std::vector<std::string> m_secrets;
     };
 
+    struct AzureBlobPutBinaryProvider : IWriteBinaryProvider
+    {
+        AzureBlobPutBinaryProvider(const Filesystem& fs,
+                                   std::vector<UrlTemplate>&& urls,
+                                   const std::vector<std::string>& secrets)
+            : m_fs(fs), m_urls(std::move(urls)), m_secrets(secrets)
+        {
+        }
+
+        size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override
+        {
+            if (!request.zip_path) return 0;
+
+            const auto& zip_path = *request.zip_path.get();
+
+            size_t count_stored = 0;
+            const auto file_size = m_fs.file_size(zip_path, VCPKG_LINE_INFO);
+            if (file_size == 0) return count_stored;
+
+            // cf.
+            // https://learn.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json
+            constexpr size_t max_single_write = 5000000000;
+            bool use_azcopy = file_size > max_single_write;
+
+            PrintingDiagnosticContext pdc{msg_sink};
+            WarningDiagnosticContext wdc{pdc};
+
+            for (auto&& templ : m_urls)
+            {
+                auto url = templ.instantiate_variables(request);
+                auto maybe_success =
+                    use_azcopy
+                        ? azcopy_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, zip_path)
+                        : store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path);
+                if (maybe_success)
+                {
+                    count_stored++;
+                }
+            }
+            return count_stored;
+        }
+
+        bool needs_nuspec_data() const override { return false; }
+        bool needs_zip_file() const override { return true; }
+
+    private:
+        const Filesystem& m_fs;
+        std::vector<UrlTemplate> m_urls;
+        std::vector<std::string> m_secrets;
+    };
+
     struct NuGetSource
     {
         StringLiteral option;
@@ -1497,7 +1548,9 @@ namespace
                         segments[0].first);
                 }
 
-                if (!Strings::starts_with(segments[1].second, "https://"))
+                if (!Strings::starts_with(segments[1].second, "https://") &&
+                    // Allow unencrypted Azurite for testing (not reflected in error msg)
+                    !Strings::starts_with(segments[1].second, "http://127.0.0.1"))
                 {
                     return add_error(msg::format(msgInvalidArgumentRequiresBaseUrl,
                                                  msg::base_url = "https://",
@@ -1538,7 +1591,7 @@ namespace
                 if (read) state->url_templates_to_get.push_back(url_template);
                 auto headers = azure_blob_headers();
                 url_template.headers.assign(headers.begin(), headers.end());
-                if (write) state->url_templates_to_put.push_back(url_template);
+                if (write) state->azblob_templates_to_put.push_back(url_template);
 
                 state->binary_cache_providers.insert("azblob");
             }
@@ -2287,6 +2340,11 @@ namespace vcpkg
                 m_config.write.push_back(
                     std::make_unique<FilesWriteBinaryProvider>(fs, std::move(s.archives_to_write)));
             }
+            if (!s.azblob_templates_to_put.empty())
+            {
+                m_config.write.push_back(
+                    std::make_unique<AzureBlobPutBinaryProvider>(fs, std::move(s.azblob_templates_to_put), s.secrets));
+            }
             if (!s.url_templates_to_put.empty())
             {
                 m_config.write.push_back(
