Merge pull request #941 from koordinates/fix-pdal-s3-fetch

Range-request based metadata extraction for point clouds

kart/byod/point_cloud_import.py

@@ -127,11 +127,4 @@ def byod_point_cloud_import(
 class ByodPointCloudImporter(ByodTileImporter, PointCloudImporter):
     def extract_tile_metadata(self, tile_location):
         oid_and_size = get_hash_and_size_of_s3_object(tile_location)
-        # TODO - download only certain ranges of the file, and extract metadata from those.
-        tmp_downloaded_tile = fetch_from_s3(tile_location)
-        result = extract_pc_tile_metadata(
-            tmp_downloaded_tile, oid_and_size=oid_and_size
-        )
-        tmp_downloaded_tile.unlink()
-        result["tile"]["url"] = tile_location
-        return None, result
+        return None, extract_pc_tile_metadata(tile_location, oid_and_size=oid_and_size)

vcpkg-vendor/vcpkg-overlay-ports/pdal/arbiter-aws-no-sign-request.patch

@@ -0,0 +1,106 @@
+diff --git a/vendor/arbiter/arbiter.cpp b/vendor/arbiter/arbiter.cpp
+index 1ccbfc89e..0a3b7e999 100644
+--- a/vendor/arbiter/arbiter.cpp
++++ b/vendor/arbiter/arbiter.cpp
+@@ -1751,6 +1751,11 @@ namespace
+         else if (auto e = env("ARBITER_VERBOSE")) verbose = *e;
+         return (!verbose.empty()) && !!std::stol(verbose);
+     }
++
++    bool doSignRequests()
++    {
++        return !env("AWS_NO_SIGN_REQUEST");
++    }
+ }
+
+ namespace drivers
+@@ -1780,9 +1785,7 @@ std::unique_ptr<S3> S3::create(
+         if (auto p = env("AWS_PROFILE")) profile = *p;
+     }
+
+-    auto auth(Auth::create(s, profile));
+-    if (!auth) return std::unique_ptr<S3>();
+-
++    auto auth(doSignRequests() ? Auth::create(s, profile) : nullptr);
+     auto config = makeUnique<Config>(s, profile);
+     return makeUnique<S3>(pool, profile, std::move(auth), std::move(config));
+ }
+@@ -2142,7 +2145,7 @@ std::unique_ptr<std::size_t> S3::tryGetSize(
+             "HEAD",
+             m_config->region(),
+             resource,
+-            m_auth->fields(),
++            authFields(),
+             query,
+             headers,
+             empty);
+@@ -2178,7 +2181,7 @@ bool S3::get(
+             "GET",
+             m_config->region(),
+             resource,
+-            m_auth->fields(),
++            authFields(),
+             query,
+             headers,
+             empty);
+@@ -2223,7 +2226,7 @@ std::vector<char> S3::put(
+             "PUT",
+             m_config->region(),
+             resource,
+-            m_auth->fields(),
++            authFields(),
+             query,
+             headers,
+             data);
+@@ -2373,6 +2376,11 @@ std::vector<std::string> S3::glob(std::string path, bool verbose) const
+     return results;
+ }
+
++S3::AuthFields S3::authFields() const
++{
++    return m_auth ? m_auth->fields() : S3::AuthFields();
++}
++
+ S3::ApiV4::ApiV4(
+         const std::string verb,
+         const std::string& region,
+@@ -2407,6 +2415,8 @@ S3::ApiV4::ApiV4(
+         m_headers.erase("Expect");
+     }
+
++    if (!m_authFields) return;
++
+     const Headers normalizedHeaders(
+             std::accumulate(
+                 m_headers.begin(),
+diff --git a/vendor/arbiter/arbiter.hpp b/vendor/arbiter/arbiter.hpp
+index c1faad41e..802b104f8 100644
+--- a/vendor/arbiter/arbiter.hpp
++++ b/vendor/arbiter/arbiter.hpp
+@@ -4321,6 +4321,8 @@ private:
+             std::string path,
+             bool verbose) const override;
+
++    AuthFields authFields() const;
++
+     class ApiV4;
+     class Resource;
+
+@@ -4331,7 +4333,7 @@ private:
+ class S3::AuthFields
+ {
+ public:
+-    AuthFields(std::string access, std::string hidden, std::string token = "")
++    AuthFields(std::string access = "", std::string hidden = "", std::string token = "")
+         : m_access(access), m_hidden(hidden), m_token(token)
+     { }
+
+@@ -4339,6 +4341,8 @@ public:
+     const std::string& hidden() const { return m_hidden; }
+     const std::string& token() const { return m_token; }
+
++    explicit operator bool() const { return m_access.size() || m_hidden.size() || m_token.size(); }
++
+ private:
+     std::string m_access;
+     std::string m_hidden;

vcpkg-vendor/vcpkg-overlay-ports/pdal/portfile.cmake

@@ -12,6 +12,7 @@ vcpkg_from_github(
         no-pkgconfig-requires.patch
         no-rpath.patch
         install-dimbuilder.patch
+        arbiter-aws-no-sign-request.patch # Merged upstream: https://github.com/connormanning/arbiter/pull/48
 )
 
 # Prefer pristine CMake find modules + wrappers and config files from vcpkg.