Merge pull request #31 from davidbrochart/gdal_store

Add xzarr_gdal_store
xtensor-stack · Jan 25, 2021 · 4774e98 · 4774e98
2 parents d5cec3e + 3c5b2ad
commit 4774e98
Show file tree

Hide file tree

Showing 15 changed files with 506 additions and 120 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -31,10 +31,17 @@ message(STATUS "Building xtensor-zarr v${${PROJECT_NAME}_VERSION}")
 
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" "${CMAKE_SOURCE_DIR}/cmake")
 
-find_package(nlohmann_json 3.2.0 REQUIRED)
+set(nlohmann_json_required_version "3.2.0" )
+find_package(nlohmann_json ${nlohmann_json_required_version} REQUIRED)
 
-find_package(xtensor-io 0.10.1 REQUIRED)
-message(STATUS "Found xtensor: ${xtensor_INCLUDE_DIRS}/xtensor")
+set(xtensor_required_version "0.23.0")
+find_package(xtensor ${xtensor_required_version} REQUIRED)
+
+set(xtensor_io_required_version "0.12.0")
+find_package(xtensor-io ${xtensor_io_required_version} REQUIRED)
+
+set(zarray_required_version "0.0.3")
+find_package(zarray ${zarray_required_version} REQUIRED)
 
 #Remove the following lines when xtensor-io is fixed
 include(CMakeFindDependencyMacro)
@@ -53,6 +60,7 @@ set(XTENSOR_ZARR_HEADERS
     ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_file_system_store.hpp
     ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_gcs_store.hpp
     ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_aws_store.hpp
+    ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_gdal_store.hpp
     ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_common.hpp
     ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_compressor.hpp
     ${XTENSOR_ZARR_INCLUDE_DIR}/xtensor-zarr/xzarr_chunked_array.hpp
@@ -64,14 +72,31 @@ OPTION(DOWNLOAD_GTEST "build gtest from downloaded sources" OFF)
 OPTION(DOWNLOAD_GBENCHMARK "download google benchmark and build from source" ON)
 
 add_library(xtensor-zarr INTERFACE)
-target_link_libraries(xtensor-zarr INTERFACE xtensor-io)
+target_link_libraries(xtensor-zarr INTERFACE xtensor-io zarray)
 
 target_include_directories(xtensor-zarr
     INTERFACE
     $<BUILD_INTERFACE:${XTENSOR_ZARR_INCLUDE_DIR}>
     $<INSTALL_INTERFACE:include>
 )
 
+message(STATUS "Trying to find GDAL for Virtual File System support")
+find_package(GDAL)
+if(${GDAL_FOUND})
+    message(STATUS "GDAL ${GDAL_VERSION} found, Virtual File System support enabled")
+    include_directories(${Blosc_INCLUDE_DIRS})
+    target_include_directories(xtensor-zarr
+        INTERFACE
+        $<BUILD_INTERFACE:${GDAL_INCLUDE_DIRS}>
+    )
+    target_link_libraries(xtensor-zarr
+        INTERFACE
+        ${GDAL_LIBRARIES}
+    )
+else()
+    message(WARNING "GDAL not found - install GDAL for Virtual File System support")
+endif()
+
 message(STATUS "Trying to find Blosc for Blosc file support")
 find_package(Blosc)
 if (${Blosc_FOUND})

diff --git a/README.md b/README.md
@@ -1,4 +1,5 @@
-# xtensor-zarr
+# ![xtensor-zarr](docs/source/xtensor-zarr.svg)
+
 [![Azure Pipelines](https://dev.azure.com/xtensor-stack/xtensor-stack/_apis/build/status/xtensor-stack.xtensor-zarr?branchName=master)](https://dev.azure.com/xtensor-stack/xtensor-stack/_build/latest?definitionId=9&branchName=master)
 [![ReadTheDocs](https://readthedocs.org/projects/xtensor-zarr/badge/?version=latest)](http://xtensor-zarr.readthedocs.io/en/latest)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/xtensor-stack/xtensor-zarr/master?filepath=examples%2Fzarr_v3.ipynb)

diff --git a/environment-dev.yml b/environment-dev.yml
@@ -10,8 +10,9 @@ dependencies:
   - aws-sdk-cpp
   - cpp-filesystem
   - zlib
-  - xtensor-io
-  - zarray
+  - xtensor-io=0.12.1
+  - gdal
+  - zarray=0.0.3
   # Test dependencies
   - fsspec
   - numpy

diff --git a/include/xtensor-zarr/xzarr_aws_store.hpp b/include/xtensor-zarr/xzarr_aws_store.hpp
@@ -115,7 +115,7 @@ namespace xt
         request.SetKey(m_path);
 
         std::shared_ptr<Aws::IOStream> writer = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", m_path.c_str(), std::ios_base::in | std::ios_base::binary);
-        writer->write(value, size);
+        writer->write(value, static_cast<std::streamsize>(size));
         writer->flush();
 
         request.SetBody(writer);

diff --git a/include/xtensor-zarr/xzarr_common.hpp b/include/xtensor-zarr/xzarr_common.hpp
@@ -10,7 +10,8 @@
 #ifndef XTENSOR_ZARR_COMMON_HPP
 #define XTENSOR_ZARR_COMMON_HPP
 
-#include "nlohmann/json.hpp"
+#include <xtensor-io/xio_binary.hpp>
+#include <nlohmann/json.hpp>
 
 namespace xt
 {
@@ -41,11 +42,11 @@ namespace xt
         std::size_t zarr_major;
         if (i == std::string::npos)
         {
-            zarr_major = std::stoi(zarr_version);
+            zarr_major = static_cast<std::size_t>(std::stoi(zarr_version));
         }
         else
         {
-            zarr_major = std::stoi(zarr_version.substr(0, i));
+            zarr_major = static_cast<std::size_t>(std::stoi(zarr_version.substr(0, i)));
         }
         if ((zarr_major < 2) || (zarr_major > 3))
         {
@@ -84,18 +85,6 @@ namespace xt
         std::size_t m_zarr_version;
     };
 
-    // xzarr_attrs is meant to serve as a base class extension for xchunked_array
-    // it provides JSON attribute getter and setter methods
-    class xzarr_attrs
-    {
-    public:
-        nlohmann::json attrs();
-        void set_attrs(const nlohmann::json& attrs);
-
-    private:
-        nlohmann::json m_attrs;
-    };
-
     /******************************
      * xindex_path implementation *
      ******************************/
@@ -147,20 +136,6 @@ namespace xt
         path = m_directory + fname;
     }
 
-    /******************************
-     * xzarr_attrs implementation *
-     ******************************/
-
-    inline nlohmann::json xzarr_attrs::attrs()
-    {
-        return m_attrs;
-    }
-
-    inline void xzarr_attrs::set_attrs(const nlohmann::json& attrs)
-    {
-        m_attrs = attrs;
-    }
-
 }
 
 #endif
diff --git a/include/xtensor-zarr/xzarr_compressor.hpp b/include/xtensor-zarr/xzarr_compressor.hpp
@@ -19,25 +19,25 @@
 namespace xt
 {
     template <class T>
-    T get_nan()
+    inline T get_nan()
     {
         return 0;
     }
 
     template <>
-    float get_nan<float>()
+    inline float get_nan<float>()
     {
         return std::nanf("");
     }
 
     template <>
-    double get_nan<double>()
+    inline double get_nan<double>()
     {
         return std::nan("");
     }
 
     template <>
-    long double get_nan<long double>()
+    inline long double get_nan<long double>()
     {
         return std::nanl("");
     }
@@ -61,14 +61,17 @@ namespace xt
         }
         if (fill_value_json.is_null())
         {
-            auto a = chunked_file_array<data_type, io_handler, layout_type::dynamic, xzarr_index_path, xzarr_attrs>(shape, chunk_shape, path, chunk_pool_size, layout);
+            auto a = chunked_file_array<data_type, io_handler, layout_type::dynamic, xzarr_index_path>(shape, chunk_shape, path, chunk_pool_size, layout);
             auto& i2p = a.chunks().get_index_path();
             i2p.set_separator(separator);
             i2p.set_zarr_version(zarr_version);
             auto io_config = store.get_io_config();
             a.chunks().configure(config, io_config);
-            a.set_attrs(attrs);
-            return zarray(std::move(a));
+            auto z = zarray(std::move(a));
+            auto metadata = z.get_metadata();
+            metadata["zarr"] = attrs;
+            z.set_metadata(metadata);
+            return z;
         }
         else
         {
@@ -81,14 +84,17 @@ namespace xt
             {
                 fill_value = fill_value_json;
             }
-            auto a = chunked_file_array<data_type, io_handler, layout_type::dynamic, xzarr_index_path, xzarr_attrs>(shape, chunk_shape, path, fill_value, chunk_pool_size, layout);
+            auto a = chunked_file_array<data_type, io_handler, layout_type::dynamic, xzarr_index_path>(shape, chunk_shape, path, fill_value, chunk_pool_size, layout);
             auto& i2p = a.chunks().get_index_path();
             i2p.set_separator(separator);
             i2p.set_zarr_version(zarr_version);
             auto io_config = store.get_io_config();
             a.chunks().configure(config, io_config);
-            a.set_attrs(attrs);
-            return zarray(std::move(a));
+            auto z = zarray(std::move(a));
+            auto metadata = z.get_metadata();
+            metadata["zarr"] = attrs;
+            z.set_metadata(metadata);
+            return z;
         }
     }
 

diff --git a/include/xtensor-zarr/xzarr_file_system_store.hpp b/include/xtensor-zarr/xzarr_file_system_store.hpp
@@ -76,23 +76,23 @@ namespace xt
      * xzarr_file_system_stream implementation *
      *******************************************/
 
-    xzarr_file_system_stream::xzarr_file_system_stream(const std::string& path)
+    inline xzarr_file_system_stream::xzarr_file_system_stream(const std::string& path)
         : m_path(path)
     {
     }
 
-    void xzarr_file_system_stream::erase()
+    inline void xzarr_file_system_stream::erase()
     {
         fs::remove(m_path);
     }
 
-    bool xzarr_file_system_stream::exists()
+    inline bool xzarr_file_system_stream::exists()
     {
         std::ifstream stream(m_path);
         return stream.good();
     }
 
-    xzarr_file_system_stream::operator std::string() const
+    inline xzarr_file_system_stream::operator std::string() const
     {
         std::ifstream stream(m_path);
         if (!stream.is_open())
@@ -103,17 +103,17 @@ namespace xt
         return bytes;
     }
 
-    void xzarr_file_system_stream::operator=(const std::vector<char>& value)
+    inline void xzarr_file_system_stream::operator=(const std::vector<char>& value)
     {
         assign(value.data(), value.size());
     }
 
-    void xzarr_file_system_stream::operator=(const std::string& value)
+    inline void xzarr_file_system_stream::operator=(const std::string& value)
     {
         assign(value.c_str(), value.size());
     }
 
-    void xzarr_file_system_stream::assign(const char* value, std::size_t size)
+    inline void xzarr_file_system_stream::assign(const char* value, std::size_t size)
     {
         // maybe create directories
         std::size_t i = m_path.rfind('/');
@@ -133,15 +133,15 @@ namespace xt
             }
         }
         std::ofstream stream(m_path, std::ofstream::binary);
-        stream.write(value, size);
+        stream.write(value, std::streamsize(size));
         stream.flush();
     }
 
     /******************************************
      * xzarr_file_system_store implementation *
      ******************************************/
 
-    xzarr_file_system_store::xzarr_file_system_store(const std::string& root)
+    inline xzarr_file_system_store::xzarr_file_system_store(const std::string& root)
         : m_root(root)
     {
         if (m_root.empty())
@@ -154,12 +154,12 @@ namespace xt
         }
     }
 
-    xzarr_file_system_stream xzarr_file_system_store::operator[](const std::string& key)
+    inline xzarr_file_system_stream xzarr_file_system_store::operator[](const std::string& key)
     {
         return xzarr_file_system_stream(m_root + '/' + key);
     }
 
-    void xzarr_file_system_store::set(const std::string& key, const std::vector<char>& value)
+    inline void xzarr_file_system_store::set(const std::string& key, const std::vector<char>& value)
     {
         xzarr_file_system_stream(m_root + '/' + key) = value;
     }
@@ -169,7 +169,7 @@ namespace xt
      * @param key the key
      * @param value the value
      */
-    void xzarr_file_system_store::set(const std::string& key, const std::string& value)
+    inline void xzarr_file_system_store::set(const std::string& key, const std::string& value)
     {
         xzarr_file_system_stream(m_root + '/' + key) = value;
     }
@@ -180,17 +180,17 @@ namespace xt
      *
      * @return returns the value for the given key.
      */
-    std::string xzarr_file_system_store::get(const std::string& key)
+    inline std::string xzarr_file_system_store::get(const std::string& key)
     {
-        return std::move(xzarr_file_system_stream(m_root + '/' + key));
+        return xzarr_file_system_stream(m_root + '/' + key);
     }
 
-    std::string xzarr_file_system_store::get_root()
+    inline std::string xzarr_file_system_store::get_root()
     {
         return m_root;
     }
 
-    xio_disk_config xzarr_file_system_store::get_io_config()
+    inline xio_disk_config xzarr_file_system_store::get_io_config()
     {
         xio_disk_config c;
         return c;
@@ -203,7 +203,7 @@ namespace xt
      * @param keys set of keys to be returned by reference
      * @param prefixes set of prefixes to be returned by reference
      */
-    void xzarr_file_system_store::list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes)
+    inline void xzarr_file_system_store::list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes)
     {
         std::string path = m_root + '/' + prefix;
         for (const auto& entry: fs::directory_iterator(path))
@@ -225,7 +225,7 @@ namespace xt
      *
      * @return returns a set of keys.
      */
-    std::vector<std::string> xzarr_file_system_store::list()
+    inline std::vector<std::string> xzarr_file_system_store::list()
     {
         return list_prefix("");
     }
@@ -237,7 +237,7 @@ namespace xt
      *
      * @return returns a set of keys with a given prefix.
      */
-    std::vector<std::string> xzarr_file_system_store::list_prefix(const std::string& prefix)
+    inline std::vector<std::string> xzarr_file_system_store::list_prefix(const std::string& prefix)
     {
         std::string path = m_root + '/' + prefix;
         std::vector<std::string> keys;
@@ -253,7 +253,7 @@ namespace xt
      * Erase the given (key, value) pair from the store.
      * @param key the key
      */
-    void xzarr_file_system_store::erase(const std::string& key)
+    inline void xzarr_file_system_store::erase(const std::string& key)
     {
         fs::remove(m_root + '/' + key);
     }
@@ -262,7 +262,7 @@ namespace xt
      * Erase all the keys with the given prefix from the store.
      * @param prefix the prefix
      */
-    void xzarr_file_system_store::erase_prefix(const std::string& prefix)
+    inline void xzarr_file_system_store::erase_prefix(const std::string& prefix)
     {
         fs::remove_all(m_root + '/' + prefix);
     }