Skip to content

Commit

Permalink
Merge pull request #14 from davidbrochart/gcs
Browse files Browse the repository at this point in the history 
Rework GCS store
  • Loading branch information
@JohanMabille
JohanMabille committed Nov 20, 2020
2 parents d837d82 + b6ea43c commit c68f75c
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 42 deletions.
29 changes: 18 additions & 11 deletions include/xtensor-zarr/xzarr_aws_store.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <aws/s3/model/DeleteObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/Object.h>
#include "xzarr_common.hpp"

namespace xt
{
Expand Down Expand Up @@ -140,10 +141,6 @@ namespace xt
{
XTENSOR_THROW(std::runtime_error, "Root directory cannot be empty");
}
while (m_root.back() == '/')
{
m_root.pop_back();
}
std::size_t i = m_root.find('/');
if (i == std::string::npos)
{
Expand All @@ -155,36 +152,45 @@ namespace xt
m_bucket = m_root.substr(0, i).c_str();
m_root = m_root.substr(i + 1);
}
while (m_root.back() == '/')
{
m_root.pop_back();
}
}

xzarr_aws_stream xzarr_aws_store::operator[](const std::string& key) const
{
return xzarr_aws_stream((m_root + '/' + key).c_str(), m_bucket, m_client);
std::string key2 = ensure_startswith_slash(key);
return xzarr_aws_stream((m_root + key2).c_str(), m_bucket, m_client);
}

void xzarr_aws_store::set(const std::string& key, const std::vector<char>& value)
{
xzarr_aws_stream((m_root + '/' + key).c_str(), m_bucket, m_client) = value;
std::string key2 = ensure_startswith_slash(key);
xzarr_aws_stream((m_root + key2).c_str(), m_bucket, m_client) = value;
}

void xzarr_aws_store::set(const std::string& key, const std::string& value)
{
xzarr_aws_stream((m_root + '/' + key).c_str(), m_bucket, m_client) = value;
std::string key2 = ensure_startswith_slash(key);
xzarr_aws_stream((m_root + key2).c_str(), m_bucket, m_client) = value;
}

std::string xzarr_aws_store::get(const std::string& key) const
{
return xzarr_aws_stream((m_root + '/' + key).c_str(), m_bucket, m_client);
std::string key2 = ensure_startswith_slash(key);
return xzarr_aws_stream((m_root + key2).c_str(), m_bucket, m_client);
}

void xzarr_aws_store::list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes) const
{
Aws::S3::Model::ListObjectsRequest request;
std::string full_prefix = prefix;
if (!m_root.empty())
{
full_prefix = m_root + '/' + prefix;
std::string prefix2 = ensure_startswith_slash(prefix);
full_prefix = m_root + prefix2;
}
Aws::S3::Model::ListObjectsRequest request;
request.WithBucket(m_bucket).WithPrefix(full_prefix.c_str());
auto outcome = m_client.ListObjects(request);
if (!outcome.IsSuccess())
Expand Down Expand Up @@ -230,7 +236,8 @@ namespace xt
std::string full_prefix = prefix;
if (!m_root.empty())
{
full_prefix = m_root + '/' + prefix;
std::string prefix2 = ensure_startswith_slash(prefix);
full_prefix = m_root + prefix2;
}
Aws::S3::Model::ListObjectsRequest request;
request.WithBucket(m_bucket).WithPrefix(full_prefix.c_str());
Expand Down
20 changes: 13 additions & 7 deletions include/xtensor-zarr/xzarr_common.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
#ifndef XTENSOR_ZARR_COMMON_HPP
#define XTENSOR_ZARR_COMMON_HPP

#include "nlohmann/json.hpp"

namespace xt
{
inline std::size_t get_zarr_major(const std::string& zarr_version)
Expand Down Expand Up @@ -77,13 +79,13 @@ namespace xt
* xindex_path implementation *
******************************/

xzarr_index_path::xzarr_index_path()
inline xzarr_index_path::xzarr_index_path()
: m_separator('/')
, m_zarr_version(3)
{
}

void xzarr_index_path::set_directory(const std::string& directory)
inline void xzarr_index_path::set_directory(const std::string& directory)
{
m_directory = directory;
if (m_directory.back() != '/')
Expand All @@ -92,18 +94,18 @@ namespace xt
}
}

void xzarr_index_path::set_separator(const char separator)
inline void xzarr_index_path::set_separator(const char separator)
{
m_separator = separator;
}

void xzarr_index_path::set_zarr_version(std::size_t zarr_version)
inline void xzarr_index_path::set_zarr_version(std::size_t zarr_version)
{
m_zarr_version = zarr_version;
}

template <class I>
void xzarr_index_path::index_to_path(I first, I last, std::string& path)
inline void xzarr_index_path::index_to_path(I first, I last, std::string& path)
{
std::string fname;
for (auto it = first; it != last; ++it)
Expand All @@ -124,12 +126,16 @@ namespace xt
path = m_directory + fname;
}

nlohmann::json xzarr_attrs::attrs()
/******************************
* xzarr_attrs implementation *
******************************/

inline nlohmann::json xzarr_attrs::attrs()
{
return m_attrs;
}

void xzarr_attrs::set_attrs(const nlohmann::json& attrs)
inline void xzarr_attrs::set_attrs(const nlohmann::json& attrs)
{
m_attrs = attrs;
}
Expand Down
49 changes: 25 additions & 24 deletions include/xtensor-zarr/xzarr_gcs_store.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ namespace xt
{
public:
xzarr_gcs_stream(const std::string& path, const std::string& bucket, gcs::Client& client);
operator std::string();
void operator=(const std::vector<char>& value);
void operator=(const std::string& value);
operator std::string() const;
xzarr_gcs_stream& operator=(const std::vector<char>& value);
xzarr_gcs_stream& operator=(const std::string& value);

private:
void assign(const char* value, std::size_t size);
Expand All @@ -44,23 +44,22 @@ namespace xt
using io_handler = xio_gcs_handler<C>;

xzarr_gcs_store(const std::string& root, gcs::Client& client);
xzarr_gcs_stream operator[](const std::string& key);
std::vector<std::string> list();
std::vector<std::string> list_prefix(const std::string& prefix);
void list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes);
void erase(const std::string& key);
void erase_prefix(const std::string& prefix);
xzarr_gcs_stream operator[](const std::string& key) const;
void set(const std::string& key, const std::vector<char>& value);
void set(const std::string& key, const std::string& value);
std::string get(const std::string& key);

xio_gcs_config get_io_config();
std::string get_root();
std::string get(const std::string& key) const;
void list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes) const;
std::vector<std::string> list() const;
std::vector<std::string> list_prefix(const std::string& prefix) const;
void erase(const std::string& key);
void erase_prefix(const std::string& prefix);
std::string get_root() const;
xio_gcs_config get_io_config() const;

private:
std::string m_root;
std::string m_bucket;
gcs::Client m_client;
gcs::Client& m_client;
};

/***********************************
Expand All @@ -74,21 +73,23 @@ namespace xt
{
}

xzarr_gcs_stream::operator std::string()
xzarr_gcs_stream::operator std::string() const
{
auto reader = m_client.ReadObject(m_bucket, m_path);
std::string bytes{std::istreambuf_iterator<char>{reader}, {}};
return bytes;
}

void xzarr_gcs_stream::operator=(const std::vector<char>& value)
xzarr_gcs_stream& xzarr_gcs_stream::operator=(const std::vector<char>& value)
{
assign(value.data(), value.size());
return *this;
}

void xzarr_gcs_stream::operator=(const std::string& value)
xzarr_gcs_stream& xzarr_gcs_stream::operator=(const std::string& value)
{
assign(value.c_str(), value.size());
return *this;
}

void xzarr_gcs_stream::assign(const char* value, std::size_t size)
Expand Down Expand Up @@ -127,7 +128,7 @@ namespace xt
}
}

xzarr_gcs_stream xzarr_gcs_store::operator[](const std::string& key)
xzarr_gcs_stream xzarr_gcs_store::operator[](const std::string& key) const
{
std::string key2 = ensure_startswith_slash(key);
return xzarr_gcs_stream(m_root + key2, m_bucket, m_client);
Expand All @@ -145,13 +146,13 @@ namespace xt
xzarr_gcs_stream(m_root + key2, m_bucket, m_client) = value;
}

std::string xzarr_gcs_store::get(const std::string& key)
std::string xzarr_gcs_store::get(const std::string& key) const
{
std::string key2 = ensure_startswith_slash(key);
return std::move(xzarr_gcs_stream(m_root + key2, m_bucket, m_client));
}

void xzarr_gcs_store::list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes)
void xzarr_gcs_store::list_dir(const std::string& prefix, std::vector<std::string>& keys, std::vector<std::string>& prefixes) const
{
std::string prefix2 = ensure_startswith_slash(prefix);
for (auto&& object_metadata: m_client.ListObjects(m_bucket, gcs::Prefix(m_root + prefix2)))
Expand Down Expand Up @@ -185,12 +186,12 @@ namespace xt
}
}

std::vector<std::string> xzarr_gcs_store::list()
std::vector<std::string> xzarr_gcs_store::list() const
{
return list_prefix("");
}

std::vector<std::string> xzarr_gcs_store::list_prefix(const std::string& prefix)
std::vector<std::string> xzarr_gcs_store::list_prefix(const std::string& prefix) const
{
std::string prefix2 = ensure_startswith_slash(prefix);
std::vector<std::string> keys;
Expand Down Expand Up @@ -224,12 +225,12 @@ namespace xt
}
}

std::string xzarr_gcs_store::get_root()
std::string xzarr_gcs_store::get_root() const
{
return m_root;
}

xio_gcs_config xzarr_gcs_store::get_io_config()
xio_gcs_config xzarr_gcs_store::get_io_config() const
{
xio_gcs_config c = {m_client, m_bucket};
return c;
Expand Down

0 comments on commit c68f75c

Please sign in to comment.