From 80b2870001b92bb7b53489641b67fd383d8c5e81 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Tue, 3 May 2022 18:34:50 +0530 Subject: [PATCH] Refactor minio client Signed-off-by: Bala.FA --- CMakeLists.txt | 6 +- examples/CMakeLists.txt | 2 + examples/check.cpp | 199 ++++++++ include/client.h | 307 +++++++++++++ include/creds.h | 60 +++ include/data.h | 51 +++ include/http.h | 156 +++++++ include/signer.h | 67 +++ include/utils.h | 130 ++++++ include/xml.h | 29 ++ src/CMakeLists.txt | 2 +- src/client.cc | 987 ++++++++++++++++++++++++++++++++++++++++ src/creds.cc | 55 +++ src/data.cc | 45 ++ src/http.cc | 395 ++++++++++++++++ src/signer.cc | 147 ++++++ src/utils.cc | 386 ++++++++++++++++ src/xml.cc | 45 ++ 18 files changed, 3065 insertions(+), 4 deletions(-) create mode 100644 examples/check.cpp create mode 100644 include/client.h create mode 100644 include/creds.h create mode 100644 include/data.h create mode 100644 include/http.h create mode 100644 include/signer.h create mode 100644 include/utils.h create mode 100644 include/xml.h create mode 100644 src/client.cc create mode 100644 src/creds.cc create mode 100644 src/data.cc create mode 100644 src/http.cc create mode 100644 src/signer.cc create mode 100644 src/utils.cc create mode 100644 src/xml.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index f666aa74..28c4099a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ macro(set_globals) endmacro() # specify the C++ standard -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -79,8 +79,8 @@ INCLUDE (CheckIncludeFiles) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/include) SET(MINIOCPP_MAJOR_VERSION "0") -SET(MINIOCPP_MINOR_VERSION "0") -SET(MINIOCPP_PATCH_VERSION "1") +SET(MINIOCPP_MINOR_VERSION "1") +SET(MINIOCPP_PATCH_VERSION "0") add_subdirectory(include) add_subdirectory(src) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 1f0ef815..8464c4bc 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,6 +1,8 @@ ADD_EXECUTABLE(s3 s3.cpp) +ADD_EXECUTABLE(check check.cpp) SET(S3_LIBS ${requiredlibs}) TARGET_LINK_LIBRARIES(s3 miniocpp ${S3_LIBS}) +TARGET_LINK_LIBRARIES(check miniocpp ${S3_LIBS}) INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/s3 DESTINATION bin) diff --git a/examples/check.cpp b/examples/check.cpp new file mode 100644 index 00000000..4b8796a1 --- /dev/null +++ b/examples/check.cpp @@ -0,0 +1,199 @@ +#include + +#include +#include +#include +#include +#include +#include "pugixml.hpp" + +#include +#include +#include +#include +#include +#include + +size_t DataCallback(minio::http::DataCallbackArgs args) { + std::cout << "writeCallBack(): [" << std::string(args.buffer, args.length) << "]" << std::endl; + return args.size * args.length; +} + +int main(int argc, char *argv[]) { + int x = 4; + auto l = [&r = x, x = x+1]()->int { + r += 2; + return x+2; + }; // Updates ::x to 6, and initializes y to 7. + std::cout << "x = " << x << std::endl; + auto y = l(); + std::cout << "x = " << x << std::endl; + std::cout << "y = " << y << std::endl; + + minio::http::BaseUrl base_url1("localhost:9000"); + base_url1.SetHttps(false); + minio::creds::StaticProvider provider("minio", "minio123"); + + minio::Client client(base_url1, &provider); + + std::cout << "StatObject()" << std::endl; + minio::StatObjectResponse sor = client.StatObject("my123", "zero1"); + std::cout << "IsSuccess: " << minio::utils::BoolToString(sor.IsSuccess()) << std::endl; + std::cout << "Error: " << sor.err_ << std::endl; + std::cout << "StatusCode: " << sor.status_code_ << std::endl; + std::cout << "Headers: " << sor.headers_.ToHttpHeaders() << std::endl; + std::cout << "Code: " << sor.code_ << std::endl; + std::cout << "Message: " << sor.message_ << std::endl; + + // std::cout << "DownloadObject()" << std::endl; + // minio::DownloadObjectResponse dor = client.DownloadObject("my123", "zero", "/tmp/a"); + // std::cout << "IsSuccess: " << minio::utils::BoolToString(dor.IsSuccess()) << std::endl; + // std::cout << "Error: " << dor.err_ << std::endl; + // std::cout << "StatusCode: " << dor.status_code_ << std::endl; + // std::cout << "Headers: " << dor.headers_.ToHttpHeaders() << std::endl; + // std::cout << "Code: " << dor.code_ << std::endl; + // std::cout << "Message: " << dor.message_ << std::endl; + + std::cout << "MakeBucket()" << std::endl; + minio::MakeBucketResponse mbr = client.MakeBucket("my123"); + std::cout << "IsSuccess: " << minio::utils::BoolToString(mbr.IsSuccess()) << std::endl; + std::cout << "Error: " << mbr.err_ << std::endl; + std::cout << "StatusCode: " << mbr.status_code_ << std::endl; + std::cout << "Headers: " << mbr.headers_.ToHttpHeaders() << std::endl; + std::cout << "Code: " << mbr.code_ << std::endl; + std::cout << "Message: " << mbr.message_ << std::endl; + + std::cout << "GetRegion()" << std::endl; + minio::GetRegionResponse grr = client.GetRegion("my123", ""); + std::cout << "IsSuccess: " << minio::utils::BoolToString(grr.IsSuccess()) << std::endl; + std::cout << "Region: " << grr.region_ << std::endl; + std::cout << "Error: " << grr.err_ << std::endl; + std::cout << "StatusCode: " << grr.status_code_ << std::endl; + std::cout << "Headers: " << grr.headers_.ToHttpHeaders() << std::endl; + std::cout << "Code: " << grr.code_ << std::endl; + std::cout << "Message: " << grr.message_ << std::endl; + + std::cout << "ListBuckets()" << std::endl; + minio::ListBucketsResponse blr = client.ListBuckets(); + std::cout << "IsSuccess: " << minio::utils::BoolToString(blr.IsSuccess()) << std::endl; + std::cout << "Buckets: "; + for (auto b: blr.buckets_) { + std::cout << "(" << b.Name() << ", " << b.CreationDate().ToHttpHeaderValue() << ") "; + } + std::cout << std::endl; + std::cout << "Error: " << blr.err_ << std::endl; + std::cout << "StatusCode: " << blr.status_code_ << std::endl; + std::cout << "Headers: " << blr.headers_.ToHttpHeaders() << std::endl; + std::cout << "Code: " << blr.code_ << std::endl; + std::cout << "Message: " << blr.message_ << std::endl; + + std::cout << "BucketExists()" << std::endl; + minio::BucketExistsResponse ber = client.BucketExists("my123"); + std::cout << "IsSuccess: " << minio::utils::BoolToString(ber.IsSuccess()) << std::endl; + std::cout << "Exist: " << minio::utils::BoolToString(ber.exist_) << std::endl; + std::cout << "Error: " << ber.err_ << std::endl; + std::cout << "StatusCode: " << ber.status_code_ << std::endl; + std::cout << "Headers: " << ber.headers_.ToHttpHeaders() << std::endl; + std::cout << "Code: " << ber.code_ << std::endl; + std::cout << "Message: " << ber.message_ << std::endl; + + std::cout << "RemoveBucket()" << std::endl; + minio::RemoveBucketResponse rbr = client.RemoveBucket("my123"); + std::cout << "IsSuccess: " << minio::utils::BoolToString(rbr.IsSuccess()) << std::endl; + std::cout << "Error: " << rbr.err_ << std::endl; + std::cout << "StatusCode: " << rbr.status_code_ << std::endl; + std::cout << "Headers: " << rbr.headers_.ToHttpHeaders() << std::endl; + std::cout << "Code: " << rbr.code_ << std::endl; + std::cout << "Message: " << rbr.message_ << std::endl; + + return 0; + + minio::http::Method method = minio::http::Method::kPut; + std::string body = "Enabled"; + minio::utils::Multimap query_params; + query_params.Add("versioning", ""); + + /////////////////////////////////////////////////////////////////////////////// + + minio::RequestBuilder rb(method, "us-east-1", base_url1); + rb.SetQueryParams(query_params); + rb.SetBucketName("put"); + rb.SetBody(body); + + minio::http::Request req = rb.Build(&provider); + minio::http::Response response = req.Execute(); + std::cout << "Error: " << response.Error() << std::endl; + std::cout << "StatusCode: " << response.StatusCode() << std::endl; + std::cout << "Headers: " << response.Headers().ToHttpHeaders() << std::endl; + std::cout << "Data: " << response.Data() << std::endl; + + /////////////////////////////////////////////////////////////////////////// + // minio::http::BaseUrl base_url("httpbin.org"); + // RequestBuilder request_builder(method, "us-east-1", base_url); + // + // // request_builder.SetUserAgent(std::string_view user_agent); + // // request_builder.SetHeaders(minio::utils::Multimap headers); + // request_builder.SetQueryParams(query_params); + // request_builder.SetBucketName("put"); + // // request_builder.SetObjectName("path/to/myobject"); + // request_builder.SetBody(body); + // // request_builder.SetNoBodyTrace(bool no_body_trace); + // + // minio::http::Request r = request_builder.Build(&provider); + // minio::http::Response resp = r.Execute(); + // // minio::http::Response resp = r.Execute(DataCallback); + // std::cout << "Error: " << resp.Error() << std::endl; + // std::cout << "StatusCode: " << resp.StatusCode() << std::endl; + // std::cout << "Headers: " << resp.Headers().ToHttpHeaders() << std::endl; + // std::cout << "Data: " << resp.Data() << std::endl; + + ///////////////////////////////////////////////////////////////// + // std::string url = "http://httpbin.org/put"; + // minio::utils::Multimap headerMap; + // headerMap.Add("Content-Length", std::to_string(body.size())); + // + // minio::http::Request req(method, url); + // req.SetHeaders(headerMap); + // req.SetBody(body); + // minio::http::Response res = req.Execute(); + // std::cout << "Error: " << res.Error() << std::endl; + // std::cout << "StatusCode: " << res.StatusCode() << std::endl; + // std::cout << "Headers: " << res.Headers().ToHttpHeaders() << std::endl; + // std::cout << "Data: " << res.Data() << std::endl; + + ////////////////////////////////////////////////////////////////////// + // curlpp::Cleanup cleaner; + // curlpp::Easy request; + // + // // Request settings. + // request.setOpt(new curlpp::options::CustomRequest{minio::http::MethodToString(method)}); + // request.setOpt(new curlpp::Options::Url(url)); + // // request.setOpt(new curlpp::Options::Verbose(true)); + // + // std::list headers = headerMap.ToHttpHeaders(); + // headers.push_back("Expect:"); // disable 100 continue from server. + // request.setOpt(new curlpp::Options::HttpHeader(headers)); + // + // std::stringstream body_stream(body); + // request.setOpt(new curlpp::Options::ReadStream(&body_stream)); + // request.setOpt(new curlpp::Options::InfileSize(body.size())); + // request.setOpt(new curlpp::Options::Upload(true)); + // + // // Response settings. + // request.setOpt(new curlpp::options::Header(true)); + // + // minio::http::Response response; + // // response.SetDataCallback(DataCallback); + // + // using namespace std::placeholders; + // request.setOpt(new curlpp::options::WriteFunction(std::bind(&minio::http::Response::ResponseCallback, &response, &request, _1, _2, _3))); + // + // // Execute. + // request.perform(); + // + // std::cout << "Error: " << response.Error() << std::endl; + // std::cout << "StatusCode: " << response.StatusCode() << std::endl; + // std::cout << "Headers: " << response.Headers().ToHttpHeaders() << std::endl; + // std::cout << "Data: " << response.Data() << std::endl; + return EXIT_SUCCESS; +} diff --git a/include/client.h b/include/client.h new file mode 100644 index 00000000..f037942d --- /dev/null +++ b/include/client.h @@ -0,0 +1,307 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_CLIENT_H +#define _MINIO_CLIENT_H + +#include "http.h" +#include "utils.h" +#include "creds.h" +#include "data.h" +#include "signer.h" +#include "xml.h" +#include +#include +#include + +namespace minio { +class RequestBuilder { + private: + minio::http::Method method_; + std::string_view region_; + minio::http::BaseUrl& base_url_; + + std::string_view user_agent_; + + minio::utils::Multimap headers_; + minio::utils::Multimap query_params_; + + std::string_view bucket_name_; + std::string_view object_name_; + + std::string_view body_ = ""; + + bool no_body_trace_ = false; + minio::http::DataCallback data_callback_ = NULL; + void* user_arg_ = NULL; + + std::string sha256_; + minio::utils::Time date_; + + public: + RequestBuilder(minio::http::Method method, std::string_view region, + minio::http::BaseUrl& base_url); + void SetUserAgent(std::string_view user_agent); + void SetHeaders(minio::utils::Multimap headers); + void SetQueryParams(minio::utils::Multimap query_params); + void SetBucketName(std::string_view bucket_name); + void SetObjectName(std::string_view object_name); + void SetBody(std::string_view body); + void SetDataCallback(minio::http::DataCallback data_callback, void* user_arg = NULL); + void SetNoBodyTrace(bool no_body_trace); + void BuildHeaders(minio::utils::Url& url, minio::creds::Provider *provider); + minio::http::Request Build(minio::creds::Provider *provider = NULL); +}; // class RequestBuilder + +class S3Response { + public: + int status_code_ = 0; + minio::utils::Multimap headers_; + std::string data_; + + std::string err_; + + std::string code_; + std::string message_; + std::string resource_; + std::string request_id_; + std::string host_id_; + std::string bucket_name_; + std::string object_name_; + + S3Response(); + S3Response(const S3Response& response); + bool IsSuccess(); +}; // class S3Response + +class GetRegionResponse : public S3Response { + public: + std::string region_; + + GetRegionResponse(); + GetRegionResponse(const S3Response& response); +}; // class GetRegionResponse + +using MakeBucketResponse = S3Response; + +class ListBucketsResponse : public S3Response { + public: + std::list buckets_; + + ListBucketsResponse(); + ListBucketsResponse(const S3Response& response); +}; // class ListBucketsResponse + +class BucketExistsResponse : public S3Response { + public: + bool exist_ = false; + + BucketExistsResponse(); + BucketExistsResponse(const S3Response& response); +}; // class BucketExistsResponse + +using RemoveBucketResponse = S3Response; + +using AbortMultipartUploadResponse = S3Response; + +class CompleteMultipartUploadResponse : public S3Response { + public: + std::string location_; + std::string etag_; + std::string version_id_; + + CompleteMultipartUploadResponse(); + CompleteMultipartUploadResponse(const S3Response& response); +}; // class CompleteMultipartUploadResponse + +class CreateMultipartUploadResponse : public S3Response { + public: + std::string upload_id_; + + CreateMultipartUploadResponse(); + CreateMultipartUploadResponse(const S3Response& response); +}; // class CreateMultipartUploadResponse + +class PutObjectResponse : public S3Response { + public: + std::string etag_; + std::string version_id_; + + PutObjectResponse(); + PutObjectResponse(const S3Response& response); +}; // class PutObjectResponse + +using UploadPartResponse = PutObjectResponse; + +enum class RetentionMode { kGovernance, kCompliance }; + +// StringToRetentionMode converts string to retention mode enum. +RetentionMode StringToRetentionMode(std::string_view str) throw(); + +// RetentionModeToString converts retention mode enum to string. +constexpr const char* RetentionModeToString(RetentionMode& retention) throw() { + switch (retention) { + case RetentionMode::kGovernance: + return "GOVERNANCE"; + case RetentionMode::kCompliance: + return "COMPLIANCE"; + default: { + std::cerr << "ABORT: Unknown retention mode. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +enum class LegalHold { kOn, kOff }; + +// StringToLegalHold converts string to legal hold enum. +LegalHold StringToLegalHold(std::string_view str) throw(); + +// LegalHoldToString converts legal hold enum to string. +constexpr const char* LegalHoldToString(LegalHold& legal_hold) throw() { + switch (legal_hold) { + case LegalHold::kOn: + return "ON"; + case LegalHold::kOff: + return "OFF"; + default: { + std::cerr << "ABORT: Unknown legal hold. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +class StatObjectResponse : public S3Response { +public: + std::string version_id_; + std::string etag_; + size_t size_ = 0; + minio::utils::Time last_modified_; + RetentionMode retention_mode_; + minio::utils::Time retention_retain_until_date_; + LegalHold legal_hold_; + bool delete_marker_; + minio::utils::Multimap user_metadata_; + + StatObjectResponse(); + StatObjectResponse(const S3Response& response); +}; // class StatObjectResponse + +using RemoveObjectResponse = S3Response; + +using DownloadObjectResponse = S3Response; + +size_t SaveData(minio::http::DataCallbackArgs args); + +class Client { + private: + minio::http::BaseUrl& base_url_; + minio::creds::Provider *provider_ = NULL; + std::map region_map_; +public: + Client(minio::http::BaseUrl& base_url, minio::creds::Provider *provider = NULL); + + void HandleRedirectResponse(std::string& code, + std::string& message, + int status_code, + minio::http::Method method, + minio::utils::Multimap headers, + std::string_view bucket_name, + bool retry = false); + S3Response GetErrorResponse(minio::http::Response resp, + std::string_view resource, + minio::http::Method method, + std::string_view bucket_name, + std::string_view object_name); + S3Response execute(minio::http::Request& request, + std::string_view bucket_name="", + std::string_view object_name=""); + S3Response Execute(minio::http::Request& request, + std::string_view bucket_name="", + std::string_view object_name=""); + + // Bucket operations + GetRegionResponse GetRegion(std::string_view bucket_name, std::string_view region = ""); + MakeBucketResponse MakeBucket(std::string_view bucket_name, std::string_view region = "", bool object_lock = false); + ListBucketsResponse ListBuckets(); + BucketExistsResponse BucketExists(std::string_view bucket_name); + RemoveBucketResponse RemoveBucket(std::string_view bucket_name); + + // Object operations + AbortMultipartUploadResponse AbortMultipartUpload(std::string_view bucket_name, std::string_view object_name, std::string_view upload_id); + CompleteMultipartUploadResponse CompleteMultipartUpload(std::string_view bucket_name, std::string_view object_name, std::string_view upload_id, std::list parts); + CreateMultipartUploadResponse CreateMultipartUpload(std::string_view bucket_name, std::string_view object_name, minio::utils::Multimap headers); + PutObjectResponse PutObject(std::string_view bucket_name, std::string_view object_name, std::string_view data, minio::utils::Multimap headers, minio::utils::Multimap query_params); + UploadPartResponse UploadPart(std::string_view bucket_name, std::string_view object_name, std::string_view upload_id, unsigned int part_number, std::string_view data, minio::utils::Multimap headers); + StatObjectResponse StatObject(std::string_view bucket_name, std::string_view object_name, std::string_view version_id = ""); + // Add + // ssec + // Long offset; + // Long length; + // String matchETag; + // String notMatchETag; + // ZonedDateTime modifiedSince; + // ZonedDateTime unmodifiedSince; + RemoveObjectResponse RemoveObject(std::string_view bucket_name, std::string_view object_name, std::string_view version_id=""); + DownloadObjectResponse DownloadObject(std::string_view bucket_name, std::string_view object_name, std::string_view filename, std::string_view version_id="", bool overwrite=false); + // protected ServerSideEncryptionCustomerKey ssec; + +// ///////////////////////////////////// +// +// void ListObjects(Minio::S3::Bucket& bucket, S3Connection** conn = NULL); +// +// // List objects (bucket.s3.amazonaws.com GET /) +// void ListObjects(const std::string& bkt, Minio::S3ClientIO& io, +// S3Connection** reqPtr = NULL); +// +// // Upload from IO stream. +// void PutObject(const std::string& bkt, const std::string& key, +// Minio::S3ClientIO& io, S3Connection** reqPtr = NULL); +// +// // Upload from local path. +// void PutObject(const std::string& bkt, const std::string& key, +// const std::string& localpath, Minio::S3ClientIO& io, +// S3Connection** reqPtr = NULL); +// +// // Multipart APIs +// // Upload from io stream to a specific part number for multipart upload_id. +// Minio::S3::CompletePart PutObject(const std::string& bkt, +// const std::string& key, +// const int& part_number, +// const std::string& upload_id, +// Minio::S3ClientIO& io, +// S3Connection** reqPtr = NULL); +// +// // Get object data (GET /key) with specific partNumber. +// void GetObject(const std::string& bkt, const std::string& key, +// const int& part_number, Minio::S3ClientIO& io, +// S3Connection** reqPtr = NULL); +// +// // Get object data fully +// void GetObject(const std::string& bkt, const std::string& key, +// Minio::S3ClientIO& io, S3Connection** reqPtr = NULL); +// +// // Copy object (COPY) +// void CopyObject(const std::string& srcbkt, const std::string& srckey, +// const std::string& dstbkt, const std::string& dstkey, +// bool copyMD, Minio::S3ClientIO& io, +// S3Connection** reqPtr = NULL); +}; // class Client +} // namespace minio +#endif // #ifndef __MINIO_CLIENT_H diff --git a/include/creds.h b/include/creds.h new file mode 100644 index 00000000..78e6fd1e --- /dev/null +++ b/include/creds.h @@ -0,0 +1,60 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_CREDS_H +#define _MINIO_CREDS_H + +#include + +namespace minio { +namespace creds { +class Credentials { + private: + std::string_view access_key_; + std::string_view secret_key_; + std::string_view session_token_; + unsigned int expiration_; + + public: + Credentials(const Credentials& creds); + Credentials(std::string_view access_key, std::string_view secret_key, + std::string_view session_token = "", unsigned int expiration = 0); + std::string AccessKey(); + std::string SecretKey(); + std::string SessionToken(); + bool IsExpired(); +}; // class Credentials + +class Provider { + public: + Provider() {} + virtual ~Provider() {} + virtual Credentials Fetch() = 0; +}; // class Provider + +class StaticProvider : public Provider { + private: + Credentials* creds_; + + public: + StaticProvider(std::string_view access_key, std::string_view secret_key, + std::string_view session_token = ""); + ~StaticProvider(); + Credentials Fetch(); +}; // class StaticProvider +} // namespace creds +} // namespace minio + +#endif // #ifndef _MINIO_CREDS_H diff --git a/include/data.h b/include/data.h new file mode 100644 index 00000000..e0fb9a4c --- /dev/null +++ b/include/data.h @@ -0,0 +1,51 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_DATA_H +#define _MINIO_DATA_H + +#include +#include "utils.h" + +namespace minio { +namespace data { +class Bucket { + private: + std::string name_; + minio::utils::Time creation_date_; + public: + Bucket(std::string name, minio::utils::Time creation_date); + std::string Name(); + minio::utils::Time CreationDate(); +}; // class Bucket + +class Part { + private: + unsigned int part_number_; + std::string_view etag_; + minio::utils::Time last_modified_; + size_t size_; + public: + Part(unsigned int part_number, std::string_view etag, minio::utils::Time last_modified, size_t size); + Part(unsigned int part_number, std::string_view etag); + unsigned int PartNumber(); + std::string ETag(); + minio::utils::Time LastModified(); + size_t Size(); +}; // class Part +} // namespace data +} // namespace minio + +#endif // #ifndef _MINIO_DATA_H diff --git a/include/http.h b/include/http.h new file mode 100644 index 00000000..ca233993 --- /dev/null +++ b/include/http.h @@ -0,0 +1,156 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_HTTP_H +#define _MINIO_HTTP_H + +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +namespace minio { +namespace http { +enum class Method { kGet, kHead, kPost, kPut, kDelete }; + +// MethodToString converts http Method enum to string. +constexpr const char* MethodToString(Method& method) throw() { + switch (method) { + case Method::kGet: + return "GET"; + case Method::kHead: + return "HEAD"; + case Method::kPost: + return "POST"; + case Method::kPut: + return "PUT"; + case Method::kDelete: + return "DELETE"; + default: { + std::cerr << "ABORT: Unimplemented HTTP method. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +// ExtractRegion extracts region value from AWS S3 host string. +std::string ExtractRegion(std::string host); + +class BaseUrl { + private: + std::string host_; + bool https_; + unsigned int port_; + std::string region_; + + bool aws_host_; + bool accelerate_host_flag_; + bool dualstack_host_flag_; + + bool virtual_style_flag_; + + public: + BaseUrl(std::string host); + std::string GetRegion(); + bool GetHttps(); + void SetHttps(bool flag); + std::string GetHost(); + unsigned int GetPort(); + bool IsAwsHost(); + bool GetAccelerateHostFlag(); + void SetAccelerateHostFlag(bool flag); + bool GetDualstackHostFlag(); + void SetDualstackHostFlag(bool flag); + bool GetVirtualStyleFlag(); + void SetVirtualStyleFlag(bool flag); + std::string GetHostHeaderValue(); + utils::Url BuildUrl(Method method, std::string region, + utils::Multimap query_params, + std::string bucket_name = "", + std::string object_name = ""); +}; // class BaseUrl + +typedef struct DataCallbackArgs DataCallbackArgs; +typedef size_t (*DataCallback)(DataCallbackArgs args); + +class Response { + private: + std::string err_ = ""; + + std::string response_ = ""; + + bool continue100_ = false; + int status_code_ = 0; + bool status_code_read_ = false; + + minio::utils::Multimap headers_; + bool headers_read_ = false; + + std::string body_ = ""; + + DataCallback callback_ = NULL; + void* user_arg_ = NULL; + + public: + void SetDataCallback(DataCallback callback, void* user_arg); + void SetError(std::string err); + size_t ReadStatusCode(char* buffer, size_t size, size_t length); + size_t ReadHeaders(curlpp::Easy *handle, char* buffer, size_t size, size_t length); + size_t ResponseCallback(curlpp::Easy *handle, char* buffer, size_t size, size_t length); + bool IsSuccess(); + std::string Error(); + int StatusCode(); + minio::utils::Multimap Headers(); + std::string Data(); +}; // class Response + +struct DataCallbackArgs { + curlpp::Easy* handle; + Response* callback; + char* buffer; + size_t size; + size_t length; + void* user_arg = NULL; +}; // struct DataCallbackArgs + +class Request { + protected: + Method method_; + std::string url_; + std::string_view body_ = ""; + minio::utils::Multimap headers_; + DataCallback data_callback_ = NULL; + std::string resource_; + void* user_arg_ = NULL; + public: + Request(Method method, std::string url, std::string resource); + void SetBody(std::string_view body); + void SetHeaders(minio::utils::Multimap headers); + void SetDataCallback(DataCallback data_callback, void* user_arg = NULL); + std::string Resource(); + Method GetMethod(); + Response Execute(); + private: + Response execute(); +}; // class Request +} // namespace http +} // namespace minio +#endif // #ifndef _MINIO_HTTP_H diff --git a/include/signer.h b/include/signer.h new file mode 100644 index 00000000..8c4b0183 --- /dev/null +++ b/include/signer.h @@ -0,0 +1,67 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_SIGNER_H +#define _MINIO_SIGNER_H + +#include +#include + +#include "http.h" +#include "utils.h" + +namespace minio { +namespace signer { +std::string GetScope(utils::Time& time, std::string_view region, + std::string_view service_name); +std::string GetCanonicalRequestHash(std::string_view method, + std::string_view uri, + std::string_view query_string, + std::string_view headers, + std::string_view signed_headers, + std::string_view content_sha256); +std::string GetStringToSign(utils::Time& date, std::string_view scope, + std::string_view canonical_request_hash); +std::string HmacHash(std::string_view key, std::string_view data); +std::string GetSigningKey(std::string_view secret_key, utils::Time& date, + std::string_view region, + std::string_view service_name); +std::string GetSignature(std::string_view signing_key, + std::string_view string_to_sign); +std::string GetAuthorization(std::string_view access_key, + std::string_view scope, + std::string_view signed_headers, + std::string_view signature); +utils::Multimap& SignV4(std::string_view service_name, http::Method& method, + std::string_view uri, std::string_view region, + utils::Multimap& headers, utils::Multimap& query_params, + std::string_view access_key, + std::string_view secret_key, + std::string_view content_sha256, utils::Time& date); +utils::Multimap& SignV4S3(http::Method& method, std::string_view uri, + std::string_view region, utils::Multimap& headers, + utils::Multimap& query_params, + std::string_view access_key, + std::string_view secret_key, + std::string_view content_sha256, utils::Time& date); +utils::Multimap& SignV4STS(http::Method& method, std::string_view uri, + std::string_view region, utils::Multimap& headers, + utils::Multimap& query_params, + std::string_view access_key, + std::string_view secret_key, + std::string_view content_sha256, utils::Time& date); +} // namespace signer +} // namespace minio +#endif // #ifndef __MINIO_SIGNER_H diff --git a/include/utils.h b/include/utils.h new file mode 100644 index 00000000..ca3a40aa --- /dev/null +++ b/include/utils.h @@ -0,0 +1,130 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_UTILS_H +#define _MINIO_UTILS_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace minio { +namespace utils { +// FormatTime formats time as per format. +std::string FormatTime(const std::tm* time, const char* format); + +// StringToBool converts string to bool. +bool StringToBool(std::string_view str); + +// BoolToString converts bool to string. +inline const char* const BoolToString(bool b) { return b ? "true" : "false"; } + +// TrimSpaces trims whitespaces of a string. +std::string TrimSpaces(std::string_view str); + +// CheckNonemptystring checks whether string is not empty after trimming whitespaces. +bool CheckNonEmptyString(std::string_view str); + +// ToLower converts string to lower case. +std::string ToLower(std::string str); + +// StartsWith returns whether str starts with prefix or not. +bool StartsWith(std::string_view str, std::string_view prefix); + +// EndsWith returns whether str ends with suffix or not. +bool EndsWith(std::string_view str, std::string_view suffix); + +// EncodePath does URL encoding of path. It also normalizes multiple slashes. +std::string EncodePath(std::string_view path); + +// Sha256hash computes SHA-256 of data and return hash as hex encoded value. +std::string Sha256Hash(std::string_view str); + +// Base64Encode encodes string to base64. +std::string Base64Encode(std::string_view str); + +// Md5sumHash computes MD5 of data and return hash as Base64 encoded value. +std::string Md5sumHash(std::string_view str); + +// Time represents date and time object. +class Time { + private: + struct timeval tv_; + bool utc_; + + public: + Time(); + Time(std::time_t tv_sec, suseconds_t tv_usec, bool utc); + std::tm* ToUTC(); + std::string ToSignerDate(); + std::string ToAmzDate(); + std::string ToHttpHeaderValue(); + static Time FromHttpHeaderValue(const char* value); + std::string ToISO8601UTC(); + static Time FromISO8601UTC(const char* value); +}; // class Time + +// Multimap represents map of key and multiple values. +class Multimap { + private: + std::map> map_; + std::map> keys_; + public: + Multimap(); + Multimap(const Multimap& headers); + void Add(std::string key, std::string value); + void AddAll(const Multimap& headers); + std::list ToHttpHeaders(); + std::string ToQueryString(); + bool IsEmpty(); + bool Contains(std::string_view key); + std::list Get(std::string_view key); + std::list Keys(); + void GetCanonicalHeaders(std::string& signed_headers, + std::string& canonical_headers); + std::string GetCanonicalQueryString(); +}; // class Multimap + +class Url { + private: + bool https_; + std::string host_; + std::string path_; + std::string query_string_; + public: + Url(bool https_flag, std::string_view host, std::string_view path = "", + std::string_view query_string = ""); + std::string String(); + std::string Host(); + std::string Path(); + bool IsHttps(); +}; // class Url + +bool CheckBucketName(std::string_view bucket_name, bool strict = false); +} // namespace utils +} // namespace minio + +#endif // #ifndef _MINIO_UTILS_H diff --git a/include/xml.h b/include/xml.h new file mode 100644 index 00000000..5616a0f1 --- /dev/null +++ b/include/xml.h @@ -0,0 +1,29 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_XML_H +#define _MINIO_XML_H + +#include +#include "pugixml.hpp" + +namespace minio { +namespace xml { +bool GetValue(std::string& value, const std::string_view xml_doc, const std::string_view xpath); +bool GetNodes(pugi::xpath_node_set& nodes, std::string_view xml_data, std::string_view xpath); +} // namespace xml +} // namespace minio + +#endif // #ifndef _MINIO_XML_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1cb8165b..ca438baf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SET(SRCS s3.cpp s3_http.cpp s3_io.cpp s3_signature_v2.cpp) +SET(SRCS s3.cpp s3_http.cpp s3_io.cpp s3_signature_v2.cpp creds.cc http.cc signer.cc utils.cc xml.cc data.cc client.cc) SET(AWS_INSTALL_LIST) add_library(miniocpp STATIC ${SRCS}) diff --git a/src/client.cc b/src/client.cc new file mode 100644 index 00000000..47bbec2a --- /dev/null +++ b/src/client.cc @@ -0,0 +1,987 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +minio::RequestBuilder::RequestBuilder(minio::http::Method method, + std::string_view region, minio::http::BaseUrl& base_url): base_url_(base_url) { + method_ = method; + region_ = region; +} + +void minio::RequestBuilder::SetUserAgent(std::string_view user_agent) { user_agent_ = user_agent; } + +void minio::RequestBuilder::SetHeaders(minio::utils::Multimap headers) { headers_ = headers; } + +void minio::RequestBuilder::SetQueryParams(minio::utils::Multimap query_params) { + query_params_ = query_params; +} + +void minio::RequestBuilder::SetBucketName(std::string_view bucket_name) { + bucket_name_ = bucket_name; +} + +void minio::RequestBuilder::SetObjectName(std::string_view object_name) { + object_name_ = object_name; +} + +void minio::RequestBuilder::SetBody(std::string_view body) { body_ = body; } + +void minio::RequestBuilder::SetDataCallback(minio::http::DataCallback data_callback, void* user_arg) { + data_callback_ = data_callback; + user_arg = user_arg; +} + +void minio::RequestBuilder::SetNoBodyTrace(bool no_body_trace) { no_body_trace_ = no_body_trace; } + +void minio::RequestBuilder::BuildHeaders(minio::utils::Url& url, minio::creds::Provider *provider) { + headers_.Add("Host", url.Host()); + + if (user_agent_.empty()) { + headers_.Add("User-Agent", "MinIO (Linux; x86_64) minio-cpp/0.1.0"); + } else { + headers_.Add("User-Agent", std::string(user_agent_)); + } + + bool md5sum_added = headers_.Contains("Content-MD5"); + std::string md5sum; + + switch (method_) { + case minio::http::Method::kPut: + case minio::http::Method::kPost: + headers_.Add("Content-Length", std::to_string(body_.size())); + if (!headers_.Contains("Content-Type")) { + headers_.Add("Content-Type", "application/octet-stream"); + } + } + + // MD5 hash of zero length byte array. + // public static final String ZERO_MD5_HASH = "1B2M2Y8AsgTpgAmY7PhCfg=="; + + if (provider != NULL) { + if (url.IsHttps()) { + sha256_ = "UNSIGNED-PAYLOAD"; + switch (method_) { + case minio::http::Method::kPut: + case minio::http::Method::kPost: + if (!md5sum_added) { + md5sum = minio::utils::Md5sumHash(body_); + } + } + } else { + switch (method_) { + case minio::http::Method::kPut: + case minio::http::Method::kPost: + sha256_ = minio::utils::Sha256Hash(body_); + break; + default: + sha256_ = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + } + } + } else { + switch (method_) { + case minio::http::Method::kPut: + case minio::http::Method::kPost: + if (!md5sum_added) { + md5sum = minio::utils::Md5sumHash(body_); + } + } + } + + if (!md5sum.empty()) headers_.Add("Content-MD5", md5sum); + if (!sha256_.empty()) headers_.Add("x-amz-content-sha256", sha256_); + + minio::utils::Time now; + date_ = now; + headers_.Add("x-amz-date", date_.ToAmzDate()); + + if (provider != NULL) { + minio::creds::Credentials creds = provider->Fetch(); + if (!creds.SessionToken().empty()) { + headers_.Add("X-Amz-Security-Token", creds.SessionToken()); + } + minio::signer::SignV4S3(method_, url.Path(), region_, headers_, + query_params_, creds.AccessKey(), + creds.SecretKey(), sha256_, date_); + } +} + +minio::http::Request minio::RequestBuilder::Build(minio::creds::Provider *provider) { + minio::utils::Url url = base_url_.BuildUrl(method_, std::string(region_), + query_params_, + std::string(bucket_name_), + std::string(object_name_)); + BuildHeaders(url, provider); + + minio::http::Request request(method_, url.String(), url.Path()); + request.SetBody(body_); + request.SetHeaders(headers_); + request.SetDataCallback(data_callback_, user_arg_); + + return request; +} + +minio::S3Response::S3Response() {} + +minio::S3Response::S3Response(const minio::S3Response& response) { + status_code_ = response.status_code_; + headers_ = response.headers_; + data_ = response.data_; + err_ = response.err_; + code_ = response.code_; + message_ = response.message_; + resource_ = response.resource_; + request_id_ = response.request_id_; + host_id_ = response.host_id_; + bucket_name_ = response.bucket_name_; + object_name_ = response.object_name_; +} + +bool minio::S3Response::IsSuccess() { + return err_.empty() && code_.empty() && message_.empty() && (status_code_ == 0 || status_code_ >= 200 && status_code_ <= 299); +} + +minio::GetRegionResponse::GetRegionResponse() {} + +minio::GetRegionResponse::GetRegionResponse(const S3Response& response) : S3Response(response) {} + +minio::ListBucketsResponse::ListBucketsResponse() {} + +minio::ListBucketsResponse::ListBucketsResponse(const S3Response& response) : S3Response(response) {} + +minio::BucketExistsResponse::BucketExistsResponse() {} + +minio::BucketExistsResponse::BucketExistsResponse(const S3Response& response) : S3Response(response) {} + +minio::CompleteMultipartUploadResponse::CompleteMultipartUploadResponse() {} + +minio::CompleteMultipartUploadResponse::CompleteMultipartUploadResponse(const S3Response& response) : S3Response(response) {} + +minio::CreateMultipartUploadResponse::CreateMultipartUploadResponse() {} + +minio::CreateMultipartUploadResponse::CreateMultipartUploadResponse(const S3Response& response) : S3Response(response) {} + +minio::PutObjectResponse::PutObjectResponse() {} + +minio::PutObjectResponse::PutObjectResponse(const S3Response& response) : S3Response(response) {} + +minio::StatObjectResponse::StatObjectResponse() {} + +minio::StatObjectResponse::StatObjectResponse(const S3Response& response) : S3Response(response) {} + +minio::Client::Client(minio::http::BaseUrl& base_url, minio::creds::Provider *provider) : base_url_(base_url) { + provider_ = provider; +} + +void minio::Client::HandleRedirectResponse(std::string& code, + std::string& message, + int status_code, + minio::http::Method method, + minio::utils::Multimap headers, + std::string_view bucket_name, + bool retry) { + switch (status_code) { + case 301: + code = "PermanentRedirect"; + message = "Moved Permanently"; + break; + case 307: + code = "Redirect"; + message = "Temporary redirect"; + break; + case 400: + code = "BadRequest"; + message = "Bad request"; + break; + default: + code = ""; + message = ""; + break; + } + + std::list values = headers.Get("x-amz-bucket-region"); + std::string region = values.front(); + + if (!message.empty() && !region.empty()) { + message += "; use region " + region; + } + + if (retry && !region.empty() && method == minio::http::Method::kHead && !bucket_name.empty() && !region_map_[std::string(bucket_name)].empty()) { + code = "RetryHead"; + message = ""; + } +} + +minio::S3Response minio::Client::GetErrorResponse(minio::http::Response resp, + std::string_view resource, + minio::http::Method method, + std::string_view bucket_name, + std::string_view object_name) { + minio::S3Response response; + response.status_code_ = resp.StatusCode(); + response.headers_ = resp.Headers(); + response.err_ = resp.Error(); + if (!response.err_.empty()) return response; + + std::string data = resp.Data(); + if (!data.empty()) { + std::list values = response.headers_.Get("Content-Type"); + std::list::iterator i; + bool found = false; + for (i = values.begin(); i != values.end(); ++i) { + std::string value = minio::utils::ToLower(*i); + if (value.find("application/xml") != std::string::npos) { + found = true; + break; + } + } + + if (found) { + minio::xml::GetValue(response.code_, data, "/Error/Code"); + minio::xml::GetValue(response.message_, data, "/Error/Message"); + minio::xml::GetValue(response.resource_, data, "/Error/Resource"); + minio::xml::GetValue(response.request_id_, data, "/Error/RequestId"); + minio::xml::GetValue(response.host_id_, data, "/Error/RequestId"); + minio::xml::GetValue(response.bucket_name_, data, "/Error/BucketName"); + minio::xml::GetValue(response.object_name_, data, "/Error/Key"); + } else { + std::string result; + for(const auto &s : values) { + if(!result.empty()) result += ","; + result += s; + } + + response.err_ = std::string("invalid response received; status code: ") + std::to_string(resp.StatusCode()) + "; content-type: " + result; + } + + return response; + } + + switch (resp.StatusCode()) { + case 301: + case 307: + case 400: + minio::Client::HandleRedirectResponse(response.code_, response.message_, + resp.StatusCode(), method, + resp.Headers(), bucket_name, true); + break; + case 403: + response.code_= "AccessDenied"; + response.message_ = "Access denied"; + break; + case 404: + if (!object_name.empty()) { + response.code_= "NoSuchKey"; + response.message_ = "Object does not exist"; + } else if (bucket_name.empty()) { + response.code_= "NoSuchBucket"; + response.message_ = "Bucket does not exist"; + } else { + response.code_= "ResourceNotFound"; + response.message_ = "Request resource not found"; + } + break; + case 405: + response.code_= "MethodNotAllowed"; + response.message_ = "The specified method is not allowed against this resource"; + break; + case 409: + if (bucket_name.empty()) { + response.code_= "NoSuchBucket"; + response.message_ = "Bucket does not exist"; + } else { + response.code_= "ResourceConflict"; + response.message_ = "Request resource conflicts"; + } + break; + case 501: + response.code_= "MethodNotAllowed"; + response.message_ = "The specified method is not allowed against this resource"; + break; + default: + response.err_ = std::string("server failed with HTTP status code ") + std::to_string(resp.StatusCode()); + return response; + } + + response.resource_ = resource; + response.request_id_ = response.headers_.Get("x-amz-request-id").front(); + response.host_id_ = response.headers_.Get("x-amz-id-2").front(); + response.bucket_name_ = bucket_name; + response.object_name_ = object_name; + + return response; +} + +minio::S3Response minio::Client::execute(minio::http::Request& request, + std::string_view bucket_name, + std::string_view object_name) { + minio::http::Response resp = request.Execute(); + if (resp.IsSuccess()) { + minio::S3Response response; + response.status_code_ = resp.StatusCode(); + response.headers_ = resp.Headers(); + response.data_ = resp.Data(); + return response; + } + + S3Response response = GetErrorResponse(resp, request.Resource(), + request.GetMethod(), bucket_name, + object_name); + if (response.code_ == "NoSuchBucket" || response.code_ == "RetryHead") { + region_map_.erase(std::string(bucket_name)); + } + + return response; +} + +minio::S3Response minio::Client::Execute(minio::http::Request& request, + std::string_view bucket_name, + std::string_view object_name) { + minio::S3Response resp = minio::Client::execute(request, bucket_name, object_name); + if (resp.IsSuccess() || resp.code_ != "RetryHead") return resp; + + // Retry only once on RetryHead error. + resp = minio::Client::execute(request, bucket_name, object_name); + if (resp.IsSuccess() || resp.code_ != "RetryHead") return resp; + + std::string code; + std::string message; + HandleRedirectResponse(code, message, resp.status_code_, request.GetMethod(), + resp.headers_, bucket_name); + resp.code_ = code; + resp.message_ = message; + + return resp; +} + +minio::GetRegionResponse minio::Client::GetRegion(std::string_view bucket_name, std::string_view region) { + std::string base_region = base_url_.GetRegion(); + if (!region.empty()) { + minio::GetRegionResponse resp; + if (!base_region.empty() && base_region != region) { + resp.err_ = "region must be " + base_region + ", but passed " + std::string(region); + } else { + resp.region_ = region; + } + return resp; + } + + if (!base_region.empty()) { + minio::GetRegionResponse resp; + resp.region_ = region; + return resp; + } + + if (bucket_name.empty() || provider_ == NULL) { + minio::GetRegionResponse resp; + resp.region_ = "us-east-1"; + return resp; + } + + std::string stored_region = region_map_[std::string(bucket_name)]; + if (!stored_region.empty()) { + minio::GetRegionResponse resp; + resp.region_ = stored_region; + return resp; + } + + RequestBuilder request_builder(minio::http::Method::kGet, "us-east-1", base_url_); + minio::utils::Multimap query_params; + query_params.Add("location", ""); + request_builder.SetQueryParams(query_params); + request_builder.SetBucketName(bucket_name); + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name); + if (response.IsSuccess()) { + std::string value; + minio::xml::GetValue(value, response.data_, "/LocationConstraint"); + if (value.empty()) { + value = "us-east-1"; + } else if (value == "EU") { + if (base_url_.IsAwsHost()) value = "eu-west-1"; + } + + region_map_[std::string(bucket_name)] = value; + + minio::GetRegionResponse resp; + resp.region_ = value; + return resp; + } + + minio::GetRegionResponse resp(response); + return resp; +} + +minio::MakeBucketResponse minio::Client::MakeBucket(std::string_view bucket_name, std::string_view region, bool object_lock) { + minio::MakeBucketResponse resp; + + if (!minio::utils::CheckBucketName(bucket_name, true)) { + resp.err_ = "invalid bucket name"; + return resp; + } + + std::string base_region = base_url_.GetRegion(); + if (!base_region.empty() && !region.empty() && base_region != region) { + resp.err_ = std::string("region must be ") + base_region + ", but passed " + std::string(region); + return resp; + } + + if (region.empty()) region = base_region; + if (region.empty()) region = "us-east-1"; + + RequestBuilder request_builder(minio::http::Method::kPut, region, base_url_); + request_builder.SetBucketName(bucket_name); + + minio::utils::Multimap headers; + if (object_lock) headers.Add("x-amz-bucket-object-lock-enabled", "true"); + request_builder.SetHeaders(headers); + + std::string body; + if (region != "us-east-1") { + std::stringstream ss; + ss << "" + << "" << region << "" + << ""; + body = ss.str(); + request_builder.SetBody(body); + } + + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name); + if (response.IsSuccess()) region_map_[std::string(bucket_name)] = region; + + return response; +} + +minio::ListBucketsResponse minio::Client::ListBuckets() { + RequestBuilder request_builder(minio::http::Method::kGet, base_url_.GetRegion(), base_url_); + minio::http::Request request = request_builder.Build(provider_); + minio::S3Response response = Execute(request); + if (!response.IsSuccess()) { + minio::ListBucketsResponse resp = response; + return resp; + } + + pugi::xpath_node_set nodes; + if (!minio::xml::GetNodes(nodes, response.data_, + "/ListAllMyBucketsResult/Buckets/Bucket")) { + minio::ListBucketsResponse resp; + resp.err_ = std::string("invalid XML received; data=") + response.data_; + return resp; + } + + std::list buckets; + for (pugi::xpath_node_set::const_iterator i = nodes.begin(); i != nodes.end(); ++i) { + std::string bucket_name; + minio::utils::Time creation_date; + for (pugi::xml_node node = i->node().first_child(); node; node = node.next_sibling()) { + std::string name = node.name(); + std::string value = node.child_value(); + if (name == "Name") { + bucket_name = value; + } else if (name == "CreationDate") { + creation_date = minio::utils::Time::FromISO8601UTC(value.c_str()); + } + } + + buckets.push_back(minio::data::Bucket(bucket_name, creation_date)); + } + + minio::ListBucketsResponse resp; + resp.buckets_ = buckets; + return resp; +} + +minio::BucketExistsResponse minio::Client::BucketExists(std::string_view bucket_name) { + minio::BucketExistsResponse resp; + + if (!minio::utils::CheckBucketName(bucket_name)) { + resp.err_ = "invalid bucket name"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) { + if (response.code_ != "NoSuchBucket") resp = response; + return resp; + } + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kHead, region, base_url_); + request_builder.SetBucketName(bucket_name); + + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name); + if (response.IsSuccess()) { + resp.exist_ = true; + } else if (response.code_ != "NoSuchBucket") { + resp = response; + } + + return resp; +} + +minio::RemoveBucketResponse minio::Client::RemoveBucket(std::string_view bucket_name) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::RemoveBucketResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kDelete, region, base_url_); + request_builder.SetBucketName(bucket_name); + + minio::http::Request request = request_builder.Build(provider_); + + return Execute(request, bucket_name); +} + +minio::AbortMultipartUploadResponse minio::Client::AbortMultipartUpload(std::string_view bucket_name, std::string_view object_name, std::string_view upload_id) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::AbortMultipartUploadResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::AbortMultipartUploadResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(upload_id)) { + minio::AbortMultipartUploadResponse resp; + resp.err_ = "upload ID cannot be empty"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kDelete, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + minio::utils::Multimap query_params; + query_params.Add("uploadId", std::string(upload_id)); + request_builder.SetQueryParams(query_params); + + minio::http::Request request = request_builder.Build(provider_); + + return Execute(request, bucket_name, object_name); +} + +minio::CompleteMultipartUploadResponse minio::Client::CompleteMultipartUpload(std::string_view bucket_name, std::string_view object_name, std::string_view upload_id, std::list parts) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::CompleteMultipartUploadResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::CompleteMultipartUploadResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(upload_id)) { + minio::CompleteMultipartUploadResponse resp; + resp.err_ = "upload ID cannot be empty"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kPost, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + minio::utils::Multimap query_params; + query_params.Add("uploadId", std::string(upload_id)); + request_builder.SetQueryParams(query_params); + + std::stringstream ss; + ss << ""; + std::list::iterator i; + for (i = parts.begin(); i != parts.end(); ++i) { + ss << "" + << "" << i->PartNumber() << "" + << "" << "\"" << i->ETag() << "\"" << "" + << ""; + } + ss << ""; + std::string body = ss.str(); + request_builder.SetBody(body); + + minio::utils::Multimap headers; + headers.Add("Content-Type", "application/xml"); + headers.Add("Content-MD5", minio::utils::Md5sumHash(body)); + request_builder.SetHeaders(headers); + + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name, object_name); + if (!response.IsSuccess()) return response; + + minio::CompleteMultipartUploadResponse resp; + + std::string value; + minio::xml::GetValue(value, response.data_, "/CompleteMultipartUploadOutput/Bucket"); + resp.bucket_name_ = value; + + minio::xml::GetValue(value, response.data_, "/CompleteMultipartUploadOutput/Key"); + resp.object_name_ = value; + + minio::xml::GetValue(value, response.data_, "/CompleteMultipartUploadOutput/Location"); + resp.location_ = value; + + minio::xml::GetValue(value, response.data_, "/CompleteMultipartUploadOutput/ETag"); + if (value.at(0) == '"') value.erase(0); + if (value.at(value.size()-1) == '"') value.erase(value.size()-1); + resp.etag_ = value; + + resp.version_id_ = response.headers_.Get("x-amz-version-id").front(); + + return resp; +} + +minio::CreateMultipartUploadResponse minio::Client::CreateMultipartUpload(std::string_view bucket_name, std::string_view object_name, minio::utils::Multimap headers) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::CreateMultipartUploadResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::CreateMultipartUploadResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + if (!headers.Contains("Content-Type")) { + headers.Add("Content-Type", "application/octet-stream"); + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kPost, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + minio::utils::Multimap query_params; + query_params.Add("uploadId", ""); + request_builder.SetQueryParams(query_params); + + request_builder.SetHeaders(headers); + + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name, object_name); + if (!response.IsSuccess()) return response; + + minio::CreateMultipartUploadResponse resp; + + std::string value; + minio::xml::GetValue(value, response.data_, "/InitiateMultipartUploadResult/UploadId"); + resp.upload_id_ = value; + + return resp; +} + +minio::PutObjectResponse minio::Client::PutObject(std::string_view bucket_name, std::string_view object_name, std::string_view data, minio::utils::Multimap headers, minio::utils::Multimap query_params) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::PutObjectResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::PutObjectResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kPut, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + request_builder.SetQueryParams(query_params); + request_builder.SetHeaders(headers); + request_builder.SetBody(data); + + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name, object_name); + if (!response.IsSuccess()) return response; + + minio::PutObjectResponse resp = response; + + std::string value = response.headers_.Get("etag").front(); + if (value.at(0) == '"') value.erase(0); + if (value.at(value.size()-1) == '"') value.erase(value.size()-1); + resp.etag_ = value; + resp.version_id_ = response.headers_.Get("x-amz-version-id").front(); + + return resp; +} + +minio::UploadPartResponse minio::Client::UploadPart(std::string_view bucket_name, std::string_view object_name, std::string_view upload_id, unsigned int part_number, std::string_view data, minio::utils::Multimap headers) { + if (!minio::utils::CheckNonEmptyString(upload_id)) { + minio::CompleteMultipartUploadResponse resp; + resp.err_ = "upload ID cannot be empty"; + return resp; + } + + minio::utils::Multimap query_params; + query_params.Add("partNumber", std::to_string(part_number)); + query_params.Add("uploadId", std::string(upload_id)); + + return PutObject(bucket_name, object_name, data, headers, query_params); +} + +minio::RetentionMode minio::StringToRetentionMode(std::string_view str) throw() { + if (str == "GOVERNANCE") return minio::RetentionMode::kGovernance; + if (str == "COMPLIANCE") return minio::RetentionMode::kCompliance; + + std::cerr << "ABORT: Unknown retention mode. This should not happen." + << std::endl; + std::terminate(); + + return minio::RetentionMode::kGovernance; +} + +minio::LegalHold minio::StringToLegalHold(std::string_view str) throw() { + if (str == "ON") return minio::LegalHold::kOn; + if (str == "OFF") return minio::LegalHold::kOff; + + std::cerr << "ABORT: Unknown legal hold. This should not happen." + << std::endl; + std::terminate(); + + return minio::LegalHold::kOff; +} + +minio::StatObjectResponse minio::Client::StatObject(std::string_view bucket_name, std::string_view object_name, std::string_view version_id) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::StatObjectResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::StatObjectResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kHead, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + minio::utils::Multimap query_params; + if (!version_id.empty()) { + query_params.Add("versionId", std::string(version_id)); + request_builder.SetQueryParams(query_params); + } + // request_builder.SetHeaders(headers); + + minio::http::Request request = request_builder.Build(provider_); + + minio::S3Response response = Execute(request, bucket_name, object_name); + if (!response.IsSuccess()) return response; + + minio::StatObjectResponse resp = response; + + resp.bucket_name_ = bucket_name; + resp.object_name_ = object_name; + resp.version_id_ = response.headers_.Get("x-amz-version-id").front(); + + std::string value = response.headers_.Get("etag").front(); + if (value.at(0) == '"') value.erase(0); + if (value.at(value.size()-1) == '"') value.erase(value.size()-1); + resp.etag_ = value; + + value = response.headers_.Get("content-length").front(); + if (!value.empty()) resp.size_ = std::stoi(value); + + value = response.headers_.Get("last-modified").front(); + if (!value.empty()) { + resp.last_modified_ = minio::utils::Time::FromHttpHeaderValue(value.c_str()); + } + + value = response.headers_.Get("x-amz-object-lock-mode").front(); + if (!value.empty()) resp.retention_mode_ = StringToRetentionMode(value); + + value = response.headers_.Get("x-amz-object-lock-retain-until-date").front(); + if (!value.empty()) { + resp.retention_retain_until_date_ = minio::utils::Time::FromISO8601UTC(value.c_str()); + } + + value = response.headers_.Get("x-amz-object-lock-legal-hold").front(); + if (!value.empty()) resp.legal_hold_ = StringToLegalHold(value); + + value = response.headers_.Get("x-amz-delete-marker").front(); + if (!value.empty()) resp.delete_marker_ = minio::utils::StringToBool(value); + + minio::utils::Multimap user_metadata_; + std::list keys = response.headers_.Keys(); + for (auto key : keys) { + if (minio::utils::StartsWith(key, "x-amz-meta-")) { + std::list values = response.headers_.Get(key); + key.erase(0, 11); + for (auto value : values) user_metadata_.Add(key, value); + } + } + resp.user_metadata_ = user_metadata_; + + return resp; +} + +minio::RemoveObjectResponse minio::Client::RemoveObject(std::string_view bucket_name, std::string_view object_name, std::string_view version_id) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::RemoveObjectResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::RemoveObjectResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kDelete, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + minio::utils::Multimap query_params; + if (!version_id.empty()) { + query_params.Add("versionId", std::string(version_id)); + request_builder.SetQueryParams(query_params); + } + + minio::http::Request request = request_builder.Build(provider_); + + return Execute(request, bucket_name, object_name); +} + +size_t minio::SaveData(minio::http::DataCallbackArgs args) { + std::ofstream *fout = (std::ofstream*) args.user_arg; + *fout << std::string(args.buffer, args.length); + return args.size * args.length; +} + +minio::DownloadObjectResponse minio::Client::DownloadObject(std::string_view bucket_name, std::string_view object_name, std::string_view filename, std::string_view version_id, bool overwrite) { + if (!minio::utils::CheckBucketName(bucket_name)) { + minio::DownloadObjectResponse resp; + resp.err_ = "invalid bucket name"; + return resp; + } + + if (!minio::utils::CheckNonEmptyString(object_name)) { + minio::DownloadObjectResponse resp; + resp.err_ = "object name cannot be empty"; + return resp; + } + + if (!overwrite && std::filesystem::exists(filename)) { + minio::DownloadObjectResponse resp; + resp.err_ = "file already exists"; + return resp; + } + + std::string etag; + size_t size; + { + StatObjectResponse response = StatObject(bucket_name, object_name, version_id); + if (!response.IsSuccess()) return response; + etag = response.etag_; + size = response.size_; + } + + std::string temp_filename = std::string(filename) + "." + curlpp::escape(etag) + ".part.minio"; + std::ofstream fout(temp_filename, fout.trunc | fout.out); + if (!fout.is_open()) { + minio::DownloadObjectResponse resp; + resp.err_ = "unable to open file " + temp_filename; + return resp; + } + + std::string region; + { + minio::GetRegionResponse response = GetRegion(bucket_name); + if (!response.IsSuccess()) return response; + region = response.region_; + } + + RequestBuilder request_builder(minio::http::Method::kGet, region, base_url_); + request_builder.SetBucketName(bucket_name); + request_builder.SetObjectName(object_name); + minio::utils::Multimap query_params; + if (!version_id.empty()) { + query_params.Add("versionId", std::string(version_id)); + request_builder.SetQueryParams(query_params); + } + request_builder.SetDataCallback(minio::SaveData, &fout); + + minio::http::Request request = request_builder.Build(provider_); + + S3Response response = Execute(request, bucket_name, object_name); + fout.close(); + if (response.IsSuccess()) std::filesystem::rename(temp_filename, filename); + return response; +} diff --git a/src/creds.cc b/src/creds.cc new file mode 100644 index 00000000..70966be8 --- /dev/null +++ b/src/creds.cc @@ -0,0 +1,55 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "creds.h" + +minio::creds::Credentials::Credentials(const Credentials& creds) { + access_key_ = creds.access_key_; + secret_key_ = creds.secret_key_; + session_token_ = creds.session_token_; + expiration_ = creds.expiration_; +} + +minio::creds::Credentials::Credentials(std::string_view access_key, + std::string_view secret_key, + std::string_view session_token, + unsigned int expiration) { + access_key_ = access_key; + secret_key_ = secret_key; + session_token_ = session_token; + expiration_ = expiration; +} + +std::string minio::creds::Credentials::AccessKey() { return std::string(access_key_); } + +std::string minio::creds::Credentials::SecretKey() { return std::string(secret_key_); } + +std::string minio::creds::Credentials::SessionToken() { + return std::string(session_token_); +} + +bool minio::creds::Credentials::IsExpired() { return expiration_ != 0; } + +minio::creds::StaticProvider::StaticProvider(std::string_view access_key, + std::string_view secret_key, + std::string_view session_token) { + creds_ = new minio::creds::Credentials(access_key, secret_key, session_token); +} + +minio::creds::StaticProvider::~StaticProvider() { delete creds_; } + +minio::creds::Credentials minio::creds::StaticProvider::Fetch() { + return *creds_; +} diff --git a/src/data.cc b/src/data.cc new file mode 100644 index 00000000..ac4cdd40 --- /dev/null +++ b/src/data.cc @@ -0,0 +1,45 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "data.h" + +minio::data::Bucket::Bucket(std::string name, minio::utils::Time creation_date) { + name_ = name; + creation_date_ = creation_date; +} + +std::string minio::data::Bucket::Name() { return name_; } + +minio::utils::Time minio::data::Bucket::CreationDate() { return creation_date_; } + +minio::data::Part::Part(unsigned int part_number, std::string_view etag, minio::utils::Time last_modified, size_t size) { + part_number_ = part_number; + etag_ = etag; + last_modified_ = last_modified; + size_ = size; +} + +minio::data::Part::Part(unsigned int part_number, std::string_view etag) { + part_number_ = part_number; + etag_ = etag; +} + +unsigned int minio::data::Part::PartNumber() { return part_number_; } + +std::string minio::data::Part::ETag() { return std::string(etag_); } + +minio::utils::Time minio::data::Part::LastModified() { return last_modified_; } + +size_t minio::data::Part::Size() { return size_; } diff --git a/src/http.cc b/src/http.cc new file mode 100644 index 00000000..9866f339 --- /dev/null +++ b/src/http.cc @@ -0,0 +1,395 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "http.h" + +std::string minio::http::ExtractRegion(std::string host) { + std::stringstream str_stream(host); + std::string token; + std::vector tokens; + while (std::getline(str_stream, token, '.')) tokens.push_back(token); + + token = tokens[1]; + + // If token is "dualstack", then region might be in next token. + if (token == "dualstack") token = tokens[2]; + + // If token is equal to "amazonaws", region is not passed in the host. + if (token == "amazonaws") return ""; + + // Return token as region. + return token; +} + +minio::http::BaseUrl::BaseUrl(std::string host) { + std::stringstream str_stream(host); + std::string port; + while (std::getline(str_stream, port, ':')) + ; + try { + port_ = std::stoi(port); + host = host.substr(0, host.rfind(":" + port)); + } catch (std::invalid_argument) { + port_ = 0; + } + + accelerate_host_flag_ = utils::StartsWith(host, "s3-accelerate."); + aws_host_ = ((utils::StartsWith(host, "s3.") || accelerate_host_flag_) && + (utils::EndsWith(host, ".amazonaws.com") || + utils::EndsWith(host, ".amazonaws.com.cn"))); + virtual_style_flag_ = aws_host_ || utils::EndsWith(host, "aliyuncs.com"); + + std::string aws_host; + if (aws_host_) { + bool is_aws_china_host = utils::EndsWith(host, ".cn"); + aws_host = "amazonaws.com"; + if (is_aws_china_host) aws_host = "amazonaws.com.cn"; + region_ = ExtractRegion(host); + + if (is_aws_china_host && region_.empty()) { + std::cerr << "ABORT: region missing in Amazon S3 China endpoint " << host + << std::endl; + std::terminate(); + } + + dualstack_host_flag_ = (host.find(".dualstack.") != std::string::npos); + host = aws_host; + } else { + accelerate_host_flag_ = false; + } + + host_ = host; +} + +std::string minio::http::BaseUrl::GetRegion() { return region_; } + +bool minio::http::BaseUrl::GetHttps() { return https_; } + +void minio::http::BaseUrl::SetHttps(bool flag) { https_ = flag; } + +std::string minio::http::BaseUrl::GetHost() { return host_; } + +unsigned int minio::http::BaseUrl::GetPort() { return port_; } + +bool minio::http::BaseUrl::IsAwsHost() { return aws_host_; } + +bool minio::http::BaseUrl::GetAccelerateHostFlag() { + return accelerate_host_flag_; +} + +void minio::http::BaseUrl::SetAccelerateHostFlag(bool flag) { + if (aws_host_) accelerate_host_flag_ = flag; +} + +bool minio::http::BaseUrl::GetDualstackHostFlag() { + return dualstack_host_flag_; +} + +void minio::http::BaseUrl::SetDualstackHostFlag(bool flag) { + if (aws_host_) dualstack_host_flag_ = flag; +} + +bool minio::http::BaseUrl::GetVirtualStyleFlag() { return virtual_style_flag_; } + +void minio::http::BaseUrl::SetVirtualStyleFlag(bool flag) { + virtual_style_flag_ = flag; +} + +std::string minio::http::BaseUrl::GetHostHeaderValue() { + // ignore port when port and service match i.e HTTP -> 80, HTTPS -> 443 + if (port_ == 0 || (!https_ && port_ == 80) || (https_ && port_ == 443)) { + return host_; + } + return host_ + ":" + std::to_string(port_); +} + +minio::utils::Url minio::http::BaseUrl::BuildUrl(Method method, + std::string region, + utils::Multimap query_params, + std::string bucket_name, + std::string object_name) { + if (bucket_name.empty() && !object_name.empty()) { + std::cerr << "ABORT: empty bucket name for object name " << object_name + << std::endl; + std::terminate(); + } + + std::string host = GetHostHeaderValue(); + + if (bucket_name.empty()) { + if (aws_host_) host = "s3." + region + "." + host; + return minio::utils::Url(https_, host, "/"); + } + + bool enforce_path_style = ( + // CreateBucket API requires path style in Amazon AWS S3. + (method == Method::kPut && object_name.empty() && + query_params.IsEmpty()) || + + // GetBucketLocation API requires path style in Amazon AWS S3. + query_params.Contains("location") || + + // Use path style for bucket name containing '.' which causes + // SSL certificate validation error. + ((bucket_name.find('.') != std::string::npos) && https_)); + + if (aws_host_) { + std::string s3_domain = "s3."; + if (accelerate_host_flag_) { + if (bucket_name.find('.') != std::string::npos) { + std::cerr << "ABORT: bucket name '" << bucket_name + << "' with '.' is not allowed for accelerate endpoint" + << std::endl; + std::terminate(); + } + + if (!enforce_path_style) s3_domain = "s3-accelerate."; + } + + if (dualstack_host_flag_) s3_domain += "dualstack."; + if (enforce_path_style || !accelerate_host_flag_) { + s3_domain += region + "."; + } + host = s3_domain + host; + } + + std::string path; + if (enforce_path_style || !virtual_style_flag_) { + path = "/" + bucket_name; + } else { + host = bucket_name + "." + host; + } + + if (!object_name.empty()) { + if (*(object_name.begin()) != '/') path += "/"; + path += utils::EncodePath(object_name); + } + + return minio::utils::Url(https_, host, path, query_params.ToQueryString()); +} + +void minio::http::Response::SetDataCallback(DataCallback callback, void* user_arg) { + callback_ = callback; + user_arg_ = user_arg; +} + +void minio::http::Response::SetError(std::string err) { err_ = err; } + +size_t minio::http::Response::ReadStatusCode(char* buffer, size_t size, + size_t length) { + size_t real_size = size * length; + + response_ += std::string(buffer, length); + + size_t pos = response_.find("\r\n"); + if (pos == std::string::npos) return real_size; + + std::string line = response_.substr(0, pos); + response_ = response_.substr(pos + 2); + + if (continue100_) { + if (!line.empty()) { + err_ = "invalid HTTP response"; + return real_size; + } + + continue100_ = false; + + pos = response_.find("\r\n"); + if (pos == std::string::npos) return real_size; + + line = response_.substr(0, pos); + response_ = response_.substr(pos + 2); + } + + // Skip HTTP/1.x. + pos = line.find(" "); + if (pos == std::string::npos) { + err_ = "invalid HTTP response"; + return real_size; + } + line = line.substr(pos + 1); + + // Read status code. + pos = line.find(" "); + if (pos == std::string::npos) { + err_ = "invalid HTTP response"; + return real_size; + } + std::string code = line.substr(0, pos); + std::string::size_type st; + status_code_ = std::stoi(code, &st); + if (st == std::string::npos) err_ = "invalid HTTP response code"; + + if (status_code_ == 100) { + continue100_ = true; + } else { + status_code_read_ = true; + } + + return real_size; +} + +size_t minio::http::Response::ReadHeaders(curlpp::Easy *handle, char* buffer, + size_t size, size_t length) { + size_t real_size = size * length; + + response_ += std::string(buffer, length); + size_t pos = response_.find("\r\n\r\n"); + if (pos == std::string::npos) return real_size; + + headers_read_ = true; + + std::string headers = response_.substr(0, pos); + body_ = response_.substr(pos + 4); + + while ((pos = headers.find("\r\n")) != std::string::npos) { + std::string line = headers.substr(0, pos); + headers.erase(0, pos + 2); + + if ((pos = line.find(": ")) == std::string::npos) { + err_ = "invalid HTTP header: " + line; + return real_size; + } + + headers_.Add(line.substr(0, pos), line.substr(pos + 2)); + } + + if (!headers.empty()) { + if ((pos = headers.find(": ")) == std::string::npos) { + err_ = "invalid HTTP header: " + headers; + return real_size; + } + + headers_.Add(headers.substr(0, pos), headers.substr(pos + 2)); + } + + if (body_.size() == 0 || callback_ == NULL || status_code_ < 200 || status_code_ > 299) return real_size; + + DataCallbackArgs args = {handle, this, body_.data(), 1, body_.size(), user_arg_}; + size_t written = callback_(args); + if (written == body_.size()) written = real_size; + body_ = ""; + return written; +} + +size_t minio::http::Response::ResponseCallback(curlpp::Easy *handle, + char* buffer, size_t size, + size_t length) { + size_t real_size = size * length; + + // As error occurred previously, just drain the connection. + if (!err_.empty()) return real_size; + + if (!status_code_read_) return ReadStatusCode(buffer, size, length); + + if (!headers_read_) return ReadHeaders(handle, buffer, size, length); + + // Received unsuccessful HTTP response code. + if (callback_ == NULL || status_code_ < 200 || status_code_ > 299) { + body_ += std::string(buffer, length); + return real_size; + } + + return callback_(DataCallbackArgs{handle, this, buffer, size, length, user_arg_}); +} + +bool minio::http::Response::IsSuccess() { + return err_.empty() && status_code_ >= 200 && status_code_ <= 299; +} + +std::string minio::http::Response::Error() { return err_; } + +int minio::http::Response::StatusCode() { return status_code_; } + +minio::utils::Multimap minio::http::Response::Headers() { return headers_; } + +std::string minio::http::Response::Data() { return body_; } + +minio::http::Request::Request(Method method, std::string url, std::string resource) { + method_ = method; + url_ = url; + resource_ = resource; +} + +void minio::http::Request::SetBody(std::string_view body) { body_ = body; } + +void minio::http::Request::SetHeaders(minio::utils::Multimap headers) { + headers_ = headers; +} + +void minio::http::Request::SetDataCallback(minio::http::DataCallback data_callback, void* user_arg) { + data_callback_ = data_callback; + user_arg_ = user_arg; +} + +std::string minio::http::Request::Resource() { return resource_; } + +minio::http::Method minio::http::Request::GetMethod() { return method_; } + +minio::http::Response minio::http::Request::execute() { + curlpp::Cleanup cleaner; + curlpp::Easy request; + + // Request settings. + request.setOpt(new curlpp::options::CustomRequest{minio::http::MethodToString(method_)}); + request.setOpt(new curlpp::Options::Url(url_)); + // request.setOpt(new curlpp::Options::Verbose(true)); + + std::stringstream body_stream(body_.data()); + + switch (method_) { + case minio::http::Method::kPut: + case minio::http::Method::kPost: + if (!headers_.Contains("Content-Length")) { + headers_.Add("Content-Length", std::to_string(body_.size())); + } + request.setOpt(new curlpp::Options::ReadStream(&body_stream)); + request.setOpt(new curlpp::Options::InfileSize(body_.size())); + request.setOpt(new curlpp::Options::Upload(true)); + } + + std::list headers = headers_.ToHttpHeaders(); + headers.push_back("Expect:"); // Disable 100 continue from server. + request.setOpt(new curlpp::Options::HttpHeader(headers)); + + // Response settings. + request.setOpt(new curlpp::options::Header(true)); + + minio::http::Response response; + if (data_callback_ != NULL) response.SetDataCallback(data_callback_, user_arg_); + + using namespace std::placeholders; + request.setOpt(new curlpp::options::WriteFunction(std::bind(&minio::http::Response::ResponseCallback, &response, &request, _1, _2, _3))); + + // Execute. + request.perform(); + + return response; +} + +minio::http::Response minio::http::Request::Execute() { + try { + return minio::http::Request::execute(); + } catch (curlpp::LogicError &e) { + minio::http::Response response; + response.SetError(std::string("curlpp::LogicError: ") + e.what()); + return response; + } catch (curlpp::RuntimeError &e) { + minio::http::Response response; + response.SetError(std::string("curlpp::RuntimeError: ") + e.what()); + return response; + } +} diff --git a/src/signer.cc b/src/signer.cc new file mode 100644 index 00000000..41432644 --- /dev/null +++ b/src/signer.cc @@ -0,0 +1,147 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "signer.h" + +const char* SIGN_V4_ALGORITHM = "AWS4-HMAC-SHA256"; +const std::regex MULTI_SPACE_REGEX("( +)"); + +std::string minio::signer::GetScope(minio::utils::Time& time, + std::string_view region, + std::string_view service_name) { + return time.ToSignerDate() + "/" + std::string(region) + "/" + + std::string(service_name) + "/aws4_request"; +} + +std::string minio::signer::GetCanonicalRequestHash( + std::string_view method, std::string_view uri, + std::string_view query_string, std::string_view headers, + std::string_view signed_headers, std::string_view content_sha256) { + // CanonicalRequest = + // HTTPRequestMethod + '\n' + + // CanonicalURI + '\n' + + // CanonicalQueryString + '\n' + + // CanonicalHeaders + '\n\n' + + // SignedHeaders + '\n' + + // HexEncode(Hash(RequestPayload)) + std::string canonical_request = std::string(method) + "\n" + + std::string(uri) + "\n" + std::string(query_string) + "\n" + + std::string(headers) + "\n\n" + std::string(signed_headers) + "\n" + + std::string(content_sha256); + return minio::utils::Sha256Hash(canonical_request); +} + +std::string minio::signer::GetStringToSign( + minio::utils::Time& date, std::string_view scope, + std::string_view canonical_request_hash) { + return "AWS4-HMAC-SHA256\n" + date.ToAmzDate() + "\n" + std::string(scope) + + "\n" + std::string(canonical_request_hash); +} + +std::string minio::signer::HmacHash(std::string_view key, + std::string_view data) { + std::array hash; + unsigned int hash_len; + + HMAC(EVP_sha256(), key.data(), static_cast(key.size()), + reinterpret_cast(data.data()), + static_cast(data.size()), hash.data(), &hash_len); + + return std::string{reinterpret_cast(hash.data()), hash_len}; +} + +std::string minio::signer::GetSigningKey(std::string_view secret_key, + minio::utils::Time& date, + std::string_view region, + std::string_view service_name) { + std::string date_key = + HmacHash("AWS4" + std::string(secret_key), date.ToSignerDate()); + std::string date_region_key = HmacHash(date_key, region); + std::string date_region_service_key = HmacHash(date_region_key, service_name); + return HmacHash(date_region_service_key, "aws4_request"); +} + +std::string minio::signer::GetSignature(std::string_view signing_key, + std::string_view string_to_sign) { + std::string hash = HmacHash(signing_key, string_to_sign); + std::string signature; + char buf[3]; + for (int i = 0; i < hash.size(); ++i) { + sprintf(buf, "%02x", (unsigned char)hash[i]); + signature += buf; + } + return signature; +} + +std::string minio::signer::GetAuthorization(std::string_view access_key, + std::string_view scope, + std::string_view signed_headers, + std::string_view signature) { + return "AWS4-HMAC-SHA256 Credential=" + std::string(access_key) + "/" + + std::string(scope) + ", " + + "SignedHeaders=" + std::string(signed_headers) + ", " + + "Signature=" + std::string(signature); +} + +minio::utils::Multimap& minio::signer::SignV4( + std::string_view service_name, minio::http::Method& method, + std::string_view uri, std::string_view region, + minio::utils::Multimap& headers, minio::utils::Multimap& query_params, + std::string_view access_key, std::string_view secret_key, + std::string_view content_sha256, minio::utils::Time& date) { + std::string scope = GetScope(date, region, service_name); + + std::string signed_headers; + std::string canonical_headers; + headers.GetCanonicalHeaders(signed_headers, canonical_headers); + + std::string canonical_query_string = query_params.GetCanonicalQueryString(); + + std::string canonical_request_hash = GetCanonicalRequestHash( + minio::http::MethodToString(method), uri, canonical_query_string, + canonical_headers, signed_headers, content_sha256); + + std::string string_to_sign = + GetStringToSign(date, scope, canonical_request_hash); + + std::string signing_key = + GetSigningKey(secret_key, date, region, service_name); + + std::string signature = GetSignature(signing_key, string_to_sign); + + std::string authorization = + GetAuthorization(access_key, scope, signed_headers, signature); + + headers.Add("Authorization", authorization); + return headers; +} + +minio::utils::Multimap& minio::signer::SignV4S3( + minio::http::Method& method, std::string_view uri, std::string_view region, + minio::utils::Multimap& headers, minio::utils::Multimap& query_params, + std::string_view access_key, std::string_view secret_key, + std::string_view content_sha256, minio::utils::Time& date) { + return SignV4("s3", method, uri, region, headers, query_params, access_key, + secret_key, content_sha256, date); +} + +minio::utils::Multimap& minio::signer::SignV4STS( + minio::http::Method& method, std::string_view uri, std::string_view region, + minio::utils::Multimap& headers, minio::utils::Multimap& query_params, + std::string_view access_key, std::string_view secret_key, + std::string_view content_sha256, minio::utils::Time& date) { + return SignV4("sts", method, uri, region, headers, query_params, access_key, + secret_key, content_sha256, date); +} diff --git a/src/utils.cc b/src/utils.cc new file mode 100644 index 00000000..98291ff0 --- /dev/null +++ b/src/utils.cc @@ -0,0 +1,386 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils.h" +#include + +const char *HTTP_HEADER_FORMAT = "%a, %d %b %Y %H:%M:%S GMT"; +const std::regex MULTI_SPACE_REGEX("( +)"); +const std::regex VALID_BUCKET_NAME_REGEX("^[A-Za-z0-9][A-Za-z0-9\\.\\-\\_\\:]{1,61}[A-Za-z0-9]$"); +const std::regex VALID_BUCKET_NAME_STRICT_REGEX("^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$"); +const std::regex VALID_IP_ADDR_REGEX("^(\\d+\\.){3}\\d+$"); + +bool minio::utils::StringToBool(std::string_view str) { + std::string s = ToLower(std::string(str)); + if (s == "false") return false; + if (s == "true") return true; + + std::cerr << "ABORT: Unknown bool string. This should not happen." + << std::endl; + std::terminate(); + + return false; +} + +std::string minio::utils::TrimSpaces(std::string_view str) { + int start, len; + for (start = 0; start < str.size() && str[start] == ' '; start++); + for (len = str.size() - start; len > 0 && str[start + len - 1] == ' '; len--); + return std::string(str.substr(start, len)); +} + +bool minio::utils::CheckNonEmptyString(std::string_view str) { + return !str.empty() && TrimSpaces(str) == str; +} + +std::string minio::utils::ToLower(std::string str) { + std::string s = str; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + return s; +} + +bool minio::utils::StartsWith(std::string_view str, std::string_view prefix) { + return (str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0); +} + +bool minio::utils::EndsWith(std::string_view str, std::string_view suffix) { + return (str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0); +} + +std::string minio::utils::EncodePath(std::string_view path) { + std::stringstream str_stream{std::string(path)}; + std::string token; + std::string out; + while (std::getline(str_stream, token, '/')) { + if (!token.empty()) { + if (!out.empty()) out += "/"; + out += curlpp::escape(token); + } + } + + if (*(path.begin()) == '/') out = "/" + out; + if (*(path.end() - 1) == '/' && out != "/") out += "/"; + + return out; +} + +std::string minio::utils::Sha256Hash(std::string_view str) { + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, str.data(), str.size()); + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_Final(hash, &ctx); + + char buf[65]; + for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i) { + sprintf(buf + (i * 2), "%02x", hash[i]); + } + buf[64] = 0; + + return std::string(buf); +} + +std::string minio::utils::Base64Encode(std::string_view str) { + const auto base64_memory = BIO_new(BIO_s_mem()); + auto base64 = BIO_new(BIO_f_base64()); + base64 = BIO_push(base64, base64_memory); + + BIO_write(base64, str.data(), str.size()); + BIO_flush(base64); + + BUF_MEM *buf_mem{}; + BIO_get_mem_ptr(base64, &buf_mem); + auto base64_encoded = std::string(buf_mem->data, buf_mem->length - 1); + + BIO_free_all(base64); + + return base64_encoded; +} + +std::string minio::utils::Md5sumHash(std::string_view str) { + MD5_CTX ctx; + MD5_Init(&ctx); + MD5_Update(&ctx, str.data(), str.size()); + unsigned char hash[MD5_DIGEST_LENGTH]; + MD5_Final(hash, &ctx); + return Base64Encode(std::string_view((const char *) hash, MD5_DIGEST_LENGTH)); +} + +std::string minio::utils::FormatTime(const std::tm *time, const char *format) { + char buf[128]; + std::strftime(buf, 128, format, time); + return std::string(buf); +} + +minio::utils::Time::Time() { + gettimeofday(&tv_, NULL); + utc_ = false; +} + +minio::utils::Time::Time(std::time_t tv_sec, suseconds_t tv_usec, bool utc) { + tv_.tv_sec = tv_sec; + tv_.tv_usec = tv_usec; + utc_ = utc; +} + +std::tm *minio::utils::Time::ToUTC() { + std::tm *t = new std::tm; + *t = utc_ ? *std::localtime(&tv_.tv_sec) : *std::gmtime(&tv_.tv_sec); + return t; +} + +std::string minio::utils::Time::ToSignerDate() { + std::tm *utc = ToUTC(); + std::string result = FormatTime(utc, "%Y%m%d"); + delete utc; + return result; +} + +std::string minio::utils::Time::ToAmzDate() { + std::tm *utc = ToUTC(); + std::string result = FormatTime(utc, "%Y%m%dT%H%M%SZ"); + delete utc; + return result; +} + +std::string minio::utils::Time::ToHttpHeaderValue() { + std::tm *utc = ToUTC(); + std::locale("C"); + std::string result = FormatTime(utc, HTTP_HEADER_FORMAT); + std::locale(""); + delete utc; + return result; +} + +minio::utils::Time minio::utils::Time::FromHttpHeaderValue(const char *value) { + std::tm t{0}; + std::locale("C"); + strptime(value, HTTP_HEADER_FORMAT, &t); + std::locale(""); + std::time_t time = std::mktime(&t); + return Time(time, 0, true); +} + +std::string minio::utils::Time::ToISO8601UTC() { + char buf[64]; + snprintf(buf, 64, "%03d", tv_.tv_usec); + std::string usec_str(buf); + if (usec_str.size() > 3) usec_str = usec_str.substr(0, 3); + std::tm *utc = ToUTC(); + std::string result = FormatTime(utc, "%Y-%m-%dT%H:%M:%S.") + usec_str + "Z"; + delete utc; + return result; +} + +minio::utils::Time minio::utils::Time::FromISO8601UTC(const char *value) { + std::tm t{0}; + suseconds_t tv_usec = 0; + char *rv = strptime(value, "%Y-%m-%dT%H:%M:%S", &t); + sscanf(rv, ".%u", &tv_usec); + std::time_t time = std::mktime(&t); + return Time(time, tv_usec, true); +} + +minio::utils::Multimap::Multimap() {} + +minio::utils::Multimap::Multimap(const Multimap &headers) { + this->AddAll(headers); +} + +void minio::utils::Multimap::Add(std::string key, std::string value) { + map_[key].insert(value); + keys_[ToLower(key)].insert(key); +} + +void minio::utils::Multimap::AddAll(const Multimap &headers) { + auto m = headers.map_; + std::map>::iterator i; + for (i = m.begin(); i != m.end(); ++i) { + map_[i->first].insert(i->second.begin(), i->second.end()); + keys_[ToLower(i->first)].insert(i->first); + } +} + +std::list minio::utils::Multimap::ToHttpHeaders() { + std::list headers; + std::map>::iterator i; + for (i = map_.begin(); i != map_.end(); ++i) { + std::set::iterator itr; + for (itr = i->second.begin(); itr != i->second.end(); ++itr) { + headers.push_back(i->first + ": " + *itr); + } + } + return headers; +} + +std::string minio::utils::Multimap::ToQueryString() { + std::string query_string; + std::map>::iterator i; + for (i = map_.begin(); i != map_.end(); ++i) { + std::set::iterator itr; + for (itr = i->second.begin(); itr != i->second.end(); ++itr) { + std::string s = curlpp::escape(i->first) + "=" + curlpp::escape(*itr); + if (!query_string.empty()) query_string += "&"; + query_string += s; + } + } + return query_string; +} + +bool minio::utils::Multimap::IsEmpty() { return map_.empty(); } + +bool minio::utils::Multimap::Contains(std::string_view key) { + return keys_.find(ToLower(std::string(key))) != map_.end(); +} + +std::list minio::utils::Multimap::Get(std::string_view key) { + std::list result; + std::set keys = keys_[minio::utils::ToLower(std::string(key))]; + std::set::iterator i; + for (i = keys.begin(); i != keys.end(); ++i) { + std::set values = map_[*i]; + result.insert(result.end(), values.begin(), values.end()); + } + return result; +} + +std::list minio::utils::Multimap::Keys() { + std::list keys; + // keys.reserve(keys_.size()); + for (const auto& [key, _] : keys_) keys.push_back(key); + return keys; +} + +void minio::utils::Multimap::GetCanonicalHeaders( + std::string &signed_headers, std::string &canonical_headers) { + std::vector signed_headers_list; + std::map map; + + std::map>::iterator i; + for (i = map_.begin(); i != map_.end(); ++i) { + std::string key = i->first; + std::transform(key.begin(), key.end(), key.begin(), ::tolower); + if ("authorization" == key || "user-agent" == key) continue; + if (std::find(signed_headers_list.begin(), signed_headers_list.end(), + key) == signed_headers_list.end()) { + signed_headers_list.push_back(key); + } + + std::string value; + std::set::iterator itr; + for (itr = i->second.begin(); itr != i->second.end(); ++itr) { + if (!value.empty()) value += ","; + value += std::regex_replace(*itr, MULTI_SPACE_REGEX, " "); + } + + map[key] = value; + } + + std::vector::iterator itr; + + std::sort(signed_headers_list.begin(), signed_headers_list.end()); + for (itr = signed_headers_list.begin(); itr != signed_headers_list.end(); + ++itr) { + if (!signed_headers.empty()) signed_headers += ";"; + signed_headers += *itr; + } + + std::vector canonical_headers_list; + std::map::iterator j; + for (j = map.begin(); j != map.end(); ++j) { + canonical_headers_list.push_back(j->first + ":" + j->second); + } + + std::sort(canonical_headers_list.begin(), canonical_headers_list.end()); + for (itr = canonical_headers_list.begin(); + itr != canonical_headers_list.end(); ++itr) { + if (!canonical_headers.empty()) canonical_headers += "\n"; + canonical_headers += *itr; + } +} + +std::string minio::utils::Multimap::GetCanonicalQueryString() { + std::vector values; + std::map>::iterator i; + for (i = map_.begin(); i != map_.end(); ++i) { + std::set::iterator itr; + for (itr = i->second.begin(); itr != i->second.end(); ++itr) { + std::string s = curlpp::escape(i->first) + "=" + curlpp::escape(*itr); + values.push_back(s); + } + } + + std::sort(values.begin(), values.end()); + + std::string result; + std::vector::iterator itr; + for (itr = values.begin(); itr != values.end(); ++itr) { + if (!result.empty()) result += "&"; + result += *itr; + } + + return result; +} + +minio::utils::Url::Url(bool https_flag, std::string_view host, + std::string_view path, std::string_view query_string) { + https_ = https_flag; + host_ = std::string(host); + path_ = std::string(path); + query_string_ = std::string(query_string); +} + +std::string minio::utils::Url::String() { + std::string url = (https_ ? "https://" : "http://") + host_; + + if (!path_.empty()) { + if (*(path_.begin()) != '/') url += "/"; + url += path_; + } + + if (!query_string_.empty()) url += "?" + query_string_; + + return url; +} + +std::string minio::utils::Url::Host() { return host_; } + +std::string minio::utils::Url::Path() { return path_; } + +bool minio::utils::Url::IsHttps() { return https_; } + +bool minio::utils::CheckBucketName(std::string_view bucket_name, bool strict) { + // whilespace check. + if (TrimSpaces(bucket_name) != bucket_name) return false; + + // size check. + if (bucket_name.size() < 3 || bucket_name.size() > 63) return false; + + // IP address check. + if (std::regex_match(bucket_name.data(), VALID_IP_ADDR_REGEX)) return false; + + // unallowed successive characters check. + if (bucket_name.find("..") != std::string::npos || + bucket_name.find(".-") != std::string::npos || + bucket_name.find("-.") != std::string::npos) return false; + + if (strict) { + return std::regex_match(bucket_name.data(), VALID_BUCKET_NAME_STRICT_REGEX); + } + + return std::regex_match(bucket_name.data(), VALID_BUCKET_NAME_REGEX); +} diff --git a/src/xml.cc b/src/xml.cc new file mode 100644 index 00000000..662b168e --- /dev/null +++ b/src/xml.cc @@ -0,0 +1,45 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "xml.h" + +bool minio::xml::GetValue(std::string& value, std::string_view xml_data, std::string_view xpath) { + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(xml_data.data()); + + if (result) { + pugi::xpath_node_set nodes = xdoc.select_nodes(xpath.data()); + for (pugi::xpath_node_set::const_iterator i = nodes.begin(); i != nodes.end(); ++i) { + for (pugi::xml_node node = i->node().first_child(); node; node = node.next_sibling()) { + value = node.value(); + return true; + } + } + } + + return false; +} + +bool minio::xml::GetNodes(pugi::xpath_node_set& nodes, std::string_view xml_data, std::string_view xpath) { + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(xml_data.data()); + + if (result) { + nodes = xdoc.select_nodes(xpath.data()); + return true; + } + + return false; +}