diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index cb956600..b00968d2 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -39,29 +39,38 @@ jobs: ./vcpkg-master/bootstrap-vcpkg.sh ./vcpkg-master/vcpkg integrate install - - name: Configure CMake - # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_TOOLCHAIN_FILE=${{github.workspace}}/vcpkg-master/scripts/buildsystems/vcpkg.cmake + - name: Run debug build + run: | + cmake -B ./build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=./vcpkg-master/scripts/buildsystems/vcpkg.cmake + cmake --build ./build --config Debug + + - name: Start MinIO server + run: | + wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio + chmod +x minio + mkdir -p ~/.minio/certs + cp ./tests/public.crt ./tests/private.key ~/.minio/certs/ + sudo cp ./tests/public.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + MINIO_CI_CD=true ./minio server /tmp/test-xl/{1...4}/ & + sleep 10 + + - name: Run tests on debug build + run: | + S3HOST=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ./build/tests/tests - - name: Build - # Build your program with the given configuration + - name: Run release build run: | - cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} - wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio && chmod +x minio - mkdir -p ~/.minio/certs/ && cp testdata/localhost.crt ~/.minio/certs/public.crt && cp testdata/localhost.key ~/.minio/certs/private.key - sudo cp testdata/localhost.crt /usr/local/share/ca-certificates/ && sudo update-ca-certificates - export MINIO_CI_CD=true - ./minio server /tmp/test-xl/{1...4}/ & sleep 10 - export ENDPOINT=https://localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin - ${{github.workspace}}/build/examples/s3 -d -a mb -n bucketname - ${{github.workspace}}/build/examples/s3 -d -a up -f ${{github.workspace}}/testdata/localhost.crt -n bucketname -k objectname - ${{github.workspace}}/build/examples/s3 -d -a down -f /tmp/local.crt -n bucketname -k objectname - ${{github.workspace}}/build/examples/s3 -d -a rm -n bucketname -k objectname - ${{github.workspace}}/build/examples/s3 -d -a rb -n bucketname + cmake --build ./build --target clean + cmake -B ./build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=./vcpkg-master/scripts/buildsystems/vcpkg.cmake + cmake --build ./build --config Release + + - name: Run tests on release build + run: | + S3HOST=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ./build/tests/tests + - name: Test working-directory: ${{github.workspace}}/build # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C ${{env.BUILD_TYPE}} - diff --git a/.gitignore b/.gitignore index ad16b150..4629ce45 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,12 @@ build/ *.pc .vs CMakeSettings.json -vcpkg_installed/ \ No newline at end of file +vcpkg_installed/ +docs/search +docs/*html +docs/*png +docs/*js +docs/*css +docs/*svg +docs/*map +docs/*md5 diff --git a/CMakeLists.txt b/CMakeLists.txt index f666aa74..69a0bf8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ macro(set_globals) endmacro() # specify the C++ standard -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -79,8 +79,8 @@ INCLUDE (CheckIncludeFiles) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/include) SET(MINIOCPP_MAJOR_VERSION "0") -SET(MINIOCPP_MINOR_VERSION "0") -SET(MINIOCPP_PATCH_VERSION "1") +SET(MINIOCPP_MINOR_VERSION "1") +SET(MINIOCPP_PATCH_VERSION "0") add_subdirectory(include) add_subdirectory(src) @@ -91,6 +91,11 @@ if (BUILD_EXAMPLES) add_subdirectory(examples) endif (BUILD_EXAMPLES) +option(BUILD_TESTS "Build tests" ON) +if (BUILD_TESTS) + add_subdirectory(tests) +endif (BUILD_TESTS) + option(BUILD_DOC "Build documentation" ON) # check if Doxygen is installed diff --git a/README.md b/README.md index 90822b7d..0acd6f14 100644 --- a/README.md +++ b/README.md @@ -2,63 +2,98 @@ # MinIO C++ Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-cpp/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-cpp?badge) [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-cpp/blob/master/LICENSE) -The MinIO C++ Client SDK provides simple APIs to access any Amazon S3 compatible object storage. This quickstart guide will show you how to install the MinIO client SDK, connect to MinIO, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [MinIO C++ Client API Reference](https://minio-cpp.min.io/) +MinIO C++ SDK is Simple Storage Service (aka S3) client to perform bucket and object operations to any Amazon S3 compatible object storage service. -This document assumes that you have a working C++ development environment. In order to build this project, you need the Cross-Platform Make CMake 3.10 or higher, [vcpkg](https://vcpkg.io/en/index.html). +For a complete list of APIs and examples, please take a look at the [MinIO C++ Client API Reference](https://minio-cpp.min.io/) + +## Build requirements +* A working C++ development environment supporting C++17 standards. +* CMake 3.10 or higher. +* [vcpkg](https://vcpkg.io/en/index.html). ## Install from `vcpkg` ``` vcpkg install minio-cpp ``` -## Source build - -``` -git clone https://github.com/minio/minio-cpp; cd minio-cpp; -cmake -B build -S . -DCMAKE_TOOLCHAIN_FILE=${VCPKGDIR}/scripts/buildsystems/vcpkg.cmake -cmake --build build +## Building source +```bash +$ git clone https://github.com/minio/minio-cpp +$ cd minio-cpp +$ wget --quiet -O vcpkg-master.zip https://github.com/microsoft/vcpkg/archive/refs/heads/master.zip +$ unzip -qq vcpkg-master.zip +$ ./vcpkg-master/bootstrap-vcpkg.sh +$ ./vcpkg-master/vcpkg integrate install +$ cmake -B ./build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=./vcpkg-master/scripts/buildsystems/vcpkg.cmake +$ cmake --build ./build --config Debug ``` -## Example code +## Example:: file-uploader.cc ```c++ -#include -#include -#include -#include -#include - -using namespace Minio; - -int -main ( int argc, char** argv ) -{ - S3Client s3("https://play.min.io:9000", "minioadmin", "minioadmin"); - S3ClientIO io; - s3.MakeBucket("newbucket", io); - if(io.Failure()) { - std::cerr << "ERROR: failed to create bucket" << endl; - std::cerr << "response:\n" << io << endl; - std::cerr << "response body:\n" << io.response.str() << endl; - return -1; +#include + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + std::string bucket_name = "asiatrip"; + + // Check 'asiatrip' bucket exist or not. + bool exist; + { + minio::s3::BucketExistsArgs args; + args.bucket_ = bucket_name; + + minio::s3::BucketExistsResponse resp = client.BucketExists(args); + if (!resp) { + std::cout << "unable to do bucket existence check; " << resp.GetError() + << std::endl; + return EXIT_FAILURE; + } + + exist = resp.exist_; } - return 0; -} -``` -## Run an example -Following example runs 'multipart' upload, uploads a single part. You would have to choose a local file to upload for `-f`, and also remote bucket to upload the object to as `-n` and final object name in the bucket as `-k`. + // Make 'asiatrip' bucket if not exist. + if (!exist) { + minio::s3::MakeBucketArgs args; + args.bucket_ = bucket_name; -``` -export ACTION="multipart" -export ACCESS_KEY=minioadmin -export SECRET_KEY=minioadmin -export ENDPOINT="https://play.min.io:9000" + minio::s3::MakeBucketResponse resp = client.MakeBucket(args); + if (!resp) { + std::cout << "unable to create bucket; " << resp.GetError() << std::endl; + return EXIT_FAILURE; + } + } -./examples/s3 -a ${ACTION} -f \ - -n -k -``` + // Upload '/home/user/Photos/asiaphotos.zip' as object name + // 'asiaphotos-2015.zip' to bucket 'asiatrip'. + minio::s3::UploadObjectArgs args; + args.bucket_ = bucket_name; + args.object_ = "asiaphotos-2015.zip"; + args.filename_ = "/home/user/Photos/asiaphotos.zip"; -Please choose a `` that exists. + minio::s3::UploadObjectResponse resp = client.UploadObject(args); + if (!resp) { + std::cout << "unable to upload object; " << resp.GetError() << std::endl; + return EXIT_FAILURE; + } + + std::cout << "'/home/user/Photos/asiaphotos.zip' is successfully uploaded as " + << "object 'asiaphotos-2015.zip' to bucket 'asiatrip'." + << std::endl; + + return EXIT_SUCCESS; +} +``` ## License This SDK is distributed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](https://github.com/minio/minio-cpp/blob/master/LICENSE) for more information. diff --git a/check-style.sh b/check-style.sh index b776203d..24ba6ad6 100755 --- a/check-style.sh +++ b/check-style.sh @@ -10,7 +10,7 @@ function clang_format() { } ec=0 -mapfile -t files < <(find . -iname "*.cpp" -o -iname "*.h") +mapfile -t files < <(find src include examples tests -iname "*.cc" -o -iname "*.h") for file in "${files[@]}"; do if ! clang_format "$file"; then ec=255 diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 4f0b82ae..d4cab419 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -1,5 +1,7 @@ -OUTPUT_DIRECTORY = @CMAKE_CURRENT_SOURCE_DIR@/docs/ -INPUT = @CMAKE_CURRENT_SOURCE_DIR@/src/ @CMAKE_CURRENT_SOURCE_DIR@/docs @CMAKE_CURRENT_SOURCE_DIR@/include @CMAKE_CURRENT_SOURCE_DIR@/examples +OUTPUT_DIRECTORY = @CMAKE_CURRENT_SOURCE_DIR@/docs +INPUT = @CMAKE_CURRENT_SOURCE_DIR@/docs/README.md @CMAKE_CURRENT_SOURCE_DIR@/src @CMAKE_CURRENT_SOURCE_DIR@/include PROJECT_NAME = "MinIO C++ SDK" GENERATE_LATEX = NO -HTML_OUTPUT = @CMAKE_CURRENT_SOURCE_DIR@/docs/ +WARN_IF_UNDOCUMENTED = NO +USE_MDFILE_AS_MAINPAGE = @CMAKE_CURRENT_SOURCE_DIR@/docs/README.md +HTML_OUTPUT = @CMAKE_CURRENT_SOURCE_DIR@/docs diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..915168b1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,97 @@ +> NOTE: This project is work in progress. + +MinIO C++ SDK is Simple Storage Service (aka S3) client to perform bucket and object operations to any Amazon S3 compatible object storage service. + +For a complete list of APIs and examples, please take a look at the [MinIO C++ Client API Reference](https://minio-cpp.min.io/) + +## Build requirements +* A working C++ development environment supporting C++17 standards. +* CMake 3.10 or higher. +* [vcpkg](https://vcpkg.io/en/index.html). + +## Install from vcpkg +``` +vcpkg install minio-cpp +``` + +## Building source +``` +$ git clone https://github.com/minio/minio-cpp +$ cd minio-cpp +$ wget --quiet -O vcpkg-master.zip https://github.com/microsoft/vcpkg/archive/refs/heads/master.zip +$ unzip -qq vcpkg-master.zip +$ ./vcpkg-master/bootstrap-vcpkg.sh +$ ./vcpkg-master/vcpkg integrate install +$ cmake -B ./build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=./vcpkg-master/scripts/buildsystems/vcpkg.cmake +$ cmake --build ./build --config Debug +``` + +## Example:: file-uploader.cc +``` +#include + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + std::string bucket_name = "asiatrip"; + + // Check 'asiatrip' bucket exist or not. + bool exist; + { + minio::s3::BucketExistsArgs args; + args.bucket_ = bucket_name; + + minio::s3::BucketExistsResponse resp = client.BucketExists(args); + if (!resp) { + std::cout << "unable to do bucket existence check; " << resp.GetError() + << std::endl; + return EXIT_FAILURE; + } + + exist = resp.exist_; + } + + // Make 'asiatrip' bucket if not exist. + if (!exist) { + minio::s3::MakeBucketArgs args; + args.bucket_ = bucket_name; + + minio::s3::MakeBucketResponse resp = client.MakeBucket(args); + if (!resp) { + std::cout << "unable to create bucket; " << resp.GetError() << std::endl; + return EXIT_FAILURE; + } + } + + // Upload '/home/user/Photos/asiaphotos.zip' as object name + // 'asiaphotos-2015.zip' to bucket 'asiatrip'. + minio::s3::UploadObjectArgs args; + args.bucket_ = bucket_name; + args.object_ = "asiaphotos-2015.zip"; + args.filename_ = "/home/user/Photos/asiaphotos.zip"; + + minio::s3::UploadObjectResponse resp = client.UploadObject(args); + if (!resp) { + std::cout << "unable to upload object; " << resp.GetError() << std::endl; + return EXIT_FAILURE; + } + + std::cout << "'/home/user/Photos/asiaphotos.zip' is successfully uploaded as " + << "object 'asiaphotos-2015.zip' to bucket 'asiatrip'." + << std::endl; + + return EXIT_SUCCESS; +} +``` + +## License +This SDK is distributed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](https://github.com/minio/minio-cpp/blob/master/LICENSE) for more information. diff --git a/examples/BucketExists.cc b/examples/BucketExists.cc new file mode 100644 index 00000000..be54bcd2 --- /dev/null +++ b/examples/BucketExists.cc @@ -0,0 +1,50 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create bucket exists arguments. + minio::s3::BucketExistsArgs args; + args.bucket = "my-bucket"; + + // Call bucket exists. + minio::s3::BucketExistsResponse resp = client.BucketExists(args); + + // Handle response. + if (resp) { + if (resp.exist) { + std::cout << "my-bucket exists" << std::endl; + } else { + std::cout << "my-bucket does not exist" << std::endl; + } + } else { + std::cout << "unable to do bucket existence check; " << resp.GetError() + << std::endl; + } + + return 0; +} diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 1f0ef815..d8bf3875 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,6 +1,40 @@ -ADD_EXECUTABLE(s3 s3.cpp) - SET(S3_LIBS ${requiredlibs}) -TARGET_LINK_LIBRARIES(s3 miniocpp ${S3_LIBS}) -INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/s3 DESTINATION bin) +ADD_EXECUTABLE(MakeBucket MakeBucket.cc) +TARGET_LINK_LIBRARIES(MakeBucket miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(RemoveBucket RemoveBucket.cc) +TARGET_LINK_LIBRARIES(RemoveBucket miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(BucketExists BucketExists.cc) +TARGET_LINK_LIBRARIES(BucketExists miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(ListBuckets ListBuckets.cc) +TARGET_LINK_LIBRARIES(ListBuckets miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(StatObject StatObject.cc) +TARGET_LINK_LIBRARIES(StatObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(RemoveObject RemoveObject.cc) +TARGET_LINK_LIBRARIES(RemoveObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(DownloadObject DownloadObject.cc) +TARGET_LINK_LIBRARIES(DownloadObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(UploadObject UploadObject.cc) +TARGET_LINK_LIBRARIES(UploadObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(GetObject GetObject.cc) +TARGET_LINK_LIBRARIES(GetObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(ListObjects ListObjects.cc) +TARGET_LINK_LIBRARIES(ListObjects miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(PutObject PutObject.cc) +TARGET_LINK_LIBRARIES(PutObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(CopyObject CopyObject.cc) +TARGET_LINK_LIBRARIES(CopyObject miniocpp ${S3_LIBS}) + +ADD_EXECUTABLE(ComposeObject ComposeObject.cc) +TARGET_LINK_LIBRARIES(ComposeObject miniocpp ${S3_LIBS}) diff --git a/examples/ComposeObject.cc b/examples/ComposeObject.cc new file mode 100644 index 00000000..a86c7c5a --- /dev/null +++ b/examples/ComposeObject.cc @@ -0,0 +1,60 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Composeright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a compose of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create compose object arguments. + minio::s3::ComposeObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + + std::list sources; + + minio::s3::ComposeSource source1; + source1.bucket = "my-src-bucket1"; + source1.object = "my-src-object1"; + sources.push_back(source1); + + minio::s3::ComposeSource source2; + source2.bucket = "my-src-bucket2"; + source2.object = "my-src-object2"; + sources.push_back(source2); + + args.sources = sources; + + // Call compose object. + minio::s3::ComposeObjectResponse resp = client.ComposeObject(args); + + // Handle response. + if (resp) { + std::cout << "my-object is successfully created" << std::endl; + } else { + std::cout << "unable to compose object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/CopyObject.cc b/examples/CopyObject.cc new file mode 100644 index 00000000..e0010922 --- /dev/null +++ b/examples/CopyObject.cc @@ -0,0 +1,52 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create copy object arguments. + minio::s3::CopyObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + + minio::s3::CopySource source; + source.bucket = "my-src-bucket"; + source.object = "my-src-object"; + args.source = source; + + // Call copy object. + minio::s3::CopyObjectResponse resp = client.CopyObject(args); + + // Handle response. + if (resp) { + std::cout << "my-object is successfully created from " + << "my-src-bucket/my-src-object" << std::endl; + } else { + std::cout << "unable to do copy object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/DownloadObject.cc b/examples/DownloadObject.cc new file mode 100644 index 00000000..49a5e751 --- /dev/null +++ b/examples/DownloadObject.cc @@ -0,0 +1,48 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create download object arguments. + minio::s3::DownloadObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + args.filename = "my-object.csv"; + + // Call download object. + minio::s3::DownloadObjectResponse resp = client.DownloadObject(args); + + // Handle response. + if (resp) { + std::cout << "my-object is successfully downloaded to my-object.csv" + << std::endl; + } else { + std::cout << "unable to download object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/GetObject.cc b/examples/GetObject.cc new file mode 100644 index 00000000..6c6fd46e --- /dev/null +++ b/examples/GetObject.cc @@ -0,0 +1,51 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create get object arguments. + minio::s3::GetObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + args.data_callback = [](minio::http::DataCallbackArgs args) -> size_t { + std::cout << std::string(args.buffer, args.length); + return args.size * args.length; + }; + + // Call get object. + minio::s3::GetObjectResponse resp = client.GetObject(args); + + // Handle response. + if (resp) { + std::cout << std::endl + << "data of my-object is received successfully" << std::endl; + } else { + std::cout << "unable to get object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/ListBuckets.cc b/examples/ListBuckets.cc new file mode 100644 index 00000000..1aaa2efe --- /dev/null +++ b/examples/ListBuckets.cc @@ -0,0 +1,44 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Call list buckets. + minio::s3::ListBucketsResponse resp = client.ListBuckets(); + + // Handle response. + if (resp) { + for (auto& bucket : resp.buckets) { + std::cout << "Bucket: " << bucket.name << "; Creation date: " + << bucket.creation_date.ToHttpHeaderValue() << std::endl; + } + } else { + std::cout << "unable to list buckets; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/ListObjects.cc b/examples/ListObjects.cc new file mode 100644 index 00000000..3f5d973d --- /dev/null +++ b/examples/ListObjects.cc @@ -0,0 +1,66 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create list objects arguments. + minio::s3::ListObjectsArgs args; + args.bucket = "my-bucket"; + + // Call list objects. + minio::s3::ListObjectsResult result = client.ListObjects(args); + for (; result; result++) { + minio::s3::Item item = *result; + if (item) { + std::cout << "Name: " << item.name << std::endl; + std::cout << "Version ID: " << item.version_id << std::endl; + std::cout << "ETag: " << item.etag << std::endl; + std::cout << "Size: " << item.size << std::endl; + std::cout << "Last Modified: " << item.last_modified << std::endl; + std::cout << "Delete Marker: " + << minio::utils::BoolToString(item.is_delete_marker) + << std::endl; + std::cout << "User Metadata: " << std::endl; + for (auto& [key, value] : item.user_metadata) { + std::cout << " " << key << ": " << value << std::endl; + } + std::cout << "Owner ID: " << item.owner_id << std::endl; + std::cout << "Owner Name: " << item.owner_name << std::endl; + std::cout << "Storage Class: " << item.storage_class << std::endl; + std::cout << "Is Latest: " << minio::utils::BoolToString(item.is_latest) + << std::endl; + std::cout << "Is Prefix: " << minio::utils::BoolToString(item.is_prefix) + << std::endl; + std::cout << "---" << std::endl; + } else { + std::cout << "unable to listobjects; " << item.GetError() << std::endl; + break; + } + } + + return 0; +} diff --git a/examples/MakeBucket.cc b/examples/MakeBucket.cc new file mode 100644 index 00000000..2823a5f8 --- /dev/null +++ b/examples/MakeBucket.cc @@ -0,0 +1,45 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create make bucket arguments. + minio::s3::MakeBucketArgs args; + args.bucket = "my-bucket"; + + // Call make bucket. + minio::s3::MakeBucketResponse resp = client.MakeBucket(args); + + // Handle response. + if (resp) { + std::cout << "my-bucket is created successfully" << std::endl; + } else { + std::cout << "unable to create bucket; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/PutObject.cc b/examples/PutObject.cc new file mode 100644 index 00000000..de1ef2bb --- /dev/null +++ b/examples/PutObject.cc @@ -0,0 +1,48 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create put object arguments. + std::ifstream file("my-object.csv"); + + minio::s3::PutObjectArgs args(file, 47615315, 0); + args.bucket = "my-bucket"; + args.object = "my-object"; + + // Call put object. + minio::s3::PutObjectResponse resp = client.PutObject(args); + + // Handle response. + if (resp) { + std::cout << "my-object is successfully created" << std::endl; + } else { + std::cout << "unable to do put object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/RemoveBucket.cc b/examples/RemoveBucket.cc new file mode 100644 index 00000000..5b8983a0 --- /dev/null +++ b/examples/RemoveBucket.cc @@ -0,0 +1,45 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create remove bucket arguments. + minio::s3::RemoveBucketArgs args; + args.bucket = "my-bucket"; + + // Call remove bucket. + minio::s3::RemoveBucketResponse resp = client.RemoveBucket(args); + + // Handle response. + if (resp) { + std::cout << "my-bucket is removed successfully" << std::endl; + } else { + std::cout << "unable to remove bucket; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/RemoveObject.cc b/examples/RemoveObject.cc new file mode 100644 index 00000000..c812236e --- /dev/null +++ b/examples/RemoveObject.cc @@ -0,0 +1,46 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create remove object arguments. + minio::s3::RemoveObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + + // Call remove object. + minio::s3::RemoveObjectResponse resp = client.RemoveObject(args); + + // Handle response. + if (resp) { + std::cout << "my-object is removed successfully" << std::endl; + } else { + std::cout << "unable to remove object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/StatObject.cc b/examples/StatObject.cc new file mode 100644 index 00000000..b51d3e45 --- /dev/null +++ b/examples/StatObject.cc @@ -0,0 +1,77 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create stat object arguments. + minio::s3::StatObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + + // Call stat object. + minio::s3::StatObjectResponse resp = client.StatObject(args); + + // Handle response. + if (resp) { + std::cout << "Version ID: " << resp.version_id << std::endl; + std::cout << "ETag: " << resp.etag << std::endl; + std::cout << "Size: " << resp.size << std::endl; + std::cout << "Last Modified: " << resp.last_modified << std::endl; + std::cout << "Retention Mode: "; + if (minio::s3::IsRetentionModeValid(resp.retention_mode)) { + std::cout << minio::s3::RetentionModeToString(resp.retention_mode) + << std::endl; + } else { + std::cout << "-" << std::endl; + } + std::cout << "Retention Retain Until Date: "; + if (resp.retention_retain_until_date) { + std::cout << resp.retention_retain_until_date.ToHttpHeaderValue() + << std::endl; + } else { + std::cout << "-" << std::endl; + } + std::cout << "Legal Hold: "; + if (minio::s3::IsLegalHoldValid(resp.legal_hold)) { + std::cout << minio::s3::LegalHoldToString(resp.legal_hold) << std::endl; + } else { + std::cout << "-" << std::endl; + } + std::cout << "Delete Marker: " + << minio::utils::BoolToString(resp.delete_marker) << std::endl; + std::cout << "User Metadata: " << std::endl; + std::list keys = resp.user_metadata.Keys(); + for (auto& key : keys) { + std::cout << " " << key << ": " << resp.user_metadata.GetFront(key) + << std::endl; + } + } else { + std::cout << "unable to get stat object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/examples/UploadObject.cc b/examples/UploadObject.cc new file mode 100644 index 00000000..31891cb5 --- /dev/null +++ b/examples/UploadObject.cc @@ -0,0 +1,48 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +int main(int argc, char* argv[]) { + // Create S3 base URL. + minio::http::BaseUrl base_url; + base_url.SetHost("play.min.io"); + + // Create credential provider. + minio::creds::StaticProvider provider( + "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); + + // Create S3 client. + minio::s3::Client client(base_url, &provider); + + // Create upload object arguments. + minio::s3::UploadObjectArgs args; + args.bucket = "my-bucket"; + args.object = "my-object"; + args.filename = "my-object.csv"; + + // Call upload object. + minio::s3::UploadObjectResponse resp = client.UploadObject(args); + + // Handle response. + if (resp) { + std::cout << "my-object.csv is successfully uploaded to my-object" + << std::endl; + } else { + std::cout << "unable to upload object; " << resp.GetError() << std::endl; + } + + return 0; +} diff --git a/include/args.h b/include/args.h new file mode 100644 index 00000000..c966f056 --- /dev/null +++ b/include/args.h @@ -0,0 +1,240 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_S3_ARGS_H +#define _MINIO_S3_ARGS_H + +#include + +#include "http.h" +#include "sse.h" +#include "types.h" + +namespace minio { +namespace s3 { +struct BaseArgs { + utils::Multimap extra_headers; + utils::Multimap extra_query_params; +}; // struct BaseArgs + +struct BucketArgs : public BaseArgs { + std::string bucket; + std::string region; + + error::Error Validate(); +}; // struct BucketArgs + +struct ObjectArgs : public BucketArgs { + std::string object; + + error::Error Validate(); +}; // struct ObjectArgs + +struct ObjectWriteArgs : public ObjectArgs { + utils::Multimap headers; + utils::Multimap user_metadata; + Sse *sse = NULL; + std::map tags; + Retention *retention = NULL; + bool legal_hold = false; + + utils::Multimap Headers(); +}; // struct ObjectWriteArgs + +struct ObjectVersionArgs : public ObjectArgs { + std::string version_id; +}; // struct ObjectVersionArgs + +struct ObjectReadArgs : public ObjectVersionArgs { + SseCustomerKey *ssec = NULL; +}; // struct ObjectReadArgs + +struct ObjectConditionalReadArgs : public ObjectReadArgs { + size_t *offset = NULL; + size_t *length = NULL; + std::string match_etag; + std::string not_match_etag; + utils::Time modified_since; + utils::Time unmodified_since; + + utils::Multimap Headers(); + utils::Multimap CopyHeaders(); +}; // struct ObjectConditionalReadArgs + +struct MakeBucketArgs : public BucketArgs { + bool object_lock = false; + + error::Error Validate(); +}; // struct MakeBucketArgs + +using ListBucketsArgs = BaseArgs; + +using BucketExistsArgs = BucketArgs; + +using RemoveBucketArgs = BucketArgs; + +struct AbortMultipartUploadArgs : public ObjectArgs { + std::string upload_id; + + error::Error Validate(); +}; // struct AbortMultipartUploadArgs + +struct CompleteMultipartUploadArgs : public ObjectArgs { + std::string upload_id; + std::list parts; + + error::Error Validate(); +}; // struct CompleteMultipartUploadArgs + +struct CreateMultipartUploadArgs : public ObjectArgs { + utils::Multimap headers; +}; // struct CreateMultipartUploadArgs + +struct PutObjectBaseArgs : public ObjectWriteArgs { + long object_size = -1; + size_t part_size = 0; + long part_count = 0; + std::string content_type; +}; // struct PutObjectBaseArgs + +struct PutObjectApiArgs : public PutObjectBaseArgs { + std::string_view data; + utils::Multimap query_params; +}; // struct PutObjectApiArgs + +struct UploadPartArgs : public ObjectWriteArgs { + std::string upload_id; + unsigned int part_number; + std::string_view data; + + error::Error Validate(); +}; // struct UploadPartArgs + +struct UploadPartCopyArgs : public ObjectWriteArgs { + std::string upload_id; + unsigned int part_number; + utils::Multimap headers; + + error::Error Validate(); +}; // struct UploadPartCopyArgs + +using StatObjectArgs = ObjectConditionalReadArgs; + +using RemoveObjectArgs = ObjectVersionArgs; + +struct DownloadObjectArgs : public ObjectReadArgs { + std::string filename; + bool overwrite; + + error::Error Validate(); +}; // struct DownloadObjectArgs + +struct GetObjectArgs : public ObjectConditionalReadArgs { + http::DataCallback data_callback; + void *user_arg = NULL; + + error::Error Validate(); +}; // struct GetObjectArgs + +struct ListObjectsArgs : public BucketArgs { + std::string delimiter; + bool use_url_encoding_type = true; + std::string marker; // only for ListObjectsV1. + std::string start_after; // only for ListObjectsV2. + std::string key_marker; // only for GetObjectVersions. + unsigned int max_keys = 1000; + std::string prefix; + std::string continuation_token; // only for ListObjectsV2. + bool fetch_owner = false; // only for ListObjectsV2. + std::string version_id_marker; // only for GetObjectVersions. + bool include_user_metadata = false; // MinIO extension for ListObjectsV2. + bool recursive = false; + bool use_api_v1 = false; + bool include_versions = false; +}; // struct ListObjectsArgs + +struct ListObjectsCommonArgs : public BucketArgs { + std::string delimiter; + std::string encoding_type; + unsigned int max_keys = 1000; + std::string prefix; +}; // struct ListObjectsCommonArgs + +struct ListObjectsV1Args : public ListObjectsCommonArgs { + std::string marker; + + ListObjectsV1Args(); + ListObjectsV1Args(ListObjectsArgs args); +}; // struct ListObjectsV1Args + +struct ListObjectsV2Args : public ListObjectsCommonArgs { + std::string start_after; + std::string continuation_token; + bool fetch_owner; + bool include_user_metadata; + + ListObjectsV2Args(); + ListObjectsV2Args(ListObjectsArgs args); +}; // struct ListObjectsV2Args + +struct ListObjectVersionsArgs : public ListObjectsCommonArgs { + std::string key_marker; + std::string version_id_marker; + + ListObjectVersionsArgs(); + ListObjectVersionsArgs(ListObjectsArgs args); +}; // struct ListObjectVersionsArgs + +struct PutObjectArgs : public PutObjectBaseArgs { + std::istream &stream; + + PutObjectArgs(std::istream &stream, long objectsize, long partsize); + error::Error Validate(); +}; // struct PutObjectArgs + +using CopySource = ObjectConditionalReadArgs; + +struct CopyObjectArgs : public ObjectWriteArgs { + CopySource source; + Directive *metadata_directive = NULL; + Directive *tagging_directive = NULL; + + error::Error Validate(); +}; // struct CopyObjectArgs + +struct ComposeSource : public ObjectConditionalReadArgs { + error::Error BuildHeaders(size_t object_size, std::string etag); + size_t ObjectSize(); + utils::Multimap Headers(); + + private: + long object_size_ = -1; + utils::Multimap headers_; +}; // struct ComposeSource + +struct ComposeObjectArgs : public ObjectWriteArgs { + std::list sources; + + error::Error Validate(); +}; // struct ComposeObjectArgs + +struct UploadObjectArgs : public PutObjectBaseArgs { + std::string filename; + + error::Error Validate(); +}; // struct PutObjectArgs +} // namespace s3 +} // namespace minio +#endif // #ifndef __MINIO_S3_ARGS_H diff --git a/include/client.h b/include/client.h new file mode 100644 index 00000000..40bb64b3 --- /dev/null +++ b/include/client.h @@ -0,0 +1,134 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_S3_CLIENT_H +#define _MINIO_S3_CLIENT_H + +#include + +#include "args.h" +#include "request-builder.h" +#include "response.h" + +namespace minio { +namespace s3 { +utils::Multimap GetCommonListObjectsQueryParams(std::string delimiter, + std::string encoding_type, + unsigned int max_keys, + std::string prefix); + +class ListObjectsResult; + +/** + * Simple Storage Service (aka S3) client to perform bucket and object + * operations asynchronously. + */ +class Client { + private: + http::BaseUrl& base_url_; + creds::Provider* provider_ = NULL; + std::map region_map_; + bool debug_ = false; + bool ignore_cert_check_ = false; + + public: + Client(http::BaseUrl& base_url, creds::Provider* provider = NULL); + + void Debug(bool flag) { debug_ = flag; } + + void IgnoreCertCheck(bool flag) { ignore_cert_check_ = flag; } + + void HandleRedirectResponse(std::string& code, std::string& message, + int status_code, http::Method method, + utils::Multimap headers, + std::string_view bucket_name, bool retry = false); + Response GetErrorResponse(http::Response resp, std::string_view resource, + http::Method method, std::string_view bucket_name, + std::string_view object_name); + Response execute(RequestBuilder& builder); + Response Execute(RequestBuilder& builder); + + // S3 APIs + ListObjectsResponse ListObjectsV1(ListObjectsV1Args args); + ListObjectsResponse ListObjectsV2(ListObjectsV2Args args); + ListObjectsResponse ListObjectVersions(ListObjectVersionsArgs args); + + // Bucket operations + GetRegionResponse GetRegion(std::string_view bucket_name, + std::string_view region = ""); + MakeBucketResponse MakeBucket(MakeBucketArgs args); + ListBucketsResponse ListBuckets(ListBucketsArgs args); + ListBucketsResponse ListBuckets(); + BucketExistsResponse BucketExists(BucketExistsArgs args); + RemoveBucketResponse RemoveBucket(RemoveBucketArgs args); + + // Object operations + AbortMultipartUploadResponse AbortMultipartUpload( + AbortMultipartUploadArgs args); + CompleteMultipartUploadResponse CompleteMultipartUpload( + CompleteMultipartUploadArgs args); + CreateMultipartUploadResponse CreateMultipartUpload( + CreateMultipartUploadArgs args); + PutObjectResponse PutObject(PutObjectApiArgs args); + UploadPartResponse UploadPart(UploadPartArgs args); + UploadPartCopyResponse UploadPartCopy(UploadPartCopyArgs args); + StatObjectResponse StatObject(StatObjectArgs args); + RemoveObjectResponse RemoveObject(RemoveObjectArgs args); + DownloadObjectResponse DownloadObject(DownloadObjectArgs args); + GetObjectResponse GetObject(GetObjectArgs args); + ListObjectsResult ListObjects(ListObjectsArgs args); + PutObjectResponse PutObject(PutObjectArgs& args, std::string& upload_id, + char* buf); + PutObjectResponse PutObject(PutObjectArgs args); + CopyObjectResponse CopyObject(CopyObjectArgs args); + StatObjectResponse CalculatePartCount(size_t& part_count, + std::list sources); + ComposeObjectResponse ComposeObject(ComposeObjectArgs args, + std::string& upload_id); + ComposeObjectResponse ComposeObject(ComposeObjectArgs args); + UploadObjectResponse UploadObject(UploadObjectArgs args); +}; // class Client + +class ListObjectsResult { + private: + Client* client_ = NULL; + ListObjectsArgs* args_ = NULL; + bool failed_ = false; + ListObjectsResponse resp_; + std::list::iterator itr_; + + void Populate(); + + public: + ListObjectsResult(error::Error err); + ListObjectsResult(Client* client, ListObjectsArgs* args); + Item& operator*() const { return *itr_; } + operator bool() const { return itr_ != resp_.contents.end(); } + ListObjectsResult& operator++() { + itr_++; + if (!failed_ && itr_ == resp_.contents.end() && resp_.is_truncated) { + Populate(); + } + return *this; + } + ListObjectsResult operator++(int) { + ListObjectsResult curr = *this; + ++(*this); + return curr; + } +}; // class ListObjectsResult +} // namespace s3 +} // namespace minio +#endif // #ifndef __MINIO_S3_CLIENT_H diff --git a/include/creds.h b/include/creds.h new file mode 100644 index 00000000..db323cb2 --- /dev/null +++ b/include/creds.h @@ -0,0 +1,70 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_CREDS_H +#define _MINIO_CREDS_H + +#include + +namespace minio { +namespace creds { +/** + * Credentials contains access key and secret key with optional session token + * and expiration. + */ +class Credentials { + private: + std::string_view access_key_; + std::string_view secret_key_; + std::string_view session_token_; + unsigned int expiration_; + + public: + Credentials(const Credentials& creds); + Credentials(std::string_view access_key, std::string_view secret_key, + std::string_view session_token = "", unsigned int expiration = 0); + std::string AccessKey(); + std::string SecretKey(); + std::string SessionToken(); + bool IsExpired(); +}; // class Credentials + +/** + * Credential provider interface. + */ +class Provider { + public: + Provider() {} + virtual ~Provider() {} + virtual Credentials Fetch() = 0; +}; // class Provider + +/** + * Static credential provider. + */ +class StaticProvider : public Provider { + private: + Credentials* creds_ = NULL; + + public: + StaticProvider(std::string_view access_key, std::string_view secret_key, + std::string_view session_token = ""); + ~StaticProvider(); + Credentials Fetch(); +}; // class StaticProvider +} // namespace creds +} // namespace minio + +#endif // #ifndef _MINIO_CREDS_H diff --git a/include/error.h b/include/error.h new file mode 100644 index 00000000..09e10308 --- /dev/null +++ b/include/error.h @@ -0,0 +1,38 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_ERROR_H +#define _MINIO_ERROR_H + +#include + +namespace minio { +namespace error { +class Error { + private: + std::string msg_; + + public: + Error() {} + Error(std::string_view msg) { msg_ = std::string(msg); } + std::string String() { return msg_; } + operator bool() const { return !msg_.empty(); } +}; // class Error + +const static Error SUCCESS; +} // namespace error +} // namespace minio + +#endif // #ifndef _MINIO_ERROR_H diff --git a/include/http.h b/include/http.h new file mode 100644 index 00000000..a6fe110b --- /dev/null +++ b/include/http.h @@ -0,0 +1,134 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_HTTP_H +#define _MINIO_HTTP_H + +#include +#include + +#include "utils.h" + +namespace minio { +namespace http { +enum class Method { kGet, kHead, kPost, kPut, kDelete }; + +// MethodToString converts http Method enum to string. +constexpr const char* MethodToString(Method& method) throw() { + switch (method) { + case Method::kGet: + return "GET"; + case Method::kHead: + return "HEAD"; + case Method::kPost: + return "POST"; + case Method::kPut: + return "PUT"; + case Method::kDelete: + return "DELETE"; + default: { + std::cerr << "ABORT: Unimplemented HTTP method. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +// ExtractRegion extracts region value from AWS S3 host string. +std::string ExtractRegion(std::string host); + +struct BaseUrl { + std::string host; + bool is_https = true; + unsigned int port = 0; + std::string region; + bool aws_host = false; + bool accelerate_host = false; + bool dualstack_host = false; + bool virtual_style = false; + + error::Error SetHost(std::string hostvalue); + std::string GetHostHeaderValue(); + error::Error BuildUrl(utils::Url& url, Method method, std::string region, + utils::Multimap query_params, + std::string bucket_name = "", + std::string object_name = ""); + operator bool() const { return !host.empty(); } +}; // struct BaseUrl + +struct DataCallbackArgs; + +typedef size_t (*DataCallback)(DataCallbackArgs args); + +struct Response; + +struct DataCallbackArgs { + curlpp::Easy* handle = NULL; + Response* response = NULL; + char* buffer = NULL; + size_t size = 0; + size_t length = 0; + void* user_arg = NULL; +}; // struct DataCallbackArgs + +struct Request { + Method method; + utils::Url url; + utils::Multimap headers; + std::string_view body = ""; + DataCallback data_callback = NULL; + void* user_arg = NULL; + bool debug = false; + bool ignore_cert_check = false; + + Request(Method httpmethod, utils::Url httpurl); + Response Execute(); + operator bool() const { + if (method < Method::kGet || method > Method::kDelete) return false; + return url; + } + + private: + Response execute(); +}; // struct Request + +struct Response { + std::string error; + DataCallback data_callback = NULL; + void* user_arg = NULL; + int status_code = 0; + utils::Multimap headers; + std::string body; + + size_t ResponseCallback(curlpp::Easy* handle, char* buffer, size_t size, + size_t length); + operator bool() const { + return error.empty() && status_code >= 200 && status_code <= 299; + } + + private: + std::string response_; + bool continue100_ = false; + bool status_code_read_ = false; + bool headers_read_ = false; + + size_t ReadStatusCode(char* buffer, size_t size, size_t length); + size_t ReadHeaders(curlpp::Easy* handle, char* buffer, size_t size, + size_t length); +}; // struct Response +} // namespace http +} // namespace minio +#endif // #ifndef _MINIO_HTTP_H diff --git a/include/request-builder.h b/include/request-builder.h new file mode 100644 index 00000000..ed2a39c3 --- /dev/null +++ b/include/request-builder.h @@ -0,0 +1,57 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_REQUEST_BUILDER_H +#define _MINIO_REQUEST_BUILDER_H + +#include "creds.h" +#include "signer.h" + +namespace minio { +namespace s3 { +struct RequestBuilder { + http::Method method; + std::string region; + http::BaseUrl& base_url; + + std::string user_agent; + + utils::Multimap headers; + utils::Multimap query_params; + + std::string bucket_name; + std::string object_name; + + std::string_view body = ""; + + http::DataCallback data_callback = NULL; + void* user_arg = NULL; + + std::string sha256; + utils::Time date; + + bool debug = false; + bool ignore_cert_check = false; + + RequestBuilder(http::Method httpmethod, std::string regionvalue, + http::BaseUrl& baseurl); + http::Request Build(creds::Provider* provider = NULL); + + private: + void BuildHeaders(utils::Url& url, creds::Provider* provider); +}; // struct RequestBuilder +} // namespace s3 +} // namespace minio +#endif // #ifndef __MINIO_REQUEST_BUILDER_H diff --git a/include/response.h b/include/response.h new file mode 100644 index 00000000..8b99ba78 --- /dev/null +++ b/include/response.h @@ -0,0 +1,198 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_S3_RESPONSE_H +#define _MINIO_S3_RESPONSE_H + +#include + +#include "types.h" + +namespace minio { +namespace s3 { +struct Response { + std::string error; + + int status_code = 0; + utils::Multimap headers; + std::string data; + + std::string code; + std::string message; + std::string resource; + std::string request_id; + std::string host_id; + std::string bucket_name; + std::string object_name; + + Response(); + Response(error::Error err); + Response(const Response& response); + operator bool() const { + return error.empty() && code.empty() && message.empty() && + (status_code == 0 || status_code >= 200 && status_code <= 299); + } + std::string GetError(); + static Response ParseXML(std::string_view data, int status_code, + utils::Multimap headers); +}; // struct Response + +struct GetRegionResponse : public Response { + std::string region; + + GetRegionResponse(std::string regionvalue); + GetRegionResponse(error::Error err); + GetRegionResponse(const Response& response); +}; // struct GetRegionResponse + +using MakeBucketResponse = Response; + +struct ListBucketsResponse : public Response { + std::list buckets; + + ListBucketsResponse(std::list bucketlist); + ListBucketsResponse(error::Error err); + ListBucketsResponse(const Response& response); + static ListBucketsResponse ParseXML(std::string_view data); +}; // struct ListBucketsResponse + +struct BucketExistsResponse : public Response { + bool exist = false; + + BucketExistsResponse(bool existflag); + BucketExistsResponse(error::Error err); + BucketExistsResponse(const Response& response); +}; // struct BucketExistsResponse + +using RemoveBucketResponse = Response; + +using AbortMultipartUploadResponse = Response; + +struct CompleteMultipartUploadResponse : public Response { + std::string location; + std::string etag; + std::string version_id; + + CompleteMultipartUploadResponse(); + CompleteMultipartUploadResponse(error::Error err); + CompleteMultipartUploadResponse(const Response& response); + static CompleteMultipartUploadResponse ParseXML(std::string_view data, + std::string version_id); +}; // struct CompleteMultipartUploadResponse + +struct CreateMultipartUploadResponse : public Response { + std::string upload_id; + + CreateMultipartUploadResponse(std::string uploadid); + CreateMultipartUploadResponse(error::Error err); + CreateMultipartUploadResponse(const Response& response); +}; // struct CreateMultipartUploadResponse + +struct PutObjectResponse : public Response { + std::string etag; + std::string version_id; + + PutObjectResponse(); + PutObjectResponse(error::Error err); + PutObjectResponse(const Response& response); +}; // struct PutObjectResponse + +using UploadPartResponse = PutObjectResponse; + +using UploadPartCopyResponse = PutObjectResponse; + +struct StatObjectResponse : public Response { + std::string version_id; + std::string etag; + size_t size = 0; + utils::Time last_modified; + RetentionMode retention_mode; + utils::Time retention_retain_until_date; + LegalHold legal_hold; + bool delete_marker; + utils::Multimap user_metadata; + + StatObjectResponse(); + StatObjectResponse(error::Error err); + StatObjectResponse(const Response& response); +}; // struct StatObjectResponse + +using RemoveObjectResponse = Response; + +using DownloadObjectResponse = Response; + +using GetObjectResponse = Response; + +struct Item : public Response { + std::string etag; // except DeleteMarker + std::string name; + utils::Time last_modified; + std::string owner_id; + std::string owner_name; + size_t size = 0; // except DeleteMarker + std::string storage_class; + bool is_latest = false; // except ListObjects V1/V2 + std::string version_id; // except ListObjects V1/V2 + std::map user_metadata; + bool is_prefix = false; + bool is_delete_marker = false; + std::string encoding_type; + + Item(); + Item(error::Error err); + Item(const Response& response); +}; // struct Item + +struct ListObjectsResponse : public Response { + // Common + std::string name; + std::string encoding_type; + std::string prefix; + std::string delimiter; + bool is_truncated; + unsigned int max_keys; + std::list contents; + + // ListObjectsV1 + std::string marker; + std::string next_marker; + + // ListObjectsV2 + unsigned int key_count; + std::string start_after; + std::string continuation_token; + std::string next_continuation_token; + + // ListObjectVersions + std::string key_marker; + std::string next_key_marker; + std::string version_id_marker; + std::string next_version_id_marker; + + ListObjectsResponse(); + ListObjectsResponse(error::Error err); + ListObjectsResponse(const Response& response); + static ListObjectsResponse ParseXML(std::string_view data, bool version); +}; // struct ListObjectsResponse + +using CopyObjectResponse = PutObjectResponse; + +using ComposeObjectResponse = PutObjectResponse; + +using UploadObjectResponse = PutObjectResponse; +} // namespace s3 +} // namespace minio + +#endif // #ifndef _MINIO_S3_RESPONSE_H diff --git a/include/signer.h b/include/signer.h new file mode 100644 index 00000000..bfa9f04d --- /dev/null +++ b/include/signer.h @@ -0,0 +1,65 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_SIGNER_H +#define _MINIO_SIGNER_H + +#include + +#include "http.h" + +namespace minio { +namespace signer { +std::string GetScope(utils::Time& time, std::string_view region, + std::string_view service_name); +std::string GetCanonicalRequestHash(std::string_view method, + std::string_view uri, + std::string_view query_string, + std::string_view headers, + std::string_view signed_headers, + std::string_view content_sha256); +std::string GetStringToSign(utils::Time& date, std::string_view scope, + std::string_view canonical_request_hash); +std::string HmacHash(std::string_view key, std::string_view data); +std::string GetSigningKey(std::string_view secret_key, utils::Time& date, + std::string_view region, + std::string_view service_name); +std::string GetSignature(std::string_view signing_key, + std::string_view string_to_sign); +std::string GetAuthorization(std::string_view access_key, + std::string_view scope, + std::string_view signed_headers, + std::string_view signature); +utils::Multimap& SignV4(std::string_view service_name, http::Method& method, + std::string_view uri, std::string_view region, + utils::Multimap& headers, utils::Multimap& query_params, + std::string_view access_key, + std::string_view secret_key, + std::string_view content_sha256, utils::Time& date); +utils::Multimap& SignV4S3(http::Method& method, std::string_view uri, + std::string_view region, utils::Multimap& headers, + utils::Multimap& query_params, + std::string_view access_key, + std::string_view secret_key, + std::string_view content_sha256, utils::Time& date); +utils::Multimap& SignV4STS(http::Method& method, std::string_view uri, + std::string_view region, utils::Multimap& headers, + utils::Multimap& query_params, + std::string_view access_key, + std::string_view secret_key, + std::string_view content_sha256, utils::Time& date); +} // namespace signer +} // namespace minio +#endif // #ifndef __MINIO_SIGNER_H diff --git a/include/sse.h b/include/sse.h new file mode 100644 index 00000000..6f386b24 --- /dev/null +++ b/include/sse.h @@ -0,0 +1,67 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_S3_SSE_H +#define _MINIO_S3_SSE_H + +#include "utils.h" + +namespace minio { +namespace s3 { +class Sse { + public: + utils::Multimap empty_; + + public: + Sse() {} + virtual ~Sse() {} + bool TlsRequired() { return true; } + utils::Multimap CopyHeaders() { return empty_; } + virtual utils::Multimap Headers() = 0; +}; // class Sse + +class SseCustomerKey : public Sse { + private: + utils::Multimap headers; + utils::Multimap copy_headers; + + public: + SseCustomerKey(std::string_view key); + utils::Multimap Headers() { return headers; } + utils::Multimap CopyHeaders() { return copy_headers; } +}; // class SseCustomerKey + +class SseKms : public Sse { + private: + utils::Multimap headers; + + public: + SseKms(std::string_view key, std::string_view context); + utils::Multimap Headers() { return headers; } +}; // class SseKms + +class SseS3 : public Sse { + private: + utils::Multimap headers; + + public: + SseS3(); + utils::Multimap Headers() { return headers; } + bool TlsRequired() { return false; } +}; // class SseS3 +} // namespace s3 +} // namespace minio + +#endif // #ifndef __MINIO_S3_SSE_H diff --git a/include/types.h b/include/types.h new file mode 100644 index 00000000..9c394b82 --- /dev/null +++ b/include/types.h @@ -0,0 +1,124 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_S3_TYPES_H +#define _MINIO_S3_TYPES_H + +#include + +#include "utils.h" + +namespace minio { +namespace s3 { +enum class RetentionMode { kGovernance, kCompliance }; + +// StringToRetentionMode converts string to retention mode enum. +RetentionMode StringToRetentionMode(std::string_view str) throw(); + +constexpr bool IsRetentionModeValid(RetentionMode& retention) { + switch (retention) { + case RetentionMode::kGovernance: + case RetentionMode::kCompliance: + return true; + } + return false; +} + +// RetentionModeToString converts retention mode enum to string. +constexpr const char* RetentionModeToString(RetentionMode& retention) throw() { + switch (retention) { + case RetentionMode::kGovernance: + return "GOVERNANCE"; + case RetentionMode::kCompliance: + return "COMPLIANCE"; + default: { + std::cerr << "ABORT: Unknown retention mode. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +enum class LegalHold { kOn, kOff }; + +// StringToLegalHold converts string to legal hold enum. +LegalHold StringToLegalHold(std::string_view str) throw(); + +constexpr bool IsLegalHoldValid(LegalHold& legal_hold) { + switch (legal_hold) { + case LegalHold::kOn: + case LegalHold::kOff: + return true; + } + return false; +} + +// LegalHoldToString converts legal hold enum to string. +constexpr const char* LegalHoldToString(LegalHold& legal_hold) throw() { + switch (legal_hold) { + case LegalHold::kOn: + return "ON"; + case LegalHold::kOff: + return "OFF"; + default: { + std::cerr << "ABORT: Unknown legal hold. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +enum class Directive { kCopy, kReplace }; + +// StringToDirective converts string to directive enum. +Directive StringToDirective(std::string_view str) throw(); + +// DirectiveToString converts directive enum to string. +constexpr const char* DirectiveToString(Directive& directive) throw() { + switch (directive) { + case Directive::kCopy: + return "COPY"; + case Directive::kReplace: + return "REPLACE"; + default: { + std::cerr << "ABORT: Unknown directive. This should not happen." + << std::endl; + std::terminate(); + } + } + return NULL; +} + +struct Bucket { + std::string name; + utils::Time creation_date; +}; // struct Bucket + +struct Part { + unsigned int number; + std::string etag; + utils::Time last_modified; + size_t size; +}; // struct Part + +struct Retention { + RetentionMode mode; + utils::Time retain_until_date; +}; // struct Retention +} // namespace s3 +} // namespace minio +#endif // #ifndef __MINIO_S3_TYPES_H diff --git a/include/utils.h b/include/utils.h new file mode 100644 index 00000000..dd5b0a09 --- /dev/null +++ b/include/utils.h @@ -0,0 +1,178 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef _MINIO_UTILS_H +#define _MINIO_UTILS_H + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "error.h" + +namespace minio { +namespace utils { +inline constexpr unsigned int kMaxMultipartCount = 10000; // 10000 parts +inline constexpr unsigned long kMaxObjectSize = + 5L * 1024 * 1024 * 1024 * 1024; // 5TiB +inline constexpr unsigned long kMaxPartSize = 5L * 1024 * 1024 * 1024; // 5GiB +inline constexpr unsigned int kMinPartSize = 5 * 1024 * 1024; // 5MiB + +// FormatTime formats time as per format. +std::string FormatTime(const std::tm* time, const char* format); + +// StringToBool converts string to bool. +bool StringToBool(std::string_view str); + +// BoolToString converts bool to string. +inline const char* const BoolToString(bool b) { return b ? "true" : "false"; } + +// Trim trims leading and trailing character of a string. +std::string Trim(std::string_view str, char ch = ' '); + +// CheckNonemptystring checks whether string is not empty after trimming +// whitespaces. +bool CheckNonEmptyString(std::string_view str); + +// ToLower converts string to lower case. +std::string ToLower(std::string str); + +// StartsWith returns whether str starts with prefix or not. +bool StartsWith(std::string_view str, std::string_view prefix); + +// EndsWith returns whether str ends with suffix or not. +bool EndsWith(std::string_view str, std::string_view suffix); + +// Contains returns whether str has ch. +bool Contains(std::string_view str, char ch); + +// Contains returns whether str has substr. +bool Contains(std::string_view str, std::string_view substr); + +// Join returns a string of joined values by delimiter. +std::string Join(std::list values, std::string delimiter); + +// Join returns a string of joined values by delimiter. +std::string Join(std::vector values, std::string delimiter); + +// EncodePath does URL encoding of path. It also normalizes multiple slashes. +std::string EncodePath(std::string_view path); + +// Sha256hash computes SHA-256 of data and return hash as hex encoded value. +std::string Sha256Hash(std::string_view str); + +// Base64Encode encodes string to base64. +std::string Base64Encode(std::string_view str); + +// Md5sumHash computes MD5 of data and return hash as Base64 encoded value. +std::string Md5sumHash(std::string_view str); + +error::Error CheckBucketName(std::string_view bucket_name, bool strict = false); +error::Error ReadPart(std::istream& stream, char* buf, size_t size, + size_t& bytes_read); +error::Error CalcPartInfo(long object_size, size_t& part_size, + long& part_count); + +/** + * Time represents date and time with timezone. + */ +class Time { + private: + struct timeval tv_; + bool utc_; + + public: + Time(); + Time(std::time_t tv_sec, suseconds_t tv_usec, bool utc); + std::tm* ToUTC(); + std::string ToSignerDate(); + std::string ToAmzDate(); + std::string ToHttpHeaderValue(); + static Time FromHttpHeaderValue(const char* value); + std::string ToISO8601UTC(); + static Time FromISO8601UTC(const char* value); + static Time Now(); + operator bool() const { return tv_.tv_sec != 0 && tv_.tv_usec != 0; } +}; // class Time + +/** + * Multimap represents dictionary of keys and their multiple values. + */ +class Multimap { + private: + std::map> map_; + std::map> keys_; + + public: + Multimap(); + Multimap(const Multimap& headers); + void Add(std::string key, std::string value); + void AddAll(const Multimap& headers); + std::list ToHttpHeaders(); + std::string ToQueryString(); + operator bool() const { return !map_.empty(); } + bool Contains(std::string_view key); + std::list Get(std::string_view key); + std::string GetFront(std::string_view key); + std::list Keys(); + void GetCanonicalHeaders(std::string& signed_headers, + std::string& canonical_headers); + std::string GetCanonicalQueryString(); +}; // class Multimap + +/** + * Url represents HTTP URL and it's components. + */ +struct Url { + bool is_https; + std::string host; + std::string path; + std::string query_string; + + operator bool() const { return !host.empty(); } + std::string String(); +}; // struct Url + +/** + * CharBuffer represents stream buffer wrapping character array and its size. + */ +struct CharBuffer : std::streambuf { + CharBuffer(char* buf, size_t size) { this->setg(buf, buf, buf + size); } + + pos_type seekoff(off_type off, std::ios_base::seekdir dir, + std::ios_base::openmode which = std::ios_base::in) override { + if (dir == std::ios_base::cur) + gbump(off); + else if (dir == std::ios_base::end) + setg(eback(), egptr() + off, egptr()); + else if (dir == std::ios_base::beg) + setg(eback(), eback() + off, egptr()); + return gptr() - eback(); + } + + pos_type seekpos(pos_type sp, std::ios_base::openmode which) override { + return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which); + } +}; // struct CharBuffer +} // namespace utils +} // namespace minio + +#endif // #ifndef _MINIO_UTILS_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b9b86465..640f276b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SET(SRCS s3.cpp s3_http.cpp s3_io.cpp s3_signature_v2.cpp) +SET(SRCS args.cc client.cc creds.cc http.cc response.cc signer.cc sse.cc types.cc utils.cc request-builder.cc) SET(S3CLIENT_INSTALL_LIST) add_library(miniocpp STATIC ${SRCS}) diff --git a/src/args.cc b/src/args.cc new file mode 100644 index 00000000..801c9d11 --- /dev/null +++ b/src/args.cc @@ -0,0 +1,363 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "args.h" + +minio::error::Error minio::s3::BucketArgs::Validate() { + return utils::CheckBucketName(bucket); +} + +minio::error::Error minio::s3::ObjectArgs::Validate() { + if (error::Error err = BucketArgs::Validate()) return err; + if (!utils::CheckNonEmptyString(object)) { + return error::Error("object name cannot be empty"); + } + + return error::SUCCESS; +} + +minio::utils::Multimap minio::s3::ObjectWriteArgs::Headers() { + utils::Multimap headers; + headers.AddAll(extra_headers); + headers.AddAll(headers); + headers.AddAll(user_metadata); + + if (sse != NULL) headers.AddAll(sse->Headers()); + + std::string tagging; + for (auto& [key, value] : tags) { + std::string tag = curlpp::escape(key) + "=" + curlpp::escape(value); + if (!tagging.empty()) tagging += "&"; + tagging += tag; + } + if (!tagging.empty()) headers.Add("x-amz-tagging", tagging); + + if (retention != NULL) { + headers.Add("x-amz-object-lock-mode", + RetentionModeToString(retention->mode)); + headers.Add("x-amz-object-lock-retain-until-date", + retention->retain_until_date.ToISO8601UTC()); + } + + if (legal_hold) headers.Add("x-amz-object-lock-legal-hold", "ON"); + + return headers; +} + +minio::utils::Multimap minio::s3::ObjectConditionalReadArgs::Headers() { + size_t* off = offset; + size_t* len = length; + + size_t zero = 0; + if (len != NULL && off == NULL) { + off = &zero; + } + + std::string range; + if (off != NULL) { + range = std::string("bytes=") + std::to_string(*off) + "-"; + if (len != NULL) { + range += std::to_string(*off + *len - 1); + } + } + + utils::Multimap headers; + if (!range.empty()) headers.Add("Range", range); + if (!match_etag.empty()) headers.Add("if-match", match_etag); + if (!not_match_etag.empty()) headers.Add("if-none-match", not_match_etag); + if (modified_since) { + headers.Add("if-modified-since", modified_since.ToHttpHeaderValue()); + } + if (unmodified_since) { + headers.Add("if-unmodified-since", unmodified_since.ToHttpHeaderValue()); + } + if (ssec != NULL) headers.AddAll(ssec->Headers()); + + return headers; +} + +minio::utils::Multimap minio::s3::ObjectConditionalReadArgs::CopyHeaders() { + utils::Multimap headers; + + std::string copy_source = curlpp::escape("/" + bucket + "/" + object); + if (!version_id.empty()) { + copy_source += "?versionId=" + curlpp::escape(version_id); + } + + headers.Add("x-amz-copy-source", copy_source); + + if (ssec != NULL) headers.AddAll(ssec->CopyHeaders()); + if (!match_etag.empty()) { + headers.Add("x-amz-copy-source-if-match", match_etag); + } + if (!not_match_etag.empty()) { + headers.Add("x-amz-copy-source-if-none-match", not_match_etag); + } + if (modified_since) { + headers.Add("x-amz-copy-source-if-modified-since", + modified_since.ToHttpHeaderValue()); + } + if (unmodified_since) { + headers.Add("x-amz-copy-source-if-unmodified-since", + unmodified_since.ToHttpHeaderValue()); + } + + return headers; +} + +minio::error::Error minio::s3::MakeBucketArgs::Validate() { + return utils::CheckBucketName(bucket, true); +} + +minio::error::Error minio::s3::AbortMultipartUploadArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + if (!utils::CheckNonEmptyString(upload_id)) { + return error::Error("upload ID cannot be empty"); + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::CompleteMultipartUploadArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + if (!utils::CheckNonEmptyString(upload_id)) { + return error::Error("upload ID cannot be empty"); + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::UploadPartArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + if (!utils::CheckNonEmptyString(upload_id)) { + return error::Error("upload ID cannot be empty"); + } + if (part_number < 1 || part_number > 10000) { + return error::Error("part number must be between 1 and 10000"); + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::UploadPartCopyArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + if (!utils::CheckNonEmptyString(upload_id)) { + return error::Error("upload ID cannot be empty"); + } + if (part_number < 1 || part_number > 10000) { + return error::Error("part number must be between 1 and 10000"); + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::DownloadObjectArgs::Validate() { + if (error::Error err = ObjectReadArgs::Validate()) return err; + if (!utils::CheckNonEmptyString(filename)) { + return error::Error("filename cannot be empty"); + } + + if (!overwrite && std::filesystem::exists(filename)) { + return error::Error("file " + filename + " already exists"); + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::GetObjectArgs::Validate() { + if (error::Error err = ObjectConditionalReadArgs::Validate()) return err; + if (data_callback == NULL) { + return error::Error("data callback must be set"); + } + + return error::SUCCESS; +} + +minio::s3::ListObjectsV1Args::ListObjectsV1Args() {} + +minio::s3::ListObjectsV1Args::ListObjectsV1Args(ListObjectsArgs args) { + extra_headers = args.extra_headers; + extra_query_params = args.extra_query_params; + bucket = args.bucket; + region = args.region; + + delimiter = args.delimiter; + encoding_type = args.use_url_encoding_type ? "url" : ""; + max_keys = args.max_keys; + prefix = args.prefix; + + marker = args.marker; +} + +minio::s3::ListObjectsV2Args::ListObjectsV2Args() {} + +minio::s3::ListObjectsV2Args::ListObjectsV2Args(ListObjectsArgs args) { + extra_headers = args.extra_headers; + extra_query_params = args.extra_query_params; + bucket = args.bucket; + region = args.region; + + delimiter = args.delimiter; + encoding_type = args.use_url_encoding_type ? "url" : ""; + max_keys = args.max_keys; + prefix = args.prefix; + + start_after = args.start_after; + continuation_token = args.continuation_token; + fetch_owner = args.fetch_owner; + include_user_metadata = args.include_user_metadata; +} + +minio::s3::ListObjectVersionsArgs::ListObjectVersionsArgs() {} + +minio::s3::ListObjectVersionsArgs::ListObjectVersionsArgs( + ListObjectsArgs args) { + extra_headers = args.extra_headers; + extra_query_params = args.extra_query_params; + bucket = args.bucket; + region = args.region; + + delimiter = args.delimiter; + encoding_type = args.use_url_encoding_type ? "url" : ""; + max_keys = args.max_keys; + prefix = args.prefix; + + key_marker = args.key_marker; + version_id_marker = args.version_id_marker; +} + +minio::s3::PutObjectArgs::PutObjectArgs(std::istream& istream, long objectsize, + long partsize) + : stream(istream) { + object_size = objectsize; + part_size = partsize; +} + +minio::error::Error minio::s3::PutObjectArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + return utils::CalcPartInfo(object_size, part_size, part_count); +} + +minio::error::Error minio::s3::CopyObjectArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + if (error::Error err = source.Validate()) return err; + + if (source.offset != NULL || source.length != NULL) { + if (metadata_directive != NULL && *metadata_directive == Directive::kCopy) { + return error::Error( + "COPY metadata directive is not applicable to source object with " + "range"); + } + + if (tagging_directive != NULL && *tagging_directive == Directive::kCopy) { + return error::Error( + "COPY tagging directive is not applicable to source object with " + "range"); + } + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::ComposeSource::BuildHeaders(size_t object_size, + std::string etag) { + std::string msg = "source " + bucket + "/" + object; + if (!version_id.empty()) msg += "?versionId=" + version_id; + msg += ": "; + + if (offset != NULL && *offset >= object_size) { + return error::Error(msg + "offset " + std::to_string(*offset) + + " is beyond object size " + + std::to_string(object_size)); + } + + if (length != NULL) { + if (*length > object_size) { + return error::Error(msg + "length " + std::to_string(*length) + + " is beyond object size " + + std::to_string(object_size)); + } + + size_t off = 0; + if (offset != NULL) off = *offset; + if ((off + *length) > object_size) { + return error::Error( + msg + "compose size " + std::to_string(off + *length) + + " is beyond object size " + std::to_string(object_size)); + } + } + + object_size_ = object_size; + headers_ = CopyHeaders(); + if (!headers_.Contains("x-amz-copy-source-if-match")) { + headers_.Add("x-amz-copy-source-if-match", etag); + } + + return error::SUCCESS; +} + +size_t minio::s3::ComposeSource::ObjectSize() { + if (object_size_ == -1) { + std::cerr << "ABORT: ComposeSource::BuildHeaders() must be called prior to " + "this method invocation. This shoud not happen." + << std::endl; + std::terminate(); + } + + return object_size_; +} + +minio::utils::Multimap minio::s3::ComposeSource::Headers() { + if (!headers_) { + std::cerr << "ABORT: ComposeSource::BuildHeaders() must be called prior to " + "this method invocation. This shoud not happen." + << std::endl; + std::terminate(); + } + + return headers_; +} + +minio::error::Error minio::s3::ComposeObjectArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + if (sources.empty()) return error::Error("compose sources cannot be empty"); + + int i = 1; + for (auto& source : sources) { + if (error::Error err = source.Validate()) { + return error::Error("source " + std::to_string(i) + ": " + err.String()); + } + i++; + } + + return error::SUCCESS; +} + +minio::error::Error minio::s3::UploadObjectArgs::Validate() { + if (error::Error err = ObjectArgs::Validate()) return err; + + if (!utils::CheckNonEmptyString(filename)) { + return error::Error("filename cannot be empty"); + } + + if (!std::filesystem::exists(filename)) { + return error::Error("file " + filename + " does not exist"); + } + + std::filesystem::path file_path = filename; + size_t obj_size = std::filesystem::file_size(file_path); + object_size = obj_size; + return utils::CalcPartInfo(object_size, part_size, part_count); +} diff --git a/src/client.cc b/src/client.cc new file mode 100644 index 00000000..4d8756e4 --- /dev/null +++ b/src/client.cc @@ -0,0 +1,1402 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +minio::utils::Multimap minio::s3::GetCommonListObjectsQueryParams( + std::string delimiter, std::string encoding_type, unsigned int max_keys, + std::string prefix) { + utils::Multimap query_params; + query_params.Add("delimiter", delimiter); + query_params.Add("max-keys", std::to_string(max_keys > 0 ? max_keys : 1000)); + query_params.Add("prefix", prefix); + if (!encoding_type.empty()) query_params.Add("encoding-type", encoding_type); + return query_params; +} + +minio::s3::Client::Client(http::BaseUrl& base_url, creds::Provider* provider) + : base_url_(base_url) { + if (!base_url_) { + std::cerr << "base URL must not be empty" << std::endl; + std::terminate(); + } + + provider_ = provider; +} + +void minio::s3::Client::HandleRedirectResponse( + std::string& code, std::string& message, int status_code, + http::Method method, utils::Multimap headers, std::string_view bucket_name, + bool retry) { + switch (status_code) { + case 301: + code = "PermanentRedirect"; + message = "Moved Permanently"; + break; + case 307: + code = "Redirect"; + message = "Temporary redirect"; + break; + case 400: + code = "BadRequest"; + message = "Bad request"; + break; + default: + code = ""; + message = ""; + break; + } + + std::string region = headers.GetFront("x-amz-bucket-region"); + + if (!message.empty() && !region.empty()) { + message += "; use region " + region; + } + + if (retry && !region.empty() && method == http::Method::kHead && + !bucket_name.empty() && !region_map_[std::string(bucket_name)].empty()) { + code = "RetryHead"; + message = ""; + } +} + +minio::s3::Response minio::s3::Client::GetErrorResponse( + http::Response resp, std::string_view resource, http::Method method, + std::string_view bucket_name, std::string_view object_name) { + if (!resp.error.empty()) return error::Error(resp.error); + + Response response; + response.status_code = resp.status_code; + response.headers = resp.headers; + std::string data = resp.body; + if (!data.empty()) { + std::list values = resp.headers.Get("Content-Type"); + for (auto& value : values) { + if (utils::Contains(utils::ToLower(value), "application/xml")) { + return Response::ParseXML(resp.body, resp.status_code, resp.headers); + } + } + + response.error = "invalid response received; status code: " + + std::to_string(resp.status_code) + + "; content-type: " + utils::Join(values, ","); + return response; + } + + switch (resp.status_code) { + case 301: + case 307: + case 400: + HandleRedirectResponse(response.code, response.message, resp.status_code, + method, resp.headers, bucket_name, true); + break; + case 403: + response.code = "AccessDenied"; + response.message = "Access denied"; + break; + case 404: + if (!object_name.empty()) { + response.code = "NoSuchKey"; + response.message = "Object does not exist"; + } else if (bucket_name.empty()) { + response.code = "NoSuchBucket"; + response.message = "Bucket does not exist"; + } else { + response.code = "ResourceNotFound"; + response.message = "Request resource not found"; + } + break; + case 405: + response.code = "MethodNotAllowed"; + response.message = + "The specified method is not allowed against this resource"; + break; + case 409: + if (bucket_name.empty()) { + response.code = "NoSuchBucket"; + response.message = "Bucket does not exist"; + } else { + response.code = "ResourceConflict"; + response.message = "Request resource conflicts"; + } + break; + case 501: + response.code = "MethodNotAllowed"; + response.message = + "The specified method is not allowed against this resource"; + break; + default: + response.error = "server failed with HTTP status code " + + std::to_string(resp.status_code); + return response; + } + + response.resource = resource; + response.request_id = response.headers.GetFront("x-amz-request-id"); + response.host_id = response.headers.GetFront("x-amz-id-2"); + response.bucket_name = bucket_name; + response.object_name = object_name; + + return response; +} + +minio::s3::Response minio::s3::Client::execute(RequestBuilder& builder) { + http::Request request = builder.Build(provider_); + http::Response resp = request.Execute(); + if (resp) { + Response response; + response.status_code = resp.status_code; + response.headers = resp.headers; + response.data = resp.body; + return response; + } + + Response response = + GetErrorResponse(resp, request.url.path, request.method, + builder.bucket_name, builder.object_name); + if (response.code == "NoSuchBucket" || response.code == "RetryHead") { + region_map_.erase(builder.bucket_name); + } + + return response; +} + +minio::s3::Response minio::s3::Client::Execute(RequestBuilder& builder) { + Response resp = execute(builder); + if (resp || resp.code != "RetryHead") return resp; + + // Retry only once on RetryHead error. + resp = execute(builder); + if (resp || resp.code != "RetryHead") return resp; + + std::string code; + std::string message; + HandleRedirectResponse(code, message, resp.status_code, builder.method, + resp.headers, builder.bucket_name); + resp.code = code; + resp.message = message; + + return resp; +} + +minio::s3::GetRegionResponse minio::s3::Client::GetRegion( + std::string_view bucket_name, std::string_view region) { + std::string base_region = base_url_.region; + if (!region.empty()) { + if (!base_region.empty() && base_region != region) { + return error::Error("region must be " + base_region + ", but passed " + + std::string(region)); + } + + return std::string(region); + } + + if (!base_region.empty()) return base_region; + + if (bucket_name.empty() || provider_ == NULL) return std::string("us-east-1"); + + std::string stored_region = region_map_[std::string(bucket_name)]; + if (!stored_region.empty()) return stored_region; + + RequestBuilder builder(http::Method::kGet, "us-east-1", base_url_); + utils::Multimap query_params; + query_params.Add("location", ""); + builder.query_params = query_params; + builder.bucket_name = bucket_name; + + Response response = Execute(builder); + if (!response) return response; + + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(response.data.data()); + if (!result) return error::Error("unable to parse XML"); + auto text = xdoc.select_node("/LocationConstraint/text()"); + std::string value = text.node().value(); + + if (value.empty()) { + value = "us-east-1"; + } else if (value == "EU") { + if (base_url_.aws_host) value = "eu-west-1"; + } + + region_map_[std::string(bucket_name)] = value; + + return value; +} + +minio::s3::MakeBucketResponse minio::s3::Client::MakeBucket( + MakeBucketArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region = args.region; + std::string base_region = base_url_.region; + if (!base_region.empty() && !region.empty() && base_region != region) { + return error::Error("region must be " + base_region + ", but passed " + + region); + } + + if (region.empty()) region = base_region; + if (region.empty()) region = "us-east-1"; + + RequestBuilder builder(http::Method::kPut, region, base_url_); + builder.bucket_name = args.bucket; + + utils::Multimap headers; + if (args.object_lock) headers.Add("x-amz-bucket-object-lock-enabled", "true"); + builder.headers = headers; + + std::string body; + if (region != "us-east-1") { + std::stringstream ss; + ss << "" + << "" << region << "" + << ""; + body = ss.str(); + builder.body = body; + } + + Response response = Execute(builder); + if (response) region_map_[std::string(args.bucket)] = region; + + return response; +} + +minio::s3::ListBucketsResponse minio::s3::Client::ListBuckets( + ListBucketsArgs args) { + RequestBuilder builder(http::Method::kGet, base_url_.region, base_url_); + builder.headers = args.extra_headers; + builder.query_params = args.extra_query_params; + Response resp = Execute(builder); + if (!resp) return resp; + return ListBucketsResponse::ParseXML(resp.data); +} + +minio::s3::ListBucketsResponse minio::s3::Client::ListBuckets() { + return ListBuckets(ListBucketsArgs()); +} + +minio::s3::BucketExistsResponse minio::s3::Client::BucketExists( + BucketExistsArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return (resp.code == "NoSuchBucket") ? false : resp; + } + + RequestBuilder builder(http::Method::kHead, region, base_url_); + builder.bucket_name = args.bucket; + if (Response resp = Execute(builder)) { + return true; + } else { + return (resp.code == "NoSuchBucket") ? false : resp; + } +} + +minio::s3::RemoveBucketResponse minio::s3::Client::RemoveBucket( + RemoveBucketArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kDelete, region, base_url_); + builder.bucket_name = args.bucket; + + return Execute(builder); +} + +minio::s3::AbortMultipartUploadResponse minio::s3::Client::AbortMultipartUpload( + AbortMultipartUploadArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kDelete, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + query_params.Add("uploadId", args.upload_id); + builder.query_params = query_params; + + return Execute(builder); +} + +minio::s3::CompleteMultipartUploadResponse +minio::s3::Client::CompleteMultipartUpload(CompleteMultipartUploadArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kPost, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + query_params.Add("uploadId", args.upload_id); + builder.query_params = query_params; + + std::stringstream ss; + ss << ""; + for (auto& part : args.parts) { + ss << "" + << "" << part.number << "" + << "" + << "\"" << part.etag << "\"" + << "" + << ""; + } + ss << ""; + std::string body = ss.str(); + builder.body = body; + + utils::Multimap headers; + headers.Add("Content-Type", "application/xml"); + headers.Add("Content-MD5", utils::Md5sumHash(body)); + builder.headers = headers; + + Response response = Execute(builder); + if (!response) return response; + return CompleteMultipartUploadResponse::ParseXML( + response.data, response.headers.GetFront("x-amz-version-id")); +} + +minio::s3::CreateMultipartUploadResponse +minio::s3::Client::CreateMultipartUpload(CreateMultipartUploadArgs args) { + if (error::Error err = args.Validate()) return err; + + if (!args.headers.Contains("Content-Type")) { + args.headers.Add("Content-Type", "application/octet-stream"); + } + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kPost, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + query_params.Add("uploads", ""); + builder.query_params = query_params; + + builder.headers = args.headers; + + if (Response resp = Execute(builder)) { + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(resp.data.data()); + if (!result) return error::Error("unable to parse XML"); + auto text = + xdoc.select_node("/InitiateMultipartUploadResult/UploadId/text()"); + return std::string(text.node().value()); + } else { + return resp; + } +} + +minio::s3::PutObjectResponse minio::s3::Client::PutObject( + PutObjectApiArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kPut, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + builder.query_params = args.query_params; + builder.headers = args.headers; + builder.body = args.data; + + Response response = Execute(builder); + if (!response) return response; + + PutObjectResponse resp; + resp.etag = utils::Trim(response.headers.GetFront("etag"), '"'); + resp.version_id = response.headers.GetFront("x-amz-version-id"); + + return resp; +} + +minio::s3::UploadPartResponse minio::s3::Client::UploadPart( + UploadPartArgs args) { + if (error::Error err = args.Validate()) return err; + + utils::Multimap query_params; + query_params.Add("partNumber", std::to_string(args.part_number)); + query_params.Add("uploadId", args.upload_id); + + PutObjectApiArgs api_args; + api_args.extra_headers = args.extra_headers; + api_args.extra_query_params = args.extra_query_params; + api_args.bucket = args.bucket; + api_args.region = args.region; + api_args.object = args.object; + api_args.data = args.data; + api_args.query_params = query_params; + + return PutObject(api_args); +} + +minio::s3::UploadPartCopyResponse minio::s3::Client::UploadPartCopy( + UploadPartCopyArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kPut, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + + utils::Multimap query_params; + query_params.AddAll(args.extra_query_params); + query_params.Add("partNumber", std::to_string(args.part_number)); + query_params.Add("uploadId", args.upload_id); + builder.query_params = query_params; + + builder.headers = args.headers; + + Response response = Execute(builder); + if (!response) return response; + + UploadPartCopyResponse resp; + resp.etag = utils::Trim(response.headers.GetFront("etag"), '"'); + + return resp; +} + +minio::s3::StatObjectResponse minio::s3::Client::StatObject( + StatObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + if (args.ssec != NULL && !base_url_.is_https) { + return error::Error( + "SSE-C operation must be performed over a secure connection"); + } + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kHead, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + if (!args.version_id.empty()) { + query_params.Add("versionId", args.version_id); + builder.query_params = query_params; + } + builder.headers = args.Headers(); + + Response response = Execute(builder); + if (!response) return response; + + StatObjectResponse resp = response; + resp.bucket_name = args.bucket; + resp.object_name = args.object; + resp.version_id = response.headers.GetFront("x-amz-version-id"); + + resp.etag = utils::Trim(response.headers.GetFront("etag"), '"'); + + std::string value = response.headers.GetFront("content-length"); + if (!value.empty()) resp.size = std::stoi(value); + + value = response.headers.GetFront("last-modified"); + if (!value.empty()) { + resp.last_modified = utils::Time::FromHttpHeaderValue(value.c_str()); + } + + value = response.headers.GetFront("x-amz-object-lock-mode"); + if (!value.empty()) resp.retention_mode = StringToRetentionMode(value); + + value = response.headers.GetFront("x-amz-object-lock-retain-until-date"); + if (!value.empty()) { + resp.retention_retain_until_date = + utils::Time::FromISO8601UTC(value.c_str()); + } + + value = response.headers.GetFront("x-amz-object-lock-legal-hold"); + if (!value.empty()) resp.legal_hold = StringToLegalHold(value); + + value = response.headers.GetFront("x-amz-delete-marker"); + if (!value.empty()) resp.delete_marker = utils::StringToBool(value); + + utils::Multimap user_metadata; + std::list keys = response.headers.Keys(); + for (auto key : keys) { + if (utils::StartsWith(key, "x-amz-meta-")) { + std::list values = response.headers.Get(key); + key.erase(0, 11); + for (auto value : values) user_metadata.Add(key, value); + } + } + resp.user_metadata = user_metadata; + + return resp; +} + +minio::s3::RemoveObjectResponse minio::s3::Client::RemoveObject( + RemoveObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kDelete, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + if (!args.version_id.empty()) { + query_params.Add("versionId", args.version_id); + builder.query_params = query_params; + } + + return Execute(builder); +} + +minio::s3::DownloadObjectResponse minio::s3::Client::DownloadObject( + DownloadObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + if (args.ssec != NULL && !base_url_.is_https) { + return error::Error( + "SSE-C operation must be performed over a secure connection"); + } + + std::string etag; + size_t size; + { + StatObjectArgs soargs; + soargs.bucket = args.bucket; + soargs.region = args.region; + soargs.object = args.object; + soargs.version_id = args.version_id; + soargs.ssec = args.ssec; + StatObjectResponse resp = StatObject(soargs); + if (!resp) return resp; + etag = resp.etag; + size = resp.size; + } + + std::string temp_filename = + args.filename + "." + curlpp::escape(etag) + ".part.minio"; + std::ofstream fout(temp_filename, fout.trunc | fout.out); + if (!fout.is_open()) { + DownloadObjectResponse resp; + resp.error = "unable to open file " + temp_filename; + return resp; + } + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kGet, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + if (!args.version_id.empty()) { + query_params.Add("versionId", std::string(args.version_id)); + builder.query_params = query_params; + } + builder.data_callback = [](http::DataCallbackArgs args) -> size_t { + std::ofstream* fout = (std::ofstream*)args.user_arg; + *fout << std::string(args.buffer, args.length); + return args.size * args.length; + }; + builder.user_arg = &fout; + + Response response = Execute(builder); + fout.close(); + if (response) std::filesystem::rename(temp_filename, args.filename); + return response; +} + +minio::s3::GetObjectResponse minio::s3::Client::GetObject(GetObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + if (args.ssec != NULL && !base_url_.is_https) { + return error::Error( + "SSE-C operation must be performed over a secure connection"); + } + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kGet, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + utils::Multimap query_params; + if (!args.version_id.empty()) { + query_params.Add("versionId", args.version_id); + builder.query_params = query_params; + } + builder.data_callback = args.data_callback; + builder.user_arg = args.user_arg; + if (args.ssec != NULL) { + builder.headers = args.ssec->Headers(); + } + + return Execute(builder); +} + +minio::s3::ListObjectsResponse minio::s3::Client::ListObjectsV1( + ListObjectsV1Args args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + utils::Multimap query_params; + query_params.AddAll(args.extra_query_params); + query_params.AddAll(GetCommonListObjectsQueryParams( + args.delimiter, args.encoding_type, args.max_keys, args.prefix)); + if (!args.marker.empty()) query_params.Add("marker", args.marker); + + RequestBuilder builder(http::Method::kGet, region, base_url_); + builder.bucket_name = args.bucket; + builder.query_params = query_params; + builder.headers = args.extra_headers; + + Response resp = Execute(builder); + if (!resp) resp; + + return ListObjectsResponse::ParseXML(resp.data, false); +} + +minio::s3::ListObjectsResponse minio::s3::Client::ListObjectsV2( + ListObjectsV2Args args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + utils::Multimap query_params; + query_params.Add("list-type", "2"); + query_params.AddAll(args.extra_query_params); + query_params.AddAll(GetCommonListObjectsQueryParams( + args.delimiter, args.encoding_type, args.max_keys, args.prefix)); + if (!args.continuation_token.empty()) + query_params.Add("continuation-token", args.continuation_token); + if (args.fetch_owner) query_params.Add("fetch-owner", "true"); + if (!args.start_after.empty()) { + query_params.Add("start-after", args.start_after); + } + if (args.include_user_metadata) query_params.Add("metadata", "true"); + + RequestBuilder builder(http::Method::kGet, region, base_url_); + builder.bucket_name = args.bucket; + builder.query_params = query_params; + builder.headers = args.extra_headers; + + Response resp = Execute(builder); + if (!resp) resp; + + return ListObjectsResponse::ParseXML(resp.data, false); +} + +minio::s3::ListObjectsResponse minio::s3::Client::ListObjectVersions( + ListObjectVersionsArgs args) { + if (error::Error err = args.Validate()) return err; + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + utils::Multimap query_params; + query_params.Add("versions", ""); + query_params.AddAll(args.extra_query_params); + query_params.AddAll(GetCommonListObjectsQueryParams( + args.delimiter, args.encoding_type, args.max_keys, args.prefix)); + if (!args.key_marker.empty()) query_params.Add("key-marker", args.key_marker); + if (!args.version_id_marker.empty()) { + query_params.Add("version-id-marker", args.version_id_marker); + } + + RequestBuilder builder(http::Method::kGet, region, base_url_); + builder.bucket_name = args.bucket; + builder.query_params = query_params; + builder.headers = args.extra_headers; + + Response resp = Execute(builder); + if (!resp) resp; + + return ListObjectsResponse::ParseXML(resp.data, true); +} + +minio::s3::ListObjectsResult::ListObjectsResult(error::Error err) { + failed_ = true; + resp_.contents.push_back(Item(err)); + itr_ = resp_.contents.begin(); +} + +minio::s3::ListObjectsResult::ListObjectsResult(Client* client, + ListObjectsArgs* args) { + client_ = client; + args_ = args; + Populate(); +} + +void minio::s3::ListObjectsResult::Populate() { + if (args_->include_versions) { + args_->key_marker = resp_.next_key_marker; + args_->version_id_marker = resp_.next_version_id_marker; + } else if (args_->use_api_v1) { + args_->marker = resp_.next_marker; + } else { + args_->start_after = resp_.start_after; + args_->continuation_token = resp_.next_continuation_token; + } + + std::string region; + if (GetRegionResponse resp = + client_->GetRegion(args_->bucket, args_->region)) { + region = resp.region; + if (args_->recursive) { + args_->delimiter = ""; + } else if (args_->delimiter.empty()) { + args_->delimiter = "/"; + } + + if (args_->include_versions || !args_->version_id_marker.empty()) { + resp_ = client_->ListObjectVersions(*args_); + } else if (args_->use_api_v1) { + resp_ = client_->ListObjectsV1(*args_); + } else { + resp_ = client_->ListObjectsV2(*args_); + } + + if (!resp_) { + failed_ = true; + resp_.contents.push_back(Item(resp_)); + } + } else { + failed_ = true; + resp_.contents.push_back(Item(resp)); + } + + itr_ = resp_.contents.begin(); +} + +minio::s3::ListObjectsResult minio::s3::Client::ListObjects( + ListObjectsArgs args) { + if (error::Error err = args.Validate()) return err; + return ListObjectsResult(this, &args); +} + +minio::s3::PutObjectResponse minio::s3::Client::PutObject( + PutObjectArgs& args, std::string& upload_id, char* buf) { + utils::Multimap headers = args.Headers(); + if (!headers.Contains("Content-Type")) { + if (args.content_type.empty()) { + headers.Add("Content-Type", "application/octet-stream"); + } else { + headers.Add("Content-Type", args.content_type); + } + } + + long object_size = args.object_size; + size_t part_size = args.part_size; + size_t uploaded_size = 0; + unsigned int part_number = 0; + std::string one_byte; + bool stop = false; + std::list parts; + long part_count = args.part_count; + + while (!stop) { + part_number++; + + size_t bytes_read = 0; + if (part_count > 0) { + if (part_number == part_count) { + part_size = object_size - uploaded_size; + stop = true; + } + + if (error::Error err = + utils::ReadPart(args.stream, buf, part_size, bytes_read)) { + return err; + } + + if (bytes_read != part_size) { + return error::Error("not enough data in the stream; expected: " + + std::to_string(part_size) + + ", got: " + std::to_string(bytes_read) + " bytes"); + } + } else { + char* b = buf; + size_t size = part_size + 1; + + if (!one_byte.empty()) { + buf[0] = one_byte.front(); + b = buf + 1; + size--; + bytes_read = 1; + one_byte = ""; + } + + size_t n = 0; + if (error::Error err = utils::ReadPart(args.stream, b, size, n)) { + return err; + } + + bytes_read += n; + + // If bytes read is less than or equals to part size, then we have reached + // last part. + if (bytes_read <= part_size) { + part_count = part_number; + part_size = bytes_read; + stop = true; + } else { + one_byte = buf[part_size + 1]; + } + } + + std::string_view data(buf, part_size); + + uploaded_size += part_size; + + if (part_count == 1) { + PutObjectApiArgs api_args; + api_args.extra_query_params = args.extra_query_params; + api_args.bucket = args.bucket; + api_args.region = args.region; + api_args.object = args.object; + api_args.data = data; + api_args.headers = headers; + + return PutObject(api_args); + } + + if (upload_id.empty()) { + CreateMultipartUploadArgs cmu_args; + cmu_args.extra_query_params = args.extra_query_params; + cmu_args.bucket = args.bucket; + cmu_args.region = args.region; + cmu_args.object = args.object; + cmu_args.headers = headers; + if (CreateMultipartUploadResponse resp = + CreateMultipartUpload(cmu_args)) { + upload_id = resp.upload_id; + } else { + return resp; + } + } + + UploadPartArgs up_args; + up_args.bucket = args.bucket; + up_args.region = args.region; + up_args.object = args.object; + up_args.upload_id = upload_id; + up_args.part_number = part_number; + up_args.data = data; + if (args.sse != NULL) { + if (SseCustomerKey* ssec = dynamic_cast(args.sse)) { + up_args.headers = ssec->Headers(); + } + } + + if (UploadPartResponse resp = UploadPart(up_args)) { + parts.push_back(Part{part_number, resp.etag}); + } else { + return resp; + } + } + + CompleteMultipartUploadArgs cmu_args; + cmu_args.bucket = args.bucket; + cmu_args.region = args.region; + cmu_args.object = args.object; + cmu_args.upload_id = upload_id; + cmu_args.parts = parts; + return CompleteMultipartUpload(cmu_args); +} + +minio::s3::PutObjectResponse minio::s3::Client::PutObject(PutObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + if (args.sse != NULL && args.sse->TlsRequired() && !base_url_.is_https) { + return error::Error( + "SSE operation must be performed over a secure connection"); + } + + char* buf = NULL; + if (args.part_count > 0) { + buf = new char[args.part_size]; + } else { + buf = new char[args.part_size + 1]; + } + + std::string upload_id; + PutObjectResponse resp = PutObject(args, upload_id, buf); + delete buf; + + if (!resp && !upload_id.empty()) { + AbortMultipartUploadArgs amu_args; + amu_args.bucket = args.bucket; + amu_args.region = args.region; + amu_args.object = args.object; + amu_args.upload_id = upload_id; + AbortMultipartUpload(amu_args); + } + + return resp; +} + +minio::s3::CopyObjectResponse minio::s3::Client::CopyObject( + CopyObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + if (args.sse != NULL && args.sse->TlsRequired() && !base_url_.is_https) { + return error::Error( + "SSE operation must be performed over a secure connection"); + } + + if (args.source.ssec != NULL && !base_url_.is_https) { + return error::Error( + "SSE-C operation must be performed over a secure connection"); + } + + std::string etag; + size_t size; + { + StatObjectArgs soargs; + soargs.extra_headers = args.source.extra_headers; + soargs.extra_query_params = args.source.extra_query_params; + soargs.bucket = args.source.bucket; + soargs.region = args.source.region; + soargs.object = args.source.object; + soargs.version_id = args.source.version_id; + soargs.ssec = args.source.ssec; + StatObjectResponse resp = StatObject(soargs); + if (!resp) return resp; + etag = resp.etag; + size = resp.size; + } + + if (args.source.offset != NULL || args.source.length != NULL || + size > utils::kMaxPartSize) { + if (args.metadata_directive != NULL && + *args.metadata_directive == Directive::kCopy) { + return error::Error( + "COPY metadata directive is not applicable to source object size " + "greater than 5 GiB"); + } + + if (args.tagging_directive != NULL && + *args.tagging_directive == Directive::kCopy) { + return error::Error( + "COPY tagging directive is not applicable to source object size " + "greater than 5 GiB"); + } + + ComposeSource src; + src.extra_headers = args.source.extra_headers; + src.extra_query_params = args.source.extra_query_params; + src.bucket = args.source.bucket; + src.region = args.source.region; + src.object = args.source.object; + src.ssec = args.source.ssec; + src.offset = args.source.offset; + src.length = args.source.length; + src.match_etag = args.source.match_etag; + src.not_match_etag = args.source.not_match_etag; + src.modified_since = args.source.modified_since; + src.unmodified_since = args.source.unmodified_since; + + ComposeObjectArgs coargs; + coargs.extra_headers = args.extra_headers; + coargs.extra_query_params = args.extra_query_params; + coargs.bucket = args.bucket; + coargs.region = args.region; + coargs.object = args.object; + coargs.sse = args.sse; + coargs.sources.push_back(src); + + return ComposeObject(coargs); + } + + utils::Multimap headers; + headers.AddAll(args.extra_headers); + headers.AddAll(args.Headers()); + if (args.metadata_directive != NULL) { + headers.Add("x-amz-metadata-directive", + DirectiveToString(*args.metadata_directive)); + } + if (args.tagging_directive != NULL) { + headers.Add("x-amz-tagging-directive", + DirectiveToString(*args.tagging_directive)); + } + headers.AddAll(args.source.CopyHeaders()); + + std::string region; + if (GetRegionResponse resp = GetRegion(args.bucket, args.region)) { + region = resp.region; + } else { + return resp; + } + + RequestBuilder builder(http::Method::kPut, region, base_url_); + builder.bucket_name = args.bucket; + builder.object_name = args.object; + builder.query_params = args.extra_query_params; + builder.headers = headers; + + Response response = Execute(builder); + if (!response) return response; + + CopyObjectResponse resp; + resp.etag = utils::Trim(response.headers.GetFront("etag"), '"'); + resp.version_id = response.headers.GetFront("x-amz-version-id"); + + return resp; +} + +minio::s3::StatObjectResponse minio::s3::Client::CalculatePartCount( + size_t& part_count, std::list sources) { + size_t object_size = 0; + int i = 0; + for (auto& source : sources) { + if (source.ssec != NULL && !base_url_.is_https) { + std::string msg = "source " + source.bucket + "/" + source.object; + if (!source.version_id.empty()) msg += "?versionId=" + source.version_id; + msg += ": SSE-C operation must be performed over a secure connection"; + return error::Error(msg); + } + + i++; + + std::string etag; + size_t size; + + StatObjectArgs soargs; + soargs.extra_headers = source.extra_headers; + soargs.extra_query_params = source.extra_query_params; + soargs.bucket = source.bucket; + soargs.region = source.region; + soargs.object = source.object; + soargs.version_id = source.version_id; + soargs.ssec = source.ssec; + StatObjectResponse resp = StatObject(soargs); + if (!resp) return resp; + etag = resp.etag; + size = resp.size; + if (error::Error err = source.BuildHeaders(size, etag)) return err; + + if (source.length != NULL) { + size = *source.length; + } else if (source.offset != NULL) { + size -= *source.offset; + } + + if (size < utils::kMinPartSize && sources.size() != 1 && + i != sources.size()) { + std::string msg = "source " + source.bucket + "/" + source.object; + if (!source.version_id.empty()) msg += "?versionId=" + source.version_id; + msg += ": size " + std::to_string(size) + " must be greater than " + + std::to_string(utils::kMinPartSize); + return error::Error(msg); + } + + object_size += size; + if (object_size > utils::kMaxObjectSize) { + return error::Error("destination object size must be less than " + + std::to_string(utils::kMaxObjectSize)); + } + + if (size > utils::kMaxPartSize) { + size_t count = size / utils::kMaxPartSize; + size_t last_part_size = size - (count * utils::kMaxPartSize); + if (last_part_size > 0) { + count++; + } else { + last_part_size = utils::kMaxPartSize; + } + + if (last_part_size < utils::kMinPartSize && sources.size() != 1 && + i != sources.size()) { + std::string msg = "source " + source.bucket + "/" + source.object; + if (!source.version_id.empty()) { + msg += "?versionId=" + source.version_id; + } + msg += ": size " + std::to_string(size) + + " for multipart split upload of " + std::to_string(size) + + ", last part size is less than " + + std::to_string(utils::kMinPartSize); + return error::Error(msg); + } + + part_count += count; + } else { + part_count++; + } + + if (part_count > utils::kMaxMultipartCount) { + return error::Error( + "Compose sources create more than allowed multipart count " + + std::to_string(utils::kMaxMultipartCount)); + } + } + + return error::SUCCESS; +} + +minio::s3::ComposeObjectResponse minio::s3::Client::ComposeObject( + ComposeObjectArgs args, std::string& upload_id) { + size_t part_count = 0; + { + StatObjectResponse resp = CalculatePartCount(part_count, args.sources); + if (!resp) return resp; + } + + ComposeSource& source = args.sources.front(); + if (part_count == 1 && source.offset == NULL && source.length == NULL) { + CopySource src; + src.extra_headers = source.extra_headers; + src.extra_query_params = source.extra_query_params; + src.bucket = source.bucket; + src.region = source.region; + src.object = source.object; + src.ssec = source.ssec; + src.offset = source.offset; + src.length = source.length; + src.match_etag = source.match_etag; + src.not_match_etag = source.not_match_etag; + src.modified_since = source.modified_since; + src.unmodified_since = source.unmodified_since; + + CopyObjectArgs coargs; + coargs.extra_headers = args.extra_headers; + coargs.extra_query_params = args.extra_query_params; + coargs.bucket = args.bucket; + coargs.region = args.region; + coargs.object = args.object; + coargs.sse = args.sse; + coargs.source = src; + + return CopyObject(coargs); + } + + utils::Multimap headers = args.Headers(); + + { + CreateMultipartUploadArgs cmu_args; + cmu_args.extra_query_params = args.extra_query_params; + cmu_args.bucket = args.bucket; + cmu_args.region = args.region; + cmu_args.object = args.object; + cmu_args.headers = headers; + if (CreateMultipartUploadResponse resp = CreateMultipartUpload(cmu_args)) { + upload_id = resp.upload_id; + } else { + return resp; + } + } + + unsigned int part_number = 0; + utils::Multimap ssecheaders; + if (args.sse != NULL) { + if (SseCustomerKey* ssec = dynamic_cast(args.sse)) { + ssecheaders = ssec->Headers(); + } + } + + std::list parts; + for (auto& source : args.sources) { + size_t size = source.ObjectSize(); + if (source.length != NULL) { + size = *source.length; + } else if (source.offset != NULL) { + size -= *source.offset; + } + + size_t offset = 0; + if (source.offset != NULL) offset = *source.offset; + + utils::Multimap headers; + headers.AddAll(source.Headers()); + headers.AddAll(ssecheaders); + + if (size <= utils::kMaxPartSize) { + part_number++; + if (source.length != NULL) { + headers.Add("x-amz-copy-source-range", + "bytes=" + std::to_string(offset) + "-" + + std::to_string(offset + *source.length - 1)); + } else if (source.offset != NULL) { + headers.Add("x-amz-copy-source-range", + "bytes=" + std::to_string(offset) + "-" + + std::to_string(offset + size - 1)); + } + + UploadPartCopyArgs upc_args; + upc_args.bucket = args.bucket; + upc_args.region = args.region; + upc_args.object = args.object; + upc_args.headers = headers; + upc_args.upload_id = upload_id; + upc_args.part_number = part_number; + UploadPartCopyResponse resp = UploadPartCopy(upc_args); + if (!resp) return resp; + parts.push_back(Part{part_number, resp.etag}); + } else { + while (size > 0) { + part_number++; + + size_t start_bytes = offset; + size_t end_bytes = start_bytes + utils::kMaxPartSize; + if (size < utils::kMaxPartSize) end_bytes = start_bytes + size; + + utils::Multimap headerscopy; + headerscopy.AddAll(headers); + headerscopy.Add("x-amz-copy-source-range", + "bytes=" + std::to_string(start_bytes) + "-" + + std::to_string(end_bytes)); + + UploadPartCopyArgs upc_args; + upc_args.bucket = args.bucket; + upc_args.region = args.region; + upc_args.object = args.object; + upc_args.headers = headerscopy; + upc_args.upload_id = upload_id; + upc_args.part_number = part_number; + UploadPartCopyResponse resp = UploadPartCopy(upc_args); + if (!resp) return resp; + parts.push_back(Part{part_number, resp.etag}); + + offset = start_bytes; + size -= (end_bytes - start_bytes); + } + } + } + + CompleteMultipartUploadArgs cmu_args; + cmu_args.bucket = args.bucket; + cmu_args.region = args.region; + cmu_args.object = args.object; + cmu_args.upload_id = upload_id; + cmu_args.parts = parts; + return CompleteMultipartUpload(cmu_args); +} + +minio::s3::ComposeObjectResponse minio::s3::Client::ComposeObject( + ComposeObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + if (args.sse != NULL && args.sse->TlsRequired() && !base_url_.is_https) { + return error::Error( + "SSE operation must be performed over a secure connection"); + } + + std::string upload_id; + ComposeObjectResponse resp = ComposeObject(args, upload_id); + if (!resp && !upload_id.empty()) { + AbortMultipartUploadArgs amu_args; + amu_args.bucket = args.bucket; + amu_args.region = args.region; + amu_args.object = args.object; + amu_args.upload_id = upload_id; + AbortMultipartUpload(amu_args); + } + + return resp; +} + +minio::s3::UploadObjectResponse minio::s3::Client::UploadObject( + UploadObjectArgs args) { + if (error::Error err = args.Validate()) return err; + + std::ifstream file; + file.exceptions(std::ifstream::failbit | std::ifstream::badbit); + try { + file.open(args.filename); + } catch (std::system_error& err) { + return error::Error("unable to open file " + args.filename + "; " + + err.code().message()); + } + + PutObjectArgs po_args(file, args.object_size, args.part_size); + po_args.extra_headers = args.extra_headers; + po_args.extra_query_params = args.extra_query_params; + po_args.bucket = args.bucket; + po_args.region = args.region; + po_args.object = args.object; + po_args.headers = args.headers; + po_args.user_metadata = args.user_metadata; + po_args.sse = args.sse; + po_args.tags = args.tags; + po_args.retention = args.retention; + po_args.legal_hold = args.legal_hold; + po_args.part_count = args.part_count; + po_args.content_type = args.content_type; + + PutObjectResponse resp = PutObject(po_args); + file.close(); + return resp; +} diff --git a/src/creds.cc b/src/creds.cc new file mode 100644 index 00000000..8a2a42e8 --- /dev/null +++ b/src/creds.cc @@ -0,0 +1,59 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "creds.h" + +minio::creds::Credentials::Credentials(const Credentials& creds) { + access_key_ = creds.access_key_; + secret_key_ = creds.secret_key_; + session_token_ = creds.session_token_; + expiration_ = creds.expiration_; +} + +minio::creds::Credentials::Credentials(std::string_view access_key, + std::string_view secret_key, + std::string_view session_token, + unsigned int expiration) { + access_key_ = access_key; + secret_key_ = secret_key; + session_token_ = session_token; + expiration_ = expiration; +} + +std::string minio::creds::Credentials::AccessKey() { + return std::string(access_key_); +} + +std::string minio::creds::Credentials::SecretKey() { + return std::string(secret_key_); +} + +std::string minio::creds::Credentials::SessionToken() { + return std::string(session_token_); +} + +bool minio::creds::Credentials::IsExpired() { return expiration_ != 0; } + +minio::creds::StaticProvider::StaticProvider(std::string_view access_key, + std::string_view secret_key, + std::string_view session_token) { + creds_ = new Credentials(access_key, secret_key, session_token); +} + +minio::creds::StaticProvider::~StaticProvider() { delete creds_; } + +minio::creds::Credentials minio::creds::StaticProvider::Fetch() { + return *creds_; +} diff --git a/src/http.cc b/src/http.cc new file mode 100644 index 00000000..f8cfbaa1 --- /dev/null +++ b/src/http.cc @@ -0,0 +1,342 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "http.h" + +std::string minio::http::ExtractRegion(std::string host) { + std::stringstream str_stream(host); + std::string token; + std::vector tokens; + while (std::getline(str_stream, token, '.')) tokens.push_back(token); + + token = tokens[1]; + + // If token is "dualstack", then region might be in next token. + if (token == "dualstack") token = tokens[2]; + + // If token is equal to "amazonaws", region is not passed in the host. + if (token == "amazonaws") return ""; + + // Return token as region. + return token; +} + +minio::error::Error minio::http::BaseUrl::SetHost(std::string hostvalue) { + std::stringstream str_stream(hostvalue); + std::string portstr; + while (std::getline(str_stream, portstr, ':')) + ; + try { + port = std::stoi(portstr); + hostvalue = hostvalue.substr(0, hostvalue.rfind(":" + portstr)); + } catch (std::invalid_argument) { + port = 0; + } + + accelerate_host = utils::StartsWith(hostvalue, "s3-accelerate."); + aws_host = ((utils::StartsWith(hostvalue, "s3.") || accelerate_host) && + (utils::EndsWith(hostvalue, ".amazonaws.com") || + utils::EndsWith(hostvalue, ".amazonaws.com.cn"))); + virtual_style = aws_host || utils::EndsWith(hostvalue, "aliyuncs.com"); + + if (aws_host) { + std::string awshost; + bool is_aws_china_host = utils::EndsWith(hostvalue, ".cn"); + awshost = "amazonaws.com"; + if (is_aws_china_host) awshost = "amazonaws.com.cn"; + region = ExtractRegion(hostvalue); + + if (is_aws_china_host && region.empty()) { + return error::Error("region missing in Amazon S3 China endpoint " + + hostvalue); + } + + dualstack_host = utils::Contains(hostvalue, ".dualstack."); + hostvalue = awshost; + } else { + accelerate_host = false; + } + + host = hostvalue; + + return error::SUCCESS; +} + +std::string minio::http::BaseUrl::GetHostHeaderValue() { + // ignore port when port and service match i.e HTTP -> 80, HTTPS -> 443 + if (port == 0 || (!is_https && port == 80) || (is_https && port == 443)) { + return host; + } + return host + ":" + std::to_string(port); +} + +minio::error::Error minio::http::BaseUrl::BuildUrl(utils::Url &url, + Method method, + std::string region, + utils::Multimap query_params, + std::string bucket_name, + std::string object_name) { + if (bucket_name.empty() && !object_name.empty()) { + return error::Error("empty bucket name for object name " + object_name); + } + + std::string host = GetHostHeaderValue(); + + if (bucket_name.empty()) { + if (aws_host) host = "s3." + region + "." + host; + url = utils::Url{is_https, host, "/"}; + return error::SUCCESS; + } + + bool enforce_path_style = ( + // CreateBucket API requires path style in Amazon AWS S3. + (method == Method::kPut && object_name.empty() && !query_params) || + + // GetBucketLocation API requires path style in Amazon AWS S3. + query_params.Contains("location") || + + // Use path style for bucket name containing '.' which causes + // SSL certificate validation error. + (utils::Contains(bucket_name, '.') && is_https)); + + if (aws_host) { + std::string s3_domain = "s3."; + if (accelerate_host) { + if (utils::Contains(bucket_name, '.')) { + return error::Error( + "bucket name '" + bucket_name + + "' with '.' is not allowed for accelerate endpoint"); + } + + if (!enforce_path_style) s3_domain = "s3-accelerate."; + } + + if (dualstack_host) s3_domain += "dualstack."; + if (enforce_path_style || !accelerate_host) { + s3_domain += region + "."; + } + host = s3_domain + host; + } + + std::string path; + if (enforce_path_style || !virtual_style) { + path = "/" + bucket_name; + } else { + host = bucket_name + "." + host; + } + + if (!object_name.empty()) { + if (*(object_name.begin()) != '/') path += "/"; + path += utils::EncodePath(object_name); + } + + url = utils::Url{is_https, host, path, query_params.ToQueryString()}; + + return error::SUCCESS; +} + +size_t minio::http::Response::ReadStatusCode(char *buffer, size_t size, + size_t length) { + size_t real_size = size * length; + + response_ += std::string(buffer, length); + + size_t pos = response_.find("\r\n"); + if (pos == std::string::npos) return real_size; + + std::string line = response_.substr(0, pos); + response_ = response_.substr(pos + 2); + + if (continue100_) { + if (!line.empty()) { + error = "invalid HTTP response"; + return real_size; + } + + continue100_ = false; + + pos = response_.find("\r\n"); + if (pos == std::string::npos) return real_size; + + line = response_.substr(0, pos); + response_ = response_.substr(pos + 2); + } + + // Skip HTTP/1.x. + pos = line.find(" "); + if (pos == std::string::npos) { + error = "invalid HTTP response"; + return real_size; + } + line = line.substr(pos + 1); + + // Read status code. + pos = line.find(" "); + if (pos == std::string::npos) { + error = "invalid HTTP response"; + return real_size; + } + std::string code = line.substr(0, pos); + std::string::size_type st; + status_code = std::stoi(code, &st); + if (st == std::string::npos) error = "invalid HTTP response code"; + + if (status_code == 100) { + continue100_ = true; + } else { + status_code_read_ = true; + } + + return real_size; +} + +size_t minio::http::Response::ReadHeaders(curlpp::Easy *handle, char *buffer, + size_t size, size_t length) { + size_t real_size = size * length; + + response_ += std::string(buffer, length); + size_t pos = response_.find("\r\n\r\n"); + if (pos == std::string::npos) return real_size; + + headers_read_ = true; + + std::string lines = response_.substr(0, pos); + body = response_.substr(pos + 4); + + while ((pos = lines.find("\r\n")) != std::string::npos) { + std::string line = lines.substr(0, pos); + lines.erase(0, pos + 2); + + if ((pos = line.find(": ")) == std::string::npos) { + error = "invalid HTTP header: " + line; + return real_size; + } + + headers.Add(line.substr(0, pos), line.substr(pos + 2)); + } + + if (!lines.empty()) { + if ((pos = lines.find(": ")) == std::string::npos) { + error = "invalid HTTP header: " + lines; + return real_size; + } + + headers.Add(lines.substr(0, pos), lines.substr(pos + 2)); + } + + if (body.size() == 0 || data_callback == NULL || status_code < 200 || + status_code > 299) + return real_size; + + DataCallbackArgs args = {handle, this, body.data(), 1, body.size(), user_arg}; + size_t written = data_callback(args); + if (written == body.size()) written = real_size; + body = ""; + return written; +} + +size_t minio::http::Response::ResponseCallback(curlpp::Easy *handle, + char *buffer, size_t size, + size_t length) { + size_t real_size = size * length; + + // As error occurred previously, just drain the connection. + if (!error.empty()) return real_size; + + if (!status_code_read_) return ReadStatusCode(buffer, size, length); + + if (!headers_read_) return ReadHeaders(handle, buffer, size, length); + + // Received unsuccessful HTTP response code. + if (data_callback == NULL || status_code < 200 || status_code > 299) { + body += std::string(buffer, length); + return real_size; + } + + return data_callback( + DataCallbackArgs{handle, this, buffer, size, length, user_arg}); +} + +minio::http::Request::Request(Method httpmethod, utils::Url httpurl) { + method = httpmethod; + url = httpurl; +} + +minio::http::Response minio::http::Request::execute() { + curlpp::Cleanup cleaner; + curlpp::Easy request; + + // Request settings. + request.setOpt( + new curlpp::options::CustomRequest{http::MethodToString(method)}); + std::string urlstring = url.String(); + request.setOpt(new curlpp::Options::Url(urlstring)); + if (debug) request.setOpt(new curlpp::Options::Verbose(true)); + if (ignore_cert_check) { + request.setOpt(new curlpp::Options::SslVerifyPeer(false)); + } + + utils::CharBuffer charbuf((char *)body.data(), body.size()); + std::istream body_stream(&charbuf); + + switch (method) { + case Method::kHead: + request.setOpt(new curlpp::options::NoBody(true)); + break; + case Method::kPut: + case Method::kPost: + if (!headers.Contains("Content-Length")) { + headers.Add("Content-Length", std::to_string(body.size())); + } + request.setOpt(new curlpp::Options::ReadStream(&body_stream)); + request.setOpt(new curlpp::Options::InfileSize(body.size())); + request.setOpt(new curlpp::Options::Upload(true)); + break; + } + + std::list headerlist = headers.ToHttpHeaders(); + headerlist.push_back("Expect:"); // Disable 100 continue from server. + request.setOpt(new curlpp::Options::HttpHeader(headerlist)); + + // Response settings. + request.setOpt(new curlpp::options::Header(true)); + + Response response; + response.data_callback = data_callback; + response.user_arg = user_arg; + + using namespace std::placeholders; + request.setOpt(new curlpp::options::WriteFunction( + std::bind(&Response::ResponseCallback, &response, &request, _1, _2, _3))); + + // Execute. + request.perform(); + + return response; +} + +minio::http::Response minio::http::Request::Execute() { + try { + return execute(); + } catch (curlpp::LogicError &e) { + Response response; + response.error = std::string("curlpp::LogicError: ") + e.what(); + return response; + } catch (curlpp::RuntimeError &e) { + Response response; + response.error = std::string("curlpp::RuntimeError: ") + e.what(); + return response; + } +} diff --git a/src/request-builder.cc b/src/request-builder.cc new file mode 100644 index 00000000..0eae662d --- /dev/null +++ b/src/request-builder.cc @@ -0,0 +1,117 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client.h" + +minio::s3::RequestBuilder::RequestBuilder(http::Method httpmethod, + std::string regionvalue, + http::BaseUrl& baseurl) + : method(httpmethod), region(regionvalue), base_url(baseurl) {} + +void minio::s3::RequestBuilder::BuildHeaders(utils::Url& url, + creds::Provider* provider) { + headers.Add("Host", url.host); + + if (user_agent.empty()) { + headers.Add("User-Agent", "MinIO (Linux; x86_64) minio-cpp/0.1.0"); + } else { + headers.Add("User-Agent", user_agent); + } + + bool md5sum_added = headers.Contains("Content-MD5"); + std::string md5sum; + + switch (method) { + case http::Method::kPut: + case http::Method::kPost: + headers.Add("Content-Length", std::to_string(body.size())); + if (!headers.Contains("Content-Type")) { + headers.Add("Content-Type", "application/octet-stream"); + } + } + + // MD5 hash of zero length byte array. + // public static final String ZERO_MD5_HASH = "1B2M2Y8AsgTpgAmY7PhCfg=="; + + if (provider != NULL) { + if (url.is_https) { + sha256 = "UNSIGNED-PAYLOAD"; + switch (method) { + case http::Method::kPut: + case http::Method::kPost: + if (!md5sum_added) { + md5sum = utils::Md5sumHash(body); + } + } + } else { + switch (method) { + case http::Method::kPut: + case http::Method::kPost: + sha256 = utils::Sha256Hash(body); + break; + default: + sha256 = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b85" + "5"; + } + } + } else { + switch (method) { + case http::Method::kPut: + case http::Method::kPost: + if (!md5sum_added) { + md5sum = utils::Md5sumHash(body); + } + } + } + + if (!md5sum.empty()) headers.Add("Content-MD5", md5sum); + if (!sha256.empty()) headers.Add("x-amz-content-sha256", sha256); + + date = utils::Time::Now(); + headers.Add("x-amz-date", date.ToAmzDate()); + + if (provider != NULL) { + creds::Credentials creds = provider->Fetch(); + if (!creds.SessionToken().empty()) { + headers.Add("X-Amz-Security-Token", creds.SessionToken()); + } + signer::SignV4S3(method, url.path, region, headers, query_params, + creds.AccessKey(), creds.SecretKey(), sha256, date); + } +} + +minio::http::Request minio::s3::RequestBuilder::Build( + creds::Provider* provider) { + utils::Url url; + if (error::Error err = + base_url.BuildUrl(url, method, std::string(region), query_params, + bucket_name, object_name)) { + std::cerr << "failed to build url. error=" << err + << ". This should not happen" << std::endl; + std::terminate(); + } + BuildHeaders(url, provider); + + http::Request request(method, url); + request.body = body; + request.headers = headers; + request.data_callback = data_callback; + request.user_arg = user_arg; + request.debug = debug; + request.ignore_cert_check = ignore_cert_check; + + return request; +} diff --git a/src/response.cc b/src/response.cc new file mode 100644 index 00000000..7a543db3 --- /dev/null +++ b/src/response.cc @@ -0,0 +1,391 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "response.h" + +minio::s3::Response::Response() {} + +minio::s3::Response::Response(error::Error err) { error = err.String(); } + +minio::s3::Response::Response(const Response& response) { + status_code = response.status_code; + headers = response.headers; + data = response.data; + error = response.error; + code = response.code; + message = response.message; + resource = response.resource; + request_id = response.request_id; + host_id = response.host_id; + bucket_name = response.bucket_name; + object_name = response.object_name; +} + +std::string minio::s3::Response::GetError() { + if (!error.empty()) return error; + if (!code.empty()) return code + ": " + message; + if (status_code && (status_code < 200 || status_code > 299)) { + return "failed with HTTP status code " + std::to_string(status_code); + } + return ""; +} + +minio::s3::Response minio::s3::Response::ParseXML(std::string_view data, + int status_code, + utils::Multimap headers) { + minio::s3::Response resp; + resp.status_code = status_code; + resp.headers = headers; + + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(data.data()); + if (!result) { + resp.error = "unable to parse XML; " + std::string(data); + return resp; + } + + auto root = xdoc.select_node("/Error"); + pugi::xpath_node text; + + text = root.node().select_node("Code/text()"); + resp.code = text.node().value(); + + text = root.node().select_node("Message/text()"); + resp.message = text.node().value(); + + text = root.node().select_node("Resource/text()"); + resp.resource = text.node().value(); + + text = root.node().select_node("RequestId/text()"); + resp.request_id = text.node().value(); + + text = root.node().select_node("HostId/text()"); + resp.host_id = text.node().value(); + + text = root.node().select_node("BucketName/text()"); + resp.bucket_name = text.node().value(); + + text = root.node().select_node("Key/text()"); + resp.object_name = text.node().value(); + + return resp; +} + +minio::s3::GetRegionResponse::GetRegionResponse(std::string regionvalue) { + region = regionvalue; +} + +minio::s3::GetRegionResponse::GetRegionResponse(error::Error err) + : Response(err) {} + +minio::s3::GetRegionResponse::GetRegionResponse(const Response& response) + : Response(response) {} + +minio::s3::ListBucketsResponse::ListBucketsResponse( + std::list bucketlist) { + buckets = bucketlist; +} + +minio::s3::ListBucketsResponse::ListBucketsResponse(error::Error err) + : Response(err) {} + +minio::s3::ListBucketsResponse::ListBucketsResponse(const Response& response) + : Response(response) {} + +minio::s3::ListBucketsResponse minio::s3::ListBucketsResponse::ParseXML( + std::string_view data) { + std::list buckets; + + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(data.data()); + if (!result) return error::Error("unable to parse XML"); + pugi::xpath_node_set xnodes = + xdoc.select_nodes("/ListAllMyBucketsResult/Buckets/Bucket"); + for (auto xnode : xnodes) { + std::string name; + utils::Time creation_date; + if (auto node = xnode.node().select_node("Name/text()").node()) { + name = node.value(); + } + if (auto node = xnode.node().select_node("CreationDate/text()").node()) { + std::string value = node.value(); + creation_date = utils::Time::FromISO8601UTC(value.c_str()); + } + + buckets.push_back(Bucket{name, creation_date}); + } + + return buckets; +} + +minio::s3::BucketExistsResponse::BucketExistsResponse(bool existflag) { + exist = existflag; +} + +minio::s3::BucketExistsResponse::BucketExistsResponse(error::Error err) + : Response(err) {} + +minio::s3::BucketExistsResponse::BucketExistsResponse(const Response& response) + : Response(response) {} + +minio::s3::CompleteMultipartUploadResponse::CompleteMultipartUploadResponse() {} + +minio::s3::CompleteMultipartUploadResponse::CompleteMultipartUploadResponse( + error::Error err) + : Response(err) {} + +minio::s3::CompleteMultipartUploadResponse::CompleteMultipartUploadResponse( + const Response& response) + : Response(response) {} + +minio::s3::CompleteMultipartUploadResponse +minio::s3::CompleteMultipartUploadResponse::ParseXML(std::string_view data, + std::string version_id) { + CompleteMultipartUploadResponse resp; + + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(data.data()); + if (!result) return error::Error("unable to parse XML"); + + auto root = xdoc.select_node("/CompleteMultipartUploadOutput"); + + pugi::xpath_node text; + + text = root.node().select_node("Bucket/text()"); + resp.bucket_name = text.node().value(); + + text = root.node().select_node("Key/text()"); + resp.object_name = text.node().value(); + + text = root.node().select_node("Location/text()"); + resp.location = text.node().value(); + + text = root.node().select_node("ETag/text()"); + resp.etag = utils::Trim(text.node().value(), '"'); + + resp.version_id = version_id; + + return resp; +} + +minio::s3::CreateMultipartUploadResponse::CreateMultipartUploadResponse( + std::string uploadid) { + upload_id = uploadid; +} + +minio::s3::CreateMultipartUploadResponse::CreateMultipartUploadResponse( + error::Error err) + : Response(err) {} + +minio::s3::CreateMultipartUploadResponse::CreateMultipartUploadResponse( + const Response& response) + : Response(response) {} + +minio::s3::PutObjectResponse::PutObjectResponse() {} + +minio::s3::PutObjectResponse::PutObjectResponse(const Response& response) + : Response(response) {} + +minio::s3::PutObjectResponse::PutObjectResponse(error::Error err) + : Response(err) {} + +minio::s3::StatObjectResponse::StatObjectResponse() {} + +minio::s3::StatObjectResponse::StatObjectResponse(error::Error err) + : Response(err) {} + +minio::s3::StatObjectResponse::StatObjectResponse(const Response& response) + : Response(response) {} + +minio::s3::Item::Item() {} + +minio::s3::Item::Item(error::Error err) : Response(err) {} + +minio::s3::Item::Item(const Response& response) : Response(response) {} + +minio::s3::ListObjectsResponse::ListObjectsResponse() {} + +minio::s3::ListObjectsResponse::ListObjectsResponse(error::Error err) + : Response(err) {} + +minio::s3::ListObjectsResponse::ListObjectsResponse(const Response& response) + : Response(response) {} + +minio::s3::ListObjectsResponse minio::s3::ListObjectsResponse::ParseXML( + std::string_view data, bool version) { + ListObjectsResponse resp; + + pugi::xml_document xdoc; + pugi::xml_parse_result result = xdoc.load_string(data.data()); + if (!result) return error::Error("unable to parse XML"); + + std::string xpath = version ? "/ListVersionsResult" : "/ListBucketResult"; + + auto root = xdoc.select_node(xpath.c_str()); + + pugi::xpath_node text; + std::string value; + + text = root.node().select_node("Name/text()"); + resp.name = text.node().value(); + + text = root.node().select_node("EncodingType/text()"); + resp.encoding_type = text.node().value(); + + text = root.node().select_node("Prefix/text()"); + value = text.node().value(); + resp.prefix = (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + text = root.node().select_node("Delimiter/text()"); + resp.delimiter = text.node().value(); + + text = root.node().select_node("IsTruncated/text()"); + value = text.node().value(); + if (!value.empty()) resp.is_truncated = utils::StringToBool(value); + + text = root.node().select_node("MaxKeys/text()"); + value = text.node().value(); + if (!value.empty()) resp.max_keys = std::stoi(value); + + // ListBucketResult V1 + { + text = root.node().select_node("Marker/text()"); + value = text.node().value(); + resp.marker = + (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + text = root.node().select_node("NextMarker/text()"); + value = text.node().value(); + resp.next_marker = + (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + } + + // ListBucketResult V2 + { + text = root.node().select_node("KeyCount/text()"); + value = text.node().value(); + if (!value.empty()) resp.key_count = std::stoi(value); + + text = root.node().select_node("StartAfter/text()"); + value = text.node().value(); + resp.start_after = + (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + text = root.node().select_node("ContinuationToken/text()"); + resp.continuation_token = text.node().value(); + + text = root.node().select_node("NextContinuationToken/text()"); + resp.next_continuation_token = text.node().value(); + } + + // ListVersionsResult + { + text = root.node().select_node("KeyMarker/text()"); + value = text.node().value(); + resp.key_marker = + (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + text = root.node().select_node("NextKeyMarker/text()"); + value = text.node().value(); + resp.next_key_marker = + (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + text = root.node().select_node("VersionIdMarker/text()"); + resp.version_id_marker = text.node().value(); + + text = root.node().select_node("NextVersionIdMarker/text()"); + resp.next_version_id_marker = text.node().value(); + } + + Item last_item; + + auto populate = [&resp = resp, &last_item = last_item]( + std::list& items, pugi::xpath_node_set& contents, + bool is_delete_marker) -> void { + for (auto content : contents) { + pugi::xpath_node text; + std::string value; + Item item; + + text = content.node().select_node("ETag/text()"); + item.etag = utils::Trim(text.node().value(), '"'); + + text = content.node().select_node("Key/text()"); + value = text.node().value(); + item.name = + (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + text = content.node().select_node("LastModified/text()"); + value = text.node().value(); + item.last_modified = utils::Time::FromISO8601UTC(value.c_str()); + + text = content.node().select_node("Owner/ID/text()"); + item.owner_id = text.node().value(); + + text = content.node().select_node("Owner/DisplayName/text()"); + item.owner_name = text.node().value(); + + text = content.node().select_node("Size/text()"); + value = text.node().value(); + if (!value.empty()) item.size = std::stoi(value); + + text = content.node().select_node("StorageClass/text()"); + item.storage_class = text.node().value(); + + text = content.node().select_node("IsLatest/text()"); + value = text.node().value(); + if (!value.empty()) item.is_latest = utils::StringToBool(value); + + text = content.node().select_node("VersionId/text()"); + item.version_id = text.node().value(); + + auto user_metadata = content.node().select_node("UserMetadata"); + for (auto metadata = user_metadata.node().first_child(); metadata; + metadata = metadata.next_sibling()) { + item.user_metadata[metadata.name()] = metadata.child_value(); + } + + item.is_delete_marker = is_delete_marker; + + items.push_back(item); + last_item = item; + } + }; + + auto contents = root.node().select_nodes(version ? "Version" : "Contents"); + populate(resp.contents, contents, false); + // Only for ListObjectsV1. + if (resp.is_truncated && resp.next_marker.empty()) { + resp.next_marker = last_item.name; + } + + auto common_prefixes = root.node().select_nodes("CommonPrefixes"); + for (auto common_prefix : common_prefixes) { + Item item; + + text = common_prefix.node().select_node("Prefix/text()"); + value = text.node().value(); + item.name = (resp.encoding_type == "url") ? curlpp::unescape(value) : value; + + item.is_prefix = true; + + resp.contents.push_back(item); + } + + auto delete_markers = root.node().select_nodes("DeleteMarker"); + populate(resp.contents, delete_markers, true); + + return resp; +} diff --git a/src/signer.cc b/src/signer.cc new file mode 100644 index 00000000..b462fdd3 --- /dev/null +++ b/src/signer.cc @@ -0,0 +1,146 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "signer.h" + +const char* SIGN_V4_ALGORITHM = "AWS4-HMAC-SHA256"; +const std::regex MULTI_SPACE_REGEX("( +)"); + +std::string minio::signer::GetScope(utils::Time& time, std::string_view region, + std::string_view service_name) { + return time.ToSignerDate() + "/" + std::string(region) + "/" + + std::string(service_name) + "/aws4_request"; +} + +std::string minio::signer::GetCanonicalRequestHash( + std::string_view method, std::string_view uri, + std::string_view query_string, std::string_view headers, + std::string_view signed_headers, std::string_view content_sha256) { + // CanonicalRequest = + // HTTPRequestMethod + '\n' + + // CanonicalURI + '\n' + + // CanonicalQueryString + '\n' + + // CanonicalHeaders + '\n\n' + + // SignedHeaders + '\n' + + // HexEncode(Hash(RequestPayload)) + std::string canonical_request = + std::string(method) + "\n" + std::string(uri) + "\n" + + std::string(query_string) + "\n" + std::string(headers) + "\n\n" + + std::string(signed_headers) + "\n" + std::string(content_sha256); + return utils::Sha256Hash(canonical_request); +} + +std::string minio::signer::GetStringToSign( + utils::Time& date, std::string_view scope, + std::string_view canonical_request_hash) { + return "AWS4-HMAC-SHA256\n" + date.ToAmzDate() + "\n" + std::string(scope) + + "\n" + std::string(canonical_request_hash); +} + +std::string minio::signer::HmacHash(std::string_view key, + std::string_view data) { + std::array hash; + unsigned int hash_len; + + HMAC(EVP_sha256(), key.data(), static_cast(key.size()), + reinterpret_cast(data.data()), + static_cast(data.size()), hash.data(), &hash_len); + + return std::string{reinterpret_cast(hash.data()), hash_len}; +} + +std::string minio::signer::GetSigningKey(std::string_view secret_key, + utils::Time& date, + std::string_view region, + std::string_view service_name) { + std::string date_key = + HmacHash("AWS4" + std::string(secret_key), date.ToSignerDate()); + std::string date_regionkey = HmacHash(date_key, region); + std::string date_regionservice_key = HmacHash(date_regionkey, service_name); + return HmacHash(date_regionservice_key, "aws4_request"); +} + +std::string minio::signer::GetSignature(std::string_view signing_key, + std::string_view string_to_sign) { + std::string hash = HmacHash(signing_key, string_to_sign); + std::string signature; + char buf[3]; + for (int i = 0; i < hash.size(); ++i) { + sprintf(buf, "%02x", (unsigned char)hash[i]); + signature += buf; + } + return signature; +} + +std::string minio::signer::GetAuthorization(std::string_view access_key, + std::string_view scope, + std::string_view signed_headers, + std::string_view signature) { + return "AWS4-HMAC-SHA256 Credential=" + std::string(access_key) + "/" + + std::string(scope) + ", " + + "SignedHeaders=" + std::string(signed_headers) + ", " + + "Signature=" + std::string(signature); +} + +minio::utils::Multimap& minio::signer::SignV4( + std::string_view service_name, http::Method& method, std::string_view uri, + std::string_view region, utils::Multimap& headers, + utils::Multimap& query_params, std::string_view access_key, + std::string_view secret_key, std::string_view content_sha256, + utils::Time& date) { + std::string scope = GetScope(date, region, service_name); + + std::string signed_headers; + std::string canonical_headers; + headers.GetCanonicalHeaders(signed_headers, canonical_headers); + + std::string canonical_query_string = query_params.GetCanonicalQueryString(); + + std::string canonical_request_hash = GetCanonicalRequestHash( + http::MethodToString(method), uri, canonical_query_string, + canonical_headers, signed_headers, content_sha256); + + std::string string_to_sign = + GetStringToSign(date, scope, canonical_request_hash); + + std::string signing_key = + GetSigningKey(secret_key, date, region, service_name); + + std::string signature = GetSignature(signing_key, string_to_sign); + + std::string authorization = + GetAuthorization(access_key, scope, signed_headers, signature); + + headers.Add("Authorization", authorization); + return headers; +} + +minio::utils::Multimap& minio::signer::SignV4S3( + http::Method& method, std::string_view uri, std::string_view region, + utils::Multimap& headers, utils::Multimap& query_params, + std::string_view access_key, std::string_view secret_key, + std::string_view content_sha256, utils::Time& date) { + return SignV4("s3", method, uri, region, headers, query_params, access_key, + secret_key, content_sha256, date); +} + +minio::utils::Multimap& minio::signer::SignV4STS( + http::Method& method, std::string_view uri, std::string_view region, + utils::Multimap& headers, utils::Multimap& query_params, + std::string_view access_key, std::string_view secret_key, + std::string_view content_sha256, utils::Time& date) { + return SignV4("sts", method, uri, region, headers, query_params, access_key, + secret_key, content_sha256, date); +} diff --git a/src/sse.cc b/src/sse.cc new file mode 100644 index 00000000..edf0990a --- /dev/null +++ b/src/sse.cc @@ -0,0 +1,45 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sse.h" + +minio::s3::SseCustomerKey::SseCustomerKey(std::string_view key) { + std::string b64key = utils::Base64Encode(key); + std::string md5key = utils::Md5sumHash(key); + + headers.Add("X-Amz-Server-Side-Encryption-Customer-Algorithm", "AES256"); + headers.Add("X-Amz-Server-Side-Encryption-Customer-Key", b64key); + headers.Add("X-Amz-Server-Side-Encryption-Customer-Key-MD5", md5key); + + copy_headers.Add( + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm", "AES256"); + copy_headers.Add("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key", + b64key); + copy_headers.Add("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5", + md5key); +} + +minio::s3::SseKms::SseKms(std::string_view key, std::string_view context) { + headers.Add("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", std::string(key)); + headers.Add("X-Amz-Server-Side-Encryption", "aws:kms"); + if (!context.empty()) { + headers.Add("X-Amz-Server-Side-Encryption-Context", + utils::Base64Encode(context)); + } +} + +minio::s3::SseS3::SseS3() { + headers.Add("X-Amz-Server-Side-Encryption", "AES256"); +} diff --git a/src/types.cc b/src/types.cc new file mode 100644 index 00000000..81dfa81a --- /dev/null +++ b/src/types.cc @@ -0,0 +1,51 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "types.h" + +minio::s3::RetentionMode minio::s3::StringToRetentionMode( + std::string_view str) throw() { + if (str == "GOVERNANCE") return RetentionMode::kGovernance; + if (str == "COMPLIANCE") return RetentionMode::kCompliance; + + std::cerr << "ABORT: Unknown retention mode. This should not happen." + << std::endl; + std::terminate(); + + return RetentionMode::kGovernance; // never reaches here. +} + +minio::s3::LegalHold minio::s3::StringToLegalHold( + std::string_view str) throw() { + if (str == "ON") return LegalHold::kOn; + if (str == "OFF") return LegalHold::kOff; + + std::cerr << "ABORT: Unknown legal hold. This should not happen." + << std::endl; + std::terminate(); + + return LegalHold::kOff; // never reaches here. +} + +minio::s3::Directive minio::s3::StringToDirective( + std::string_view str) throw() { + if (str == "COPY") return Directive::kCopy; + if (str == "REPLACE") return Directive::kReplace; + + std::cerr << "ABORT: Unknown directive. This should not happen." << std::endl; + std::terminate(); + + return Directive::kCopy; // never reaches here. +} diff --git a/src/utils.cc b/src/utils.cc new file mode 100644 index 00000000..2f0314f1 --- /dev/null +++ b/src/utils.cc @@ -0,0 +1,514 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils.h" + +#include + +const char* HTTP_HEADER_FORMAT = "%a, %d %b %Y %H:%M:%S GMT"; +const std::regex MULTI_SPACE_REGEX("( +)"); +const std::regex VALID_BUCKET_NAME_REGEX( + "^[A-Za-z0-9][A-Za-z0-9\\.\\-\\_\\:]{1,61}[A-Za-z0-9]$"); +const std::regex VALID_BUCKET_NAME_STRICT_REGEX( + "^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$"); +const std::regex VALID_IP_ADDR_REGEX("^(\\d+\\.){3}\\d+$"); + +bool minio::utils::StringToBool(std::string_view str) { + std::string s = ToLower(std::string(str)); + if (s == "false") return false; + if (s == "true") return true; + + std::cerr << "ABORT: Unknown bool string. This should not happen." + << std::endl; + std::terminate(); + + return false; +} + +std::string minio::utils::Trim(std::string_view str, char ch) { + int start, len; + for (start = 0; start < str.size() && str[start] == ch; start++) + ; + for (len = str.size() - start; len > 0 && str[start + len - 1] == ch; len--) + ; + return std::string(str.substr(start, len)); +} + +bool minio::utils::CheckNonEmptyString(std::string_view str) { + return !str.empty() && Trim(str) == str; +} + +std::string minio::utils::ToLower(std::string str) { + std::string s = str; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + return s; +} + +bool minio::utils::StartsWith(std::string_view str, std::string_view prefix) { + return (str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0); +} + +bool minio::utils::EndsWith(std::string_view str, std::string_view suffix) { + return (str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0); +} + +bool minio::utils::Contains(std::string_view str, char ch) { + return str.find(ch) != std::string::npos; +} + +bool minio::utils::Contains(std::string_view str, std::string_view substr) { + return str.find(substr) != std::string::npos; +} + +std::string minio::utils::Join(std::list values, + std::string delimiter) { + std::string result; + for (const auto& value : values) { + if (!result.empty()) result += delimiter; + result += value; + } + return result; +} + +std::string minio::utils::Join(std::vector values, + std::string delimiter) { + std::string result; + for (const auto& value : values) { + if (!result.empty()) result += delimiter; + result += value; + } + return result; +} + +std::string minio::utils::EncodePath(std::string_view path) { + std::stringstream str_stream{std::string(path)}; + std::string token; + std::string out; + while (std::getline(str_stream, token, '/')) { + if (!token.empty()) { + if (!out.empty()) out += "/"; + out += curlpp::escape(token); + } + } + + if (*(path.begin()) == '/') out = "/" + out; + if (*(path.end() - 1) == '/' && out != "/") out += "/"; + + return out; +} + +std::string minio::utils::Sha256Hash(std::string_view str) { + EVP_MD_CTX* ctx = EVP_MD_CTX_create(); + if (ctx == NULL) { + std::cerr << "failed to create EVP_MD_CTX" << std::endl; + std::terminate(); + } + + if (1 != EVP_DigestInit_ex(ctx, EVP_sha256(), NULL)) { + std::cerr << "failed to init SHA-256 digest" << std::endl; + std::terminate(); + } + + if (1 != EVP_DigestUpdate(ctx, str.data(), str.size())) { + std::cerr << "failed to update digest" << std::endl; + std::terminate(); + } + + unsigned int length = EVP_MD_size(EVP_sha256()); + unsigned char* digest = (unsigned char*)OPENSSL_malloc(length); + if (digest == NULL) { + std::cerr << "failed to allocate memory for hash value" << std::endl; + std::terminate(); + } + + if (1 != EVP_DigestFinal_ex(ctx, digest, &length)) { + OPENSSL_free(digest); + std::cerr << "failed to finalize digest" << std::endl; + std::terminate(); + } + + EVP_MD_CTX_destroy(ctx); + + std::string hash; + char buf[3]; + for (int i = 0; i < length; ++i) { + sprintf(buf, "%02x", digest[i]); + hash += buf; + } + + OPENSSL_free(digest); + + return hash; +} + +std::string minio::utils::Base64Encode(std::string_view str) { + const auto base64_memory = BIO_new(BIO_s_mem()); + auto base64 = BIO_new(BIO_f_base64()); + base64 = BIO_push(base64, base64_memory); + + BIO_write(base64, str.data(), str.size()); + BIO_flush(base64); + + BUF_MEM* buf_mem{}; + BIO_get_mem_ptr(base64, &buf_mem); + auto base64_encoded = std::string(buf_mem->data, buf_mem->length - 1); + + BIO_free_all(base64); + + return base64_encoded; +} + +std::string minio::utils::Md5sumHash(std::string_view str) { + EVP_MD_CTX* ctx = EVP_MD_CTX_create(); + if (ctx == NULL) { + std::cerr << "failed to create EVP_MD_CTX" << std::endl; + std::terminate(); + } + + if (1 != EVP_DigestInit_ex(ctx, EVP_md5(), NULL)) { + std::cerr << "failed to init MD5 digest" << std::endl; + std::terminate(); + } + + if (1 != EVP_DigestUpdate(ctx, str.data(), str.size())) { + std::cerr << "failed to update digest" << std::endl; + std::terminate(); + } + + unsigned int length = EVP_MD_size(EVP_md5()); + unsigned char* digest = (unsigned char*)OPENSSL_malloc(length); + if (digest == NULL) { + std::cerr << "failed to allocate memory for hash value" << std::endl; + std::terminate(); + } + + if (1 != EVP_DigestFinal_ex(ctx, digest, &length)) { + OPENSSL_free(digest); + std::cerr << "failed to finalize digest" << std::endl; + std::terminate(); + } + + EVP_MD_CTX_destroy(ctx); + + std::string hash = std::string((const char*)digest, length); + OPENSSL_free(digest); + + return minio::utils::Base64Encode(hash); +} + +std::string minio::utils::FormatTime(const std::tm* time, const char* format) { + char buf[128]; + std::strftime(buf, 128, format, time); + return std::string(buf); +} + +minio::utils::Time::Time() { + tv_.tv_sec = 0; + tv_.tv_usec = 0; +} + +minio::utils::Time::Time(std::time_t tv_sec, suseconds_t tv_usec, bool utc) { + tv_.tv_sec = tv_sec; + tv_.tv_usec = tv_usec; + utc_ = utc; +} + +std::tm* minio::utils::Time::ToUTC() { + std::tm* t = new std::tm; + *t = utc_ ? *std::localtime(&tv_.tv_sec) : *std::gmtime(&tv_.tv_sec); + return t; +} + +std::string minio::utils::Time::ToSignerDate() { + std::tm* utc = ToUTC(); + std::string result = FormatTime(utc, "%Y%m%d"); + delete utc; + return result; +} + +std::string minio::utils::Time::ToAmzDate() { + std::tm* utc = ToUTC(); + std::string result = FormatTime(utc, "%Y%m%dT%H%M%SZ"); + delete utc; + return result; +} + +std::string minio::utils::Time::ToHttpHeaderValue() { + std::tm* utc = ToUTC(); + std::locale("C"); + std::string result = FormatTime(utc, HTTP_HEADER_FORMAT); + std::locale(""); + delete utc; + return result; +} + +minio::utils::Time minio::utils::Time::FromHttpHeaderValue(const char* value) { + std::tm t{0}; + std::locale("C"); + strptime(value, HTTP_HEADER_FORMAT, &t); + std::locale(""); + std::time_t time = std::mktime(&t); + return Time(time, 0, true); +} + +std::string minio::utils::Time::ToISO8601UTC() { + char buf[64]; + snprintf(buf, 64, "%03d", tv_.tv_usec); + std::string usec_str(buf); + if (usec_str.size() > 3) usec_str = usec_str.substr(0, 3); + std::tm* utc = ToUTC(); + std::string result = FormatTime(utc, "%Y-%m-%dT%H:%M:%S.") + usec_str + "Z"; + delete utc; + return result; +} + +minio::utils::Time minio::utils::Time::FromISO8601UTC(const char* value) { + std::tm t{0}; + suseconds_t tv_usec = 0; + char* rv = strptime(value, "%Y-%m-%dT%H:%M:%S", &t); + sscanf(rv, ".%u", &tv_usec); + std::time_t time = std::mktime(&t); + return Time(time, tv_usec, true); +} + +minio::utils::Time minio::utils::Time::Now() { + Time t; + gettimeofday(&t.tv_, NULL); + t.utc_ = false; + return t; +} + +minio::utils::Multimap::Multimap() {} + +minio::utils::Multimap::Multimap(const Multimap& headers) { + this->AddAll(headers); +} + +void minio::utils::Multimap::Add(std::string key, std::string value) { + map_[key].insert(value); + keys_[ToLower(key)].insert(key); +} + +void minio::utils::Multimap::AddAll(const Multimap& headers) { + auto m = headers.map_; + for (auto& [key, values] : m) { + map_[key].insert(values.begin(), values.end()); + keys_[ToLower(key)].insert(key); + } +} + +std::list minio::utils::Multimap::ToHttpHeaders() { + std::list headers; + for (auto& [key, values] : map_) { + for (auto& value : values) { + headers.push_back(key + ": " + value); + } + } + return headers; +} + +std::string minio::utils::Multimap::ToQueryString() { + std::string query_string; + for (auto& [key, values] : map_) { + for (auto& value : values) { + std::string s = curlpp::escape(key) + "=" + curlpp::escape(value); + if (!query_string.empty()) query_string += "&"; + query_string += s; + } + } + return query_string; +} + +bool minio::utils::Multimap::Contains(std::string_view key) { + return keys_.find(ToLower(std::string(key))) != keys_.end(); +} + +std::list minio::utils::Multimap::Get(std::string_view key) { + std::list result; + std::set keys = keys_[ToLower(std::string(key))]; + for (auto& key : keys) { + std::set values = map_[key]; + result.insert(result.end(), values.begin(), values.end()); + } + return result; +} + +std::string minio::utils::Multimap::GetFront(std::string_view key) { + std::list values = Get(key); + return (values.size() > 0) ? values.front() : ""; +} + +std::list minio::utils::Multimap::Keys() { + std::list keys; + for (const auto& [key, _] : keys_) keys.push_back(key); + return keys; +} + +void minio::utils::Multimap::GetCanonicalHeaders( + std::string& signed_headers, std::string& canonical_headers) { + std::vector signed_headerslist; + std::map map; + + for (auto& [k, values] : map_) { + std::string key = ToLower(k); + if ("authorization" == key || "user-agent" == key) continue; + if (std::find(signed_headerslist.begin(), signed_headerslist.end(), key) == + signed_headerslist.end()) { + signed_headerslist.push_back(key); + } + + std::string value; + for (auto& v : values) { + if (!value.empty()) value += ","; + value += std::regex_replace(v, MULTI_SPACE_REGEX, " "); + } + + map[key] = value; + } + + std::sort(signed_headerslist.begin(), signed_headerslist.end()); + signed_headers = utils::Join(signed_headerslist, ";"); + + std::vector canonical_headerslist; + for (auto& [key, value] : map) { + canonical_headerslist.push_back(key + ":" + value); + } + + std::sort(canonical_headerslist.begin(), canonical_headerslist.end()); + canonical_headers = utils::Join(canonical_headerslist, "\n"); +} + +std::string minio::utils::Multimap::GetCanonicalQueryString() { + std::vector values; + for (auto& [key, vals] : map_) { + for (auto& value : vals) { + std::string s = curlpp::escape(key) + "=" + curlpp::escape(value); + values.push_back(s); + } + } + + std::sort(values.begin(), values.end()); + return utils::Join(values, "&"); +} + +std::string minio::utils::Url::String() { + if (host.empty()) return ""; + + std::string url = (is_https ? "https://" : "http://") + host; + + if (!path.empty()) { + if (*(path.begin()) != '/') url += "/"; + url += path; + } + + if (!query_string.empty()) url += "?" + query_string; + + return url; +} + +minio::error::Error minio::utils::CheckBucketName(std::string_view bucket_name, + bool strict) { + if (Trim(bucket_name).empty()) { + return error::Error("bucket name cannot be empty"); + } + + if (bucket_name.length() < 3) { + return error::Error("bucket name cannot be less than 3 characters"); + } + + if (bucket_name.length() > 63) { + return error::Error("Bucket name cannot be greater than 63 characters"); + } + + if (std::regex_match(bucket_name.data(), VALID_IP_ADDR_REGEX)) { + return error::Error("bucket name cannot be an IP address"); + } + + // unallowed successive characters check. + if (Contains(bucket_name, "..") || Contains(bucket_name, ".-") || + Contains(bucket_name, "-.")) { + return error::Error( + "Bucket name contains invalid successive characters '..', '.-' or " + "'-.'"); + } + + if (strict) { + if (!std::regex_match(bucket_name.data(), VALID_BUCKET_NAME_STRICT_REGEX)) { + return error::Error("bucket name does not follow S3 standards strictly"); + } + } else if (!std::regex_match(bucket_name.data(), VALID_BUCKET_NAME_REGEX)) { + return error::Error("bucket name does not follow S3 standards"); + } + + return error::SUCCESS; +} + +minio::error::Error minio::utils::ReadPart(std::istream& stream, char* buf, + size_t size, size_t& bytes_read) { + stream.read(buf, size); + bytes_read = stream.gcount(); + return minio::error::SUCCESS; +} + +minio::error::Error minio::utils::CalcPartInfo(long object_size, + size_t& part_size, + long& part_count) { + if (part_size > 0) { + if (part_size < kMinPartSize) { + return error::Error("part size " + std::to_string(part_size) + + " is not supported; minimum allowed 5MiB"); + } + + if (part_size > kMaxPartSize) { + return error::Error("part size " + std::to_string(part_size) + + " is not supported; maximum allowed 5GiB"); + } + } + + if (object_size >= 0) { + if (object_size > kMaxObjectSize) { + return error::Error("object size " + std::to_string(object_size) + + " is not supported; maximum allowed 5TiB"); + } + } else if (part_size <= 0) { + return error::Error( + "valid part size must be provided when object size is unknown"); + } + + if (object_size < 0) { + part_count = -1; + return error::SUCCESS; + } + + if (part_size <= 0) { + // Calculate part size by multiple of kMinPartSize. + double psize = std::ceil((double)object_size / kMaxMultipartCount); + part_size = (size_t)std::ceil(psize / kMinPartSize) * kMinPartSize; + } + + if (part_size > object_size) part_size = object_size; + part_count = + part_size > 0 ? (long)std::ceil((double)object_size / part_size) : 1; + if (part_count > kMaxMultipartCount) { + return error::Error( + "object size " + std::to_string(object_size) + " and part size " + + std::to_string(part_size) + " make more than " + + std::to_string(kMaxMultipartCount) + "parts for upload"); + } + + return error::SUCCESS; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000..44d14397 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +ADD_EXECUTABLE(tests tests.cc) +TARGET_LINK_LIBRARIES(tests miniocpp ${requiredlibs}) diff --git a/tests/private.key b/tests/private.key new file mode 100644 index 00000000..ffc07f0b --- /dev/null +++ b/tests/private.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAupkiVeaM9e9fj/xtV9O2OB6U0mUf0wuZRwGIdmMXnDr7rkBw +WAoI120gZkUF7j0WSMvJlwGuposLLBoEpTCmh+mVf7PNlYgkn+E49VvaJ9Q7w/Gz +YAlbNGmANg5mzsnkskMsh9+UyyGmdgvX1EQK4sQ3Hq7FIyzL/xm6y9qNiGvseLwU +hKlBF5X7O5wNY25tH2KCpXMY57w5pCGAQnTKjb3hwwy18wrlyquGjC5TPtbPIeUo +VQIcaQs2N8Z3HzrwBgNSVlYxLDFHfD0OPivPCSlSBh9x2Zi9HsYzReCrO6HCDakC +ZO0WNGVCXNdCZwoHZGOW5x7his4+A3at02uRawIDAQABAoIBAElz2mZCGR7+mXmO +fmRiPIqezyp7ECn9mNqwqc0geLzRIx2W1CJz4MMce/KGHS2I8mq5faNp0BxTA5Ta +sRVtr0A1HNpmJvlD3FbrS4aaH6gqDVS2okudoz9ggE3HIYUpSFM7yh26T1Ie7u3s +/4rZNgfKAYCcf5G3Ip5KvJNedvRKC2w5UX4iYU5eMuXs0YPy2+DtTKxZKIEnI3x4 +VpMe652I7GhjkLSsyuW5NoDeSWmmqhitm1bibuZQBR0ogFyAU94rZk8+j4hiUl9k +7ZVi00tKNp7EcLIwbY02ZCpZ2xvat8W1SLeWRUpExPj/7nnA9Fafrl5M8aHgl8R1 +N0Fd+zECgYEA4GruRDPPNPEXPByes6jjcxfvdoYIDyma1V+FN5Nn+/QPL/vWp9t0 ++mH344iwWkEyZZHNUpNuy5dpOA9724iFFLp9Btq06OgzzDuJj1LDbDpB4zwltuQA +4s7ekTFi9rHDOeAyTREHgDu3uya/2qq53Ms6w8gXM4/kmp9/S4N5VhcCgYEA1Nuv +J1khBitNiNbzHYG9DRV4HLfIB8FFDKtgnW8HvSaWUtHKI4YitwJVFfa5Epz9MDFy +tkFAD9Bu0HqZQ6OQZxGo0rDUcFics9Ftw+w3p/PIocPQBK7PNQ3L9BQS9M3stL+k +fW5r+kUvfZaJVE8agCQQnzkJJh9oexJjMWyxB80CgYBa5BQSPWWLjKWba//+xcUx +BR2wREKZWYFjL+e1hZcU3VkVVwsuOtza17jdR6wdMdCmgHHHIv05qd4snWDNnjJA +HfOrRgMFXZ409lwVVzDc8Y9j6CViOF//fEd6SKVLQt3N3/afbek6z3TvcJc9ie3y +9cCcMLrs4Dd3RGf6/omzCwKBgQChwWxCf6Xr9T5PjeFke/I5niYP1M2KryGU9itO +mFCOOmOj/k8ZXdbFsl0Meti7v1dcp0cgH0fafK+peHE+CG81FCNyMPTPh1dWAwHi +EIFe/ZBq9c3/sQQ/sgNasWKSbGbEGJqcwywFHUxwqNQloJNn64BCL2q3cMjKNffx +WELTxQKBgQDe7adrUBtk8+YZ1/c2VsV1oBq6eI2U+1bWa/8fwXWd9ylETBMtcv20 +Fs8UAFWLz78aWaXWWSSEmFvUbjXPBDvzXCObb6HdhXOCF1n74hKU+z06MAUa1eFC +V0GvBx4v1rc1pZ7grBZwGteeVWVgCAZ487m5TATWqxuarH6yfU0Ptg== +-----END RSA PRIVATE KEY----- diff --git a/tests/public.crt b/tests/public.crt new file mode 100644 index 00000000..eefae856 --- /dev/null +++ b/tests/public.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDojCCAoqgAwIBAgIUHE3HUt7gKVBp7wT9Hjj4wRT1xywwDQYJKoZIhvcNAQEL +BQAwejELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRgwFgYDVQQHDA9jdXN0b20t +bG9jYXRpb24xHDAaBgNVBAoME2N1c3RvbS1vcmdhbml6YXRpb24xEjAQBgNVBAsM +CWN1c3RvbS1vdTESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIxMDkyMDEyMTAyOFoY +DzIxMjEwODI3MTIxMDI4WjB6MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExGDAW +BgNVBAcMD2N1c3RvbS1sb2NhdGlvbjEcMBoGA1UECgwTY3VzdG9tLW9yZ2FuaXph +dGlvbjESMBAGA1UECwwJY3VzdG9tLW91MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6mSJV5oz171+P/G1X07Y4HpTS +ZR/TC5lHAYh2YxecOvuuQHBYCgjXbSBmRQXuPRZIy8mXAa6miwssGgSlMKaH6ZV/ +s82ViCSf4Tj1W9on1DvD8bNgCVs0aYA2DmbOyeSyQyyH35TLIaZ2C9fURArixDce +rsUjLMv/GbrL2o2Ia+x4vBSEqUEXlfs7nA1jbm0fYoKlcxjnvDmkIYBCdMqNveHD +DLXzCuXKq4aMLlM+1s8h5ShVAhxpCzY3xncfOvAGA1JWVjEsMUd8PQ4+K88JKVIG +H3HZmL0exjNF4Ks7ocINqQJk7RY0ZUJc10JnCgdkY5bnHuGKzj4Ddq3Ta5FrAgMB +AAGjHjAcMBoGA1UdEQQTMBGHBH8AAAGCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsF +AAOCAQEAREoEGX34C0+osOgCPwZ4mFXVvssJRJTc4JL/kqOGPyzS/085MzeIpOhu +gkOm3zj175SKiZMWWpYhB+Q/1T6tDiVLidEY7HK/dcbjYfoTAd42cQWY4qmG68vG +E6WnTQjal0uPuCwKqvqIRgkWLpaIV4TmHtpCFLLlVlnDWQBpAdiK/Vk0EeTDDaqd +cROr4tm/8EBxqwUnIQF8vgbXgVT5BNdcp44LWs7A558CCRrCGifch5+kxkSSdAFG +Y7BEXvnnsxU1n9AAVKJYnp1wUN2Hk4KWyPcQb9/Ee45DKDR5KNyMqClsWWXMJr40 +FrgI6euT50Wzo8Qqbls5Bt/0IzObnA== +-----END CERTIFICATE----- diff --git a/tests/tests.cc b/tests/tests.cc new file mode 100644 index 00000000..9f1f5a6e --- /dev/null +++ b/tests/tests.cc @@ -0,0 +1,516 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "client.h" + +thread_local static std::mt19937 rg{std::random_device{}()}; + +const static std::string charset = + "0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + +thread_local static std::uniform_int_distribution pick( + 0, charset.length() - 2); + +class RandomBuf : public std::streambuf { + private: + size_t size_; + std::array buf_; + + protected: + int_type underflow() override { + if (size_ == 0) return EOF; + + size_t size = std::min(size_, buf_.size()); + setg(&buf_[0], &buf_[0], &buf_[size]); + for (size_t i = 0; i < size; ++i) buf_[i] = charset[pick(rg)]; + size_ -= size; + return 0; + } + + public: + RandomBuf(size_t size) : size_(size) {} +}; + +class RandCharStream : public std::istream { + private: + RandomBuf buf_; + + public: + RandCharStream(size_t size) : buf_(size) { rdbuf(&buf_); } +}; + +std::string RandomString(std::string chrs, std::string::size_type length) { + thread_local static std::uniform_int_distribution + pick(0, chrs.length() - 2); + + std::string s; + s.reserve(length); + while (length--) s += chrs[pick(rg)]; + return s; +} + +std::string RandBucketName() { + return RandomString("0123456789abcdefghijklmnopqrstuvwxyz", 8); +} + +std::string RandObjectName() { return RandomString(charset, 8); } + +struct MakeBucketError : public std::runtime_error { + MakeBucketError(std::string err) : runtime_error(err) {} +}; + +struct RemoveBucketError : public std::runtime_error { + RemoveBucketError(std::string err) : runtime_error(err) {} +}; + +struct BucketExistsError : public std::runtime_error { + BucketExistsError(std::string err) : runtime_error(err) {} +}; + +class Tests { + private: + minio::s3::Client& client_; + std::string bucket_name_; + + public: + Tests(minio::s3::Client& client) : client_(client) { + bucket_name_ = RandBucketName(); + minio::s3::MakeBucketArgs args; + args.bucket = bucket_name_; + minio::s3::MakeBucketResponse resp = client_.MakeBucket(args); + if (!resp) throw std::runtime_error("MakeBucket(): " + resp.GetError()); + } + + ~Tests() noexcept(false) { + minio::s3::RemoveBucketArgs args; + args.bucket = bucket_name_; + minio::s3::RemoveBucketResponse resp = client_.RemoveBucket(args); + if (!resp) throw std::runtime_error("RemoveBucket(): " + resp.GetError()); + } + + void MakeBucket(std::string bucket_name) noexcept(false) { + minio::s3::MakeBucketArgs args; + args.bucket = bucket_name; + minio::s3::MakeBucketResponse resp = client_.MakeBucket(args); + if (resp) return; + throw MakeBucketError("MakeBucket(): " + resp.GetError()); + } + + void RemoveBucket(std::string bucket_name) noexcept(false) { + minio::s3::RemoveBucketArgs args; + args.bucket = bucket_name; + minio::s3::RemoveBucketResponse resp = client_.RemoveBucket(args); + if (resp) return; + throw RemoveBucketError("RemoveBucket(): " + resp.GetError()); + } + + void RemoveObject(std::string bucket_name, std::string object_name) { + minio::s3::RemoveObjectArgs args; + args.bucket = bucket_name; + args.object = object_name; + minio::s3::RemoveObjectResponse resp = client_.RemoveObject(args); + if (!resp) throw std::runtime_error("RemoveObject(): " + resp.GetError()); + } + + void MakeBucket() { + std::cout << "MakeBucket()" << std::endl; + + std::string bucket_name = RandBucketName(); + MakeBucket(bucket_name); + RemoveBucket(bucket_name); + } + + void RemoveBucket() { + std::cout << "RemoveBucket()" << std::endl; + + std::string bucket_name = RandBucketName(); + MakeBucket(bucket_name); + RemoveBucket(bucket_name); + } + + void BucketExists() { + std::cout << "BucketExists()" << std::endl; + + std::string bucket_name = RandBucketName(); + try { + MakeBucket(bucket_name); + minio::s3::BucketExistsArgs args; + args.bucket = bucket_name; + minio::s3::BucketExistsResponse resp = client_.BucketExists(args); + if (!resp) { + throw BucketExistsError("BucketExists(): " + resp.GetError()); + } + if (!resp.exist) { + throw std::runtime_error("BucketExists(): expected: true; got: false"); + } + RemoveBucket(bucket_name); + } catch (const MakeBucketError& err) { + throw err; + } catch (const std::runtime_error& err) { + RemoveBucket(bucket_name); + throw err; + } + } + + void ListBuckets() { + std::cout << "ListBuckets()" << std::endl; + + std::list bucket_names; + try { + for (int i = 0; i < 3; i++) { + std::string bucket_name = RandBucketName(); + MakeBucket(bucket_name); + bucket_names.push_back(bucket_name); + } + + minio::s3::ListBucketsResponse resp = client_.ListBuckets(); + if (!resp) throw std::runtime_error("ListBuckets(): " + resp.GetError()); + + int c = 0; + for (auto& bucket : resp.buckets) { + if (std::find(bucket_names.begin(), bucket_names.end(), bucket.name) != + bucket_names.end()) { + c++; + } + } + if (c != bucket_names.size()) { + throw std::runtime_error( + "ListBuckets(): expected: " + std::to_string(bucket_names.size()) + + "; got: " + std::to_string(c)); + } + for (auto& bucket_name : bucket_names) RemoveBucket(bucket_name); + } catch (const std::runtime_error& err) { + for (auto& bucket_name : bucket_names) RemoveBucket(bucket_name); + throw err; + } + } + + void StatObject() { + std::cout << "StatObject()" << std::endl; + + std::string object_name = RandObjectName(); + + std::string data = "StatObject()"; + std::stringstream ss(data); + minio::s3::PutObjectArgs args(ss, data.length(), 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + try { + minio::s3::StatObjectArgs args; + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::StatObjectResponse resp = client_.StatObject(args); + if (!resp) throw std::runtime_error("StatObject(): " + resp.GetError()); + if (resp.size != data.length()) { + throw std::runtime_error( + "StatObject(): expected: " + std::to_string(data.length()) + + "; got: " + std::to_string(resp.size)); + } + RemoveObject(bucket_name_, object_name); + } catch (const std::runtime_error& err) { + RemoveObject(bucket_name_, object_name); + throw err; + } + } + + void RemoveObject() { + std::cout << "RemoveObject()" << std::endl; + + std::string object_name = RandObjectName(); + std::string data = "RemoveObject()"; + std::stringstream ss(data); + minio::s3::PutObjectArgs args(ss, data.length(), 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + RemoveObject(bucket_name_, object_name); + } + + void DownloadObject() { + std::cout << "DownloadObject()" << std::endl; + + std::string object_name = RandObjectName(); + + std::string data = "DownloadObject()"; + std::stringstream ss(data); + minio::s3::PutObjectArgs args(ss, data.length(), 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + + try { + std::string filename = RandObjectName(); + minio::s3::DownloadObjectArgs args; + args.bucket = bucket_name_; + args.object = object_name; + args.filename = filename; + minio::s3::DownloadObjectResponse resp = client_.DownloadObject(args); + if (!resp) { + throw std::runtime_error("DownloadObject(): " + resp.GetError()); + } + + std::ifstream file(filename); + file.seekg(0, std::ios::end); + size_t length = file.tellg(); + file.seekg(0, std::ios::beg); + char* buf = new char[length]; + file.read(buf, length); + file.close(); + + if (data != buf) { + throw std::runtime_error("DownloadObject(): expected: " + data + + "; got: " + buf); + } + std::filesystem::remove(filename); + RemoveObject(bucket_name_, object_name); + } catch (const std::runtime_error& err) { + RemoveObject(bucket_name_, object_name); + throw err; + } + } + + void GetObject() { + std::cout << "GetObject()" << std::endl; + + std::string object_name = RandObjectName(); + + std::string data = "GetObject()"; + std::stringstream ss(data); + minio::s3::PutObjectArgs args(ss, data.length(), 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + + try { + minio::s3::GetObjectArgs args; + args.bucket = bucket_name_; + args.object = object_name; + std::string content; + args.data_callback = [](minio::http::DataCallbackArgs args) -> size_t { + std::string* content = (std::string*)args.user_arg; + *content += std::string(args.buffer, args.length); + return args.size * args.length; + }; + args.user_arg = &content; + minio::s3::GetObjectResponse resp = client_.GetObject(args); + if (!resp) throw std::runtime_error("GetObject(): " + resp.GetError()); + if (data != content) { + throw std::runtime_error("GetObject(): expected: " + data + + "; got: " + content); + } + RemoveObject(bucket_name_, object_name); + } catch (const std::runtime_error& err) { + RemoveObject(bucket_name_, object_name); + throw err; + } + } + + void ListObjects() { + std::cout << "ListObjects()" << std::endl; + + std::list object_names; + try { + for (int i = 0; i < 3; i++) { + std::string object_name = RandObjectName(); + std::stringstream ss; + minio::s3::PutObjectArgs args(ss, 0, 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + object_names.push_back(object_name); + } + + int c = 0; + minio::s3::ListObjectsArgs args; + args.bucket = bucket_name_; + minio::s3::ListObjectsResult result = client_.ListObjects(args); + for (; result; result++) { + minio::s3::Item item = *result; + if (!item) { + throw std::runtime_error("ListObjects(): " + item.GetError()); + } + if (std::find(object_names.begin(), object_names.end(), item.name) != + object_names.end()) { + c++; + } + } + + if (c != object_names.size()) { + throw std::runtime_error( + "ListObjects(): expected: " + std::to_string(object_names.size()) + + "; got: " + std::to_string(c)); + } + for (auto& object_name : object_names) { + RemoveObject(bucket_name_, object_name); + } + } catch (const std::runtime_error& err) { + for (auto& object_name : object_names) { + RemoveObject(bucket_name_, object_name); + } + throw err; + } + } + + void PutObject() { + std::cout << "PutObject()" << std::endl; + + { + std::string object_name = RandObjectName(); + std::string data = "PutObject()"; + std::stringstream ss(data); + minio::s3::PutObjectArgs args(ss, data.length(), 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + RemoveObject(bucket_name_, object_name); + } + + { + std::string object_name = RandObjectName(); + size_t size = 13930573; + RandCharStream stream(size); + minio::s3::PutObjectArgs args(stream, size, 0); + args.bucket = bucket_name_; + args.object = object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) + std::runtime_error(" PutObject(): " + resp.GetError()); + RemoveObject(bucket_name_, object_name); + } + } + + void CopyObject() { + std::cout << "CopyObject()" << std::endl; + + std::string object_name = RandObjectName(); + std::string src_object_name = RandObjectName(); + std::string data = "CopyObject()"; + std::stringstream ss(data); + minio::s3::PutObjectArgs args(ss, data.length(), 0); + args.bucket = bucket_name_; + args.object = src_object_name; + minio::s3::PutObjectResponse resp = client_.PutObject(args); + if (!resp) std::runtime_error("PutObject(): " + resp.GetError()); + + try { + minio::s3::CopySource source; + source.bucket = bucket_name_; + source.object = src_object_name; + minio::s3::CopyObjectArgs args; + args.bucket = bucket_name_; + args.object = object_name; + args.source = source; + minio::s3::CopyObjectResponse resp = client_.CopyObject(args); + if (!resp) std::runtime_error("CopyObject(): " + resp.GetError()); + RemoveObject(bucket_name_, src_object_name); + RemoveObject(bucket_name_, object_name); + } catch (const std::runtime_error& err) { + RemoveObject(bucket_name_, src_object_name); + RemoveObject(bucket_name_, object_name); + throw err; + } + } + + void UploadObject() { + std::cout << "UploadObject()" << std::endl; + + std::string data = "UploadObject()"; + std::string filename = RandObjectName(); + std::ofstream file(filename); + file << data; + file.close(); + + std::string object_name = RandObjectName(); + minio::s3::UploadObjectArgs args; + args.bucket = bucket_name_; + args.object = object_name; + args.filename = filename; + minio::s3::UploadObjectResponse resp = client_.UploadObject(args); + if (!resp) std::runtime_error("UploadObject(): " + resp.GetError()); + std::filesystem::remove(filename); + RemoveObject(bucket_name_, object_name); + } +}; // class Tests + +bool GetEnv(std::string& var, const char* name) { + if (const char* value = std::getenv(name)) { + var = value; + return true; + } + return false; +} + +int main(int argc, char* argv[]) { + std::string host; + if (!GetEnv(host, "S3HOST")) { + std::cerr << "S3HOST environment variable must be set" << std::endl; + return EXIT_FAILURE; + } + + std::string access_key; + if (!GetEnv(access_key, "ACCESS_KEY")) { + std::cerr << "ACCESS_KEY environment variable must be set" << std::endl; + return EXIT_FAILURE; + } + + std::string secret_key; + if (!GetEnv(secret_key, "SECRET_KEY")) { + std::cerr << "SECRET_KEY environment variable must be set" << std::endl; + return EXIT_FAILURE; + } + + std::string value; + bool secure = true; + if (GetEnv(value, "IS_HTTP")) secure = false; + + bool ignore_cert_check = false; + if (GetEnv(value, "IGNORE_CERT_CHECK")) secure = true; + + std::string region; + GetEnv(region, "REGION"); + + minio::http::BaseUrl base_url; + base_url.SetHost(host); + base_url.is_https = secure; + minio::creds::StaticProvider provider(access_key, secret_key); + minio::s3::Client client(base_url, &provider); + + Tests tests(client); + tests.MakeBucket(); + tests.RemoveBucket(); + tests.BucketExists(); + tests.ListBuckets(); + tests.StatObject(); + tests.RemoveObject(); + tests.DownloadObject(); + tests.GetObject(); + tests.ListObjects(); + tests.PutObject(); + tests.CopyObject(); + tests.UploadObject(); + + return EXIT_SUCCESS; +}