docs(480-implemend-the-missing-keys): fix partial download/upload

src/core/include/awsmock/core/FileUtils.h

@@ -7,21 +7,14 @@
 
 // Standard C includes
 #include <fcntl.h>
-#ifndef _WIN32
 #include <grp.h>
 #include <pwd.h>
+#include <sys/sendfile.h>
+#include <sys/stat.h>
 #include <sys/time.h>
+#include <sys/types.h>
 #include <unistd.h>
 #include <utime.h>
-#else
-#include "accctrl.h"
-#include "aclapi.h"
-#include <stdio.h>
-#include <tchar.h>
-#include <windows.h>
-#endif
-#include <sys/stat.h>
-#include <sys/types.h>
 
 // Standard C++ includes
 #include <cstdio>
@@ -53,6 +46,8 @@
 #include <awsmock/core/LogStream.h>
 #include <awsmock/core/StringUtils.h>
 
+#define BUFFER_LEN 8092
+
 namespace AwsMock::Core {
 
     /**
@@ -193,7 +188,7 @@ namespace AwsMock::Core {
          * @param inDir input directory
          * @param files string vector of binary files to append to output file
          */
-        static void AppendBinaryFiles(const std::string &outFile, const std::string &inDir, const std::vector<std::string> &files);
+        static long AppendBinaryFiles(const std::string &outFile, const std::string &inDir, const std::vector<std::string> &files);
 
         /**
          * @brief Append several text files to a single output file.

src/core/include/awsmock/core/MemoryMappedFile.h

@@ -6,17 +6,19 @@
 #define AWS_MOCK_CORE_MEMORY_MAPPED_FILE_H
 
 // C includes
-#include <fcntl.h>
-#include <sys/stat.h>
-#ifndef _WIN32
 #include <cerrno>
+#include <fcntl.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
 #include <unistd.h>
-#endif
 
 // C++ includes
 #include <string>
 
+// Boost includes
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/thread.hpp>
+
 // AwsMock includes
 #include <awsmock/core/FileUtils.h>
 #include <awsmock/core/LogStream.h>
@@ -76,24 +78,10 @@ namespace AwsMock::Core {
          * @return true, in case file is already mapped.
          */
         [[nodiscard]] bool IsMapped() const { return _mapped; }
-#ifdef _WIN32
-        /// tweak performance
-        enum CacheHint {
-            Normal,        ///< good overall performance
-            SequentialScan,///< read file only once with few seeks
-            RandomAccess   ///< jump around
-        };
-#endif
 
       private:
 
-#ifdef _WIN32
-        typedef void *FileHandle;
-        /// Windows handle to memory mapping of _file
-        void *_mappedFile;
-#else
         typedef int FileHandle;
-#endif
 
         /**
          * Start pointer
@@ -124,6 +112,11 @@ namespace AwsMock::Core {
          * File size
          */
         long _fileSize = 0;
+
+        /**
+         * Mutex
+         */
+        static boost::mutex _mutex;
     };
 
 }// namespace AwsMock::Core

src/core/src/utils/FileUtils.cpp

@@ -1,4 +1,5 @@
 #include <awsmock/core/FileUtils.h>
+#include <boost/asio/streambuf.hpp>
 
 namespace AwsMock::Core {
 
@@ -66,20 +67,25 @@ namespace AwsMock::Core {
         sourceFile.copyTo(targetFileName);
     }
 
-    void FileUtils::AppendBinaryFiles(const std::string &outFile, const std::string &inDir, const std::vector<std::string> &files) {
-        std::ofstream ofs(outFile, std::ios::out | std::ios::trunc | std::ios::binary | std::ios::app);
+    long FileUtils::AppendBinaryFiles(const std::string &outFile, const std::string &inDir, const std::vector<std::string> &files) {
+
+        int dest = open(outFile.c_str(), O_WRONLY | O_CREAT, 0644);
+
+        size_t copied = 0;
         for (auto &it: files) {
-            std::string inFile = inDir;
-            inFile.append(Poco::Path::separator() + it);
-            std::ifstream ifs(inFile, std::ios::in);
-            long copied = Poco::StreamCopier::copyStream(ifs, ofs);
-            if (copied != Core::FileUtils::FileSize(inFile)) {
-                throw Poco::Exception("Invalid input file");
-            }
-            ofs.flush();
-            ifs.close();
+
+            std::string inFile = inDir + "/" + it;
+            int source = open(inFile.c_str(), O_RDONLY, 0);
+
+            // struct required, rationale: function stat() exists also
+            struct stat stat_source {};
+            fstat(source, &stat_source);
+            copied += sendfile(dest, source, nullptr, stat_source.st_size);
+
+            close(source);
         }
-        ofs.close();
+        close(dest);
+        return static_cast<long>(copied);
     }
 
     // TODO: Calculate correct checksum: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html

src/core/src/utils/MemoryMappedFile.cpp

@@ -6,48 +6,18 @@
 
 namespace AwsMock::Core {
 
-    bool MemoryMappedFile::OpenFile(const std::string &filename) {
-#ifdef _WIN32
-        CacheHint _hint = Normal;
-        DWORD winHint = 0;
-        switch (_hint) {
-            case Normal:
-                winHint = FILE_ATTRIBUTE_NORMAL;
-                break;
-            case SequentialScan:
-                winHint = FILE_FLAG_SEQUENTIAL_SCAN;
-                break;
-            case RandomAccess:
-                winHint = FILE_FLAG_RANDOM_ACCESS;
-                break;
-            default:
-                break;
-        }
+    boost::mutex MemoryMappedFile::_mutex;
 
-        // open file
-        _file = ::CreateFileA(filename.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, winHint, NULL);
-        if (!_file)
-            return false;
+    bool MemoryMappedFile::OpenFile(const std::string &filename) {
 
-        // file size
-        LARGE_INTEGER result;
-        if (!GetFileSizeEx(_file, &result))
-            return false;
-        _fileSize = static_cast<uint64_t>(result.QuadPart);
+        _fileSize = FileUtils::FileSize(filename);
 
-        // convert to mapped mode
-        _mappedFile = ::CreateFileMapping(_file, NULL, PAGE_READONLY, 0, 0, NULL);
-        _mapped = true;
-#else
         int fd = open(filename.c_str(), O_RDONLY);
-
         if (fd < 0) {
             log_error << "Could not open file for memory mapping";
             return false;
         }
 
-        _fileSize = FileUtils::FileSize(filename);
-
         _start = mmap(nullptr, _fileSize, PROT_READ, MAP_SHARED, fd, 0);
         _membuffer = (char *) _start;
 
@@ -59,30 +29,24 @@ namespace AwsMock::Core {
         close(fd);
 
         _mapped = true;
-#endif
         return _mapped;
     }
 
     void MemoryMappedFile::CloseFile() {
-#ifdef _WIN32
-        if (_mappedFile) {
-            ::CloseHandle(_mappedFile);
-            _mappedFile = nullptr;
-            log_debug << "Memory mapped file closed";
-        }
-#else
+        boost::mutex::scoped_lock lock(_mutex);
         if (munmap(_start, _fileSize) < 0) {
             log_error << "Could not unmap file";
         }
         log_debug << "Memory mapped file closed";
-#endif
         _mapped = false;
     }
 
     long MemoryMappedFile::ReadChunk(long start, long length, char *buffer) {
-        if (length > _fileSize) {
-            length = _fileSize;
+        if (start + length > _fileSize) {
+            length = _fileSize - start;
         }
+        boost::mutex::scoped_lock lock(_mutex);
+        log_debug << "start: " << start << " length: " << length << " size: " << _fileSize;
         memcpy(buffer, _membuffer + start, length);
         return length;
     }

src/dto/src/s3/GetObjectRequest.cpp

@@ -17,6 +17,8 @@ namespace AwsMock::Dto::S3 {
             rootJson.set("min", min);
             rootJson.set("max", max);
 
+            return Core::JsonUtils::ToJsonString(rootJson);
+
         } catch (Poco::Exception &exc) {
             log_error << exc.message();
             throw Core::JsonException(exc.message());

src/service/include/awsmock/service/s3/S3Service.h

@@ -89,13 +89,13 @@ namespace AwsMock::Service {
         explicit S3Service() : _database(Database::S3Database::instance()){};
 
         /**
-         * @brief Checks wether a bucket exists
+         * @brief Checks whether a bucket exists
          *
          * @param region AWS region
          * @param bucket bucket name
          * @return true if bucket exists
          */
-        bool BucketExists(const std::string &region, const std::string bucket);
+        bool BucketExists(const std::string &region, const std::string &bucket);
 
         /**
          * @brief Returns the meta data of an S3 bucket
@@ -159,7 +159,7 @@ namespace AwsMock::Service {
          * @param updateId upload ID
          * @return ETag
          */
-        std::string UploadPart(std::istream &stream, int part, const std::string &updateId);
+        static std::string UploadPart(std::istream &stream, int part, const std::string &updateId);
 
         /**
          * @brief Upload a partial file copy.
@@ -271,13 +271,6 @@ namespace AwsMock::Service {
          */
         void DeleteBucket(const Dto::S3::DeleteBucketRequest &request);
 
-        /**
-         * @brief Calculates the MD5, SHA1, SHA256 asynchronously
-         *
-         * @param object S3 object
-         */
-        void CalculateHashes(Database::Entity::S3::Object &object);
-
       private:
 
         /**
@@ -342,10 +335,10 @@ namespace AwsMock::Service {
         /**
          * @brief Get the temporary upload directory for a uploadId.
          *
-         * @param uploadId S3 multipart uplaod ID
+         * @param uploadId S3 multipart upload ID
          * @return temporary directory path.
          */
-        std::string GetMultipartUploadDirectory(const std::string &uploadId);
+        static std::string GetMultipartUploadDirectory(const std::string &uploadId);
 
         /**
          * @brief Create a queue notification
@@ -394,7 +387,7 @@ namespace AwsMock::Service {
         Dto::S3::PutObjectResponse SaveVersionedObject(Dto::S3::PutObjectRequest &request, const Database::Entity::S3::Bucket &bucket, std::istream &stream);
 
         /**
-         * @brief Save a unversioned S3 object.
+         * @brief Save a un-versioned S3 object.
          *
          * @param request put object request
          * @param bucket S3 bucket
@@ -420,7 +413,7 @@ namespace AwsMock::Service {
         static void GetTopicNotificationConfigurations(Database::Entity::S3::Bucket &bucket, const std::vector<Dto::S3::TopicConfiguration> &topicConfigurations);
 
         /**
-         * A@brief dds the lambda notification configuration to the provided bucket.
+         * @brief Adds the lambda notification configuration to the provided bucket.
          *
          * @param bucket bucket entity.
          * @param lambdaConfigurations lambda notification configurations vector.

src/service/src/s3/S3Handler.cpp

@@ -377,8 +377,6 @@ namespace AwsMock::Service {
                         s3Request.max = std::stol(Core::StringUtils::Split(parts, '-')[1]);
                         log_info << "Requested multipart download range: " << std::to_string(s3Request.min) << "-" << std::to_string(s3Request.max);
                     }
-
-                    std::string range = Core::HttpUtils::GetHeaderValue(request, "x-amz-copy-source-range");
                     log_debug << "S3 multipart upload part copy: " << partNumber;
 
                     Dto::S3::UploadPartCopyResponse s3Response = _s3Service.UploadPartCopy(s3Request);