Move to AWS signature V4 (required for eu-central-1)

AUTHORS

@@ -17,3 +17,7 @@ Bugfixes, performance and other improvements.
 5. Takeshi Nakatani <ggtakec@gmail.com>
 
 Bugfixes, performance and other improvements.
+
+6. Wang, Shun <shunwang@amazon.com>
+
+Signature Version 4 Support.

src/common.h

@@ -74,6 +74,7 @@ extern std::string program_name;
 extern std::string service_path;
 extern std::string host;
 extern std::string bucket;
+extern std::string endpoint;
 extern std::string mount_prefix;
 
 #endif // S3FS_COMMON_H_

src/common_auth.cpp

@@ -102,4 +102,24 @@ string s3fs_md5sum(int fd, off_t start, ssize_t size)
   return string(md5);
 }
 
+string s3fs_sha256sum(int fd, off_t start, ssize_t size)
+{
+  size_t digestlen = get_sha256_digest_length();
+  char sha256[2 * digestlen + 1];
+  char hexbuf[3];
+  unsigned char* sha256hex;
+
+  if(NULL == (sha256hex = s3fs_sha256hexsum(fd, start, size))){
+    return string("");
+  }
+
+  memset(sha256, 0, 2 * digestlen + 1);
+  for(size_t pos = 0; pos < digestlen; pos++){
+    snprintf(hexbuf, 3, "%02x", sha256hex[pos]);
+    strncat(sha256, hexbuf, 2);
+  }
+  free(sha256hex);
+
+  return string(sha256);
+}
 /// END

src/curl.h

@@ -4,12 +4,12 @@
 //----------------------------------------------
 // class BodyData
 //----------------------------------------------
-// memory class for curl write memory callback 
+// memory class for curl write memory callback
 //
 class BodyData
 {
   private:
-    char* text;    
+    char* text;
     size_t lastpos;
     size_t bufsize;
 
@@ -110,7 +110,7 @@ typedef std::map<std::string, std::string> iamcredmap_t;
 //
 class S3fsCurl
 {
-    friend class S3fsMultiCurl;  
+    friend class S3fsMultiCurl;
 
   private:
     enum REQTYPE {
@@ -219,7 +219,12 @@ class S3fsCurl
     bool ResetHandle(void);
     bool RemakeHandle(void);
     bool ClearInternalData(void);
-    std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
+    std::string CalcSignaturev2(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
+    std::string CalcSignature(std::string method, std::string canonical_uri, std::string date2, std::string cononical_headers,
+                    std::string payload_hash, std::string signed_headers, std::string date3);
+    std::string CalcSignatureReal(std::string method, std::string canonical_uri, std::string query_string, std::string date2, std::string cononical_headers,
+                    std::string payload_hash, std::string signed_headers, std::string date3);
+
     bool GetUploadId(std::string& upload_id);
     int GetIAMCredentials(void);
 

src/openssl_auth.cpp

@@ -32,6 +32,7 @@
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
 #include <openssl/md5.h>
+#include <openssl/sha.h>
 #include <openssl/crypto.h>
 #include <openssl/err.h>
 #include <string>
@@ -189,7 +190,7 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
   if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){
     return false;
   }
-  HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen);
+  HMAC(EVP_sha256(), key, keylen, data, datalen, *digest, digestlen);
 
   return true;
 }
@@ -253,4 +254,82 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
   return result;
 }
 
+//-------------------------------------------------------------------
+// Utility Function for SHA256
+//-------------------------------------------------------------------
+size_t get_sha256_digest_length(void)
+{
+  return SHA256_DIGEST_LENGTH;
+}
+
+bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  (*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char);
+  if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){
+    return false;
+  }
+
+  const EVP_MD *md = EVP_get_digestbyname("sha256");
+  EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
+  EVP_DigestInit_ex(mdctx, md, NULL);
+  EVP_DigestUpdate(mdctx, data, datalen);
+  EVP_DigestFinal_ex(mdctx, *digest, digestlen);
+  EVP_MD_CTX_destroy(mdctx);
+
+  return true;
+}
+
+unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
+{
+
+  const EVP_MD *md = EVP_get_digestbyname("sha256");
+  EVP_MD_CTX *sha256ctx = EVP_MD_CTX_create();
+  EVP_DigestInit_ex(sha256ctx, md, NULL);
+
+  char buf[512];
+  ssize_t bytes;
+  unsigned char* result;
+
+  if(-1 == size) {
+    struct stat st;
+    if(-1 == fstat(fd, &st)){
+      return NULL;
+    }
+    size = static_cast<ssize_t>(st.st_size);
+  }
+
+  // seek to top of file.
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    return NULL;
+  }
+
+  memset(buf, 0, 512);
+  for(ssize_t total = 0; total < size; total += bytes){
+    bytes = 512 < (size - total) ? 512 : (size - total);
+    bytes = read(fd, buf, bytes);
+    if(0 == bytes){
+      // end of file
+      break;
+    }else if(-1 == bytes){
+      // error
+      DPRNNN("file read error(%d)", errno);
+      return NULL;
+    }
+    EVP_DigestUpdate(sha256ctx, buf, bytes);
+    memset(buf, 0, 512);
+  }
+  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+    return NULL;
+  }
+  EVP_DigestFinal_ex(sha256ctx, result, NULL);
+  EVP_MD_CTX_destroy(sha256ctx);
+
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    free(result);
+    return NULL;
+  }
+
+  return result;
+
+}
 /// END

src/s3fs.cpp

@@ -87,6 +87,7 @@ bool pathrequeststyle             = false;
 std::string program_name;
 std::string service_path          = "/";
 std::string host                  = "http://s3.amazonaws.com";
+std::string endpoint              = "us-east-1";
 std::string bucket                = "";
 
 //-------------------------------------------------------------------
@@ -2230,7 +2231,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter)
   if(delimiter && 0 < strlen(delimiter)){
     query += "delimiter=";
     query += delimiter;
-    query += "&";
+    query += "&max-keys=1000&";
   }
   query += "prefix=";
 
@@ -2241,12 +2242,12 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter)
   }else{
     query += urlEncode(s3_realpath.substr(1));
   }
-  query += "&max-keys=1000";
 
   while(truncated){
     string each_query = query;
     if(next_marker != ""){
-      each_query += "&marker=" + urlEncode(next_marker);
+      //each_query += "&marker=" + urlEncode(next_marker);
+      each_query.insert(each_query.find("&max-keys=1000&"),string("&marker=" + urlEncode(next_marker)));
       next_marker = "";
     }
     // request
@@ -3687,6 +3688,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       }
       return 0;
     }
+    if(0 == STR2NCMP(arg, "endpoint=")){
+      endpoint = strchr(arg, '=') + sizeof(char);
+      return 0;
+    }
     if(0 == strcmp(arg, "use_path_request_style")){
       pathrequeststyle = true;
       return 0;

src/s3fs_auth.h

@@ -10,6 +10,7 @@
 char* s3fs_base64(unsigned char* input, size_t length);
 std::string s3fs_get_content_md5(int fd);
 std::string s3fs_md5sum(int fd, off_t start, ssize_t size);
+std::string s3fs_sha256sum(int fd, off_t start, ssize_t size);
 
 //
 // in xxxxxx_auth.cpp
@@ -22,5 +23,8 @@ bool s3fs_destroy_crypt_mutex(void);
 bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen);
 size_t get_md5_digest_length(void);
 unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size);
+bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen);
+size_t get_sha256_digest_length(void);
+unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size);
 
 #endif // S3FS_AUTH_H_

src/s3fs_util.cpp

@@ -955,6 +955,9 @@ void show_help (void)
     "   url (default=\"http://s3.amazonaws.com\")\n"
     "      - sets the url to use to access amazon s3\n"
     "\n"
+    "   endpoint (default=\"us-east-1\")\n"
+    "      - sets the endpoint to use\n"
+    "\n"
     "   nomultipart (disable multipart uploads)\n"
     "\n"
     "   enable_content_md5 (default is disable)\n"

src/string_util.cpp

@@ -125,6 +125,41 @@ string urlEncode(const string &s)
   for (unsigned i = 0; i < s.length(); ++i) {
     if (s[i] == '/') { // Note- special case for fuse paths...
       result += s[i];
+    }else if (s[i] == '=') { // Note- special case for s3...
+      result += s[i];
+    }else if (s[i] == '&') { // Note- special case for s3...
+      result += s[i];
+    } else if (isalnum(s[i])) {
+      result += s[i];
+    } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
+      result += s[i];
+    } else if (s[i] == ' ') {
+      result += '%';
+      result += '2';
+      result += '0';
+    } else {
+      result += "%";
+      result += hexAlphabet[static_cast<unsigned char>(s[i]) / 16];
+      result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16];
+    }
+  }
+
+  return result;
+}
+
+/**
+ * urlEncode a fuse path,
+ * taking into special consideration "/",
+ * otherwise regular urlEncode.
+ */
+string urlEncode2(const string &s)
+{
+  string result;
+  for (unsigned i = 0; i < s.length(); ++i) {
+    if (s[i] == '=') { // Note- special case for fuse paths...
+      result += s[i];
+    }else if (s[i] == '&') { // Note- special case for s3...
+      result += s[i];
     } else if (isalnum(s[i])) {
       result += s[i];
     } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
@@ -217,3 +252,17 @@ string get_date()
   strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));
   return buf;
 }
+string get_date2()
+{
+  char buf[100];
+  time_t t = time(NULL);
+  strftime(buf, sizeof(buf), "%Y%m%d", gmtime(&t));
+  return buf;
+}
+string get_date3()
+{
+  char buf[100];
+  time_t t = time(NULL);
+  strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime(&t));
+  return buf;
+}

src/string_util.h

@@ -27,7 +27,10 @@ std::string trim(const std::string &s, const std::string &t = SPACES);
 std::string lower(std::string s);
 std::string IntToStr(int);
 std::string get_date();
+std::string get_date2();
+std::string get_date3();
 std::string urlEncode(const std::string &s);
+std::string urlEncode2(const std::string &s);
 std::string prepare_url(const char* url);
 bool get_keyword_value(std::string& target, const char* keyword, std::string& value);