From f6daa8940ce0cbd84e1ecf1f58d15fdcf9d8f9c5 Mon Sep 17 00:00:00 2001
From: Ethan Moore <github@proxyman.com>
Date: Wed, 1 Jun 2016 20:41:08 +0000
Subject: [PATCH] backport #33599 to 2016.3

---
 salt/utils/aws.py |  5 +++--
 salt/utils/s3.py  | 41 +++++++++++++++++++++++++++++++----------
 2 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/salt/utils/aws.py b/salt/utils/aws.py
index c4f26f04204b..fcfcfdf7a114 100644
--- a/salt/utils/aws.py
+++ b/salt/utils/aws.py
@@ -204,7 +204,7 @@ def assumed_creds(prov_dict, role_arn, location=None):
 def sig4(method, endpoint, params, prov_dict,
          aws_api_version=DEFAULT_AWS_API_VERSION, location=None,
          product='ec2', uri='/', requesturl=None, data='', headers=None,
-         role_arn=None):
+         role_arn=None, payload_hash=None):
     '''
     Sign a query against AWS services using Signature Version 4 Signing
     Process. This is documented at:
@@ -257,7 +257,8 @@ def sig4(method, endpoint, params, prov_dict,
 
     # Create payload hash (hash of the request body content). For GET
     # requests, the payload is an empty string ('').
-    payload_hash = hashlib.sha256(data).hexdigest()
+    if not payload_hash:
+        payload_hash = hashlib.sha256(data).hexdigest()
 
     # Combine elements to create create canonical request
     canonical_request = '\n'.join((
diff --git a/salt/utils/s3.py b/salt/utils/s3.py
index 436d4ea5378e..a242d31dc892 100644
--- a/salt/utils/s3.py
+++ b/salt/utils/s3.py
@@ -29,7 +29,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
           requesturl=None, return_url=False, bucket=None, service_url=None,
           path='', return_bin=False, action=None, local_file=None,
           verify_ssl=True, full_headers=False, kms_keyid=None,
-          location=None, role_arn=None):
+          location=None, role_arn=None, chunk_size=16384):
     '''
     Perform a query against an S3-like API. This function requires that a
     secret key and the id for that key are passed in. For instance:
@@ -97,10 +97,10 @@ def query(key, keyid, method='GET', params=None, headers=None,
         headers['x-amz-server-side-encryption-aws-kms-key-id'] = kms_keyid
 
     data = ''
+    payload_hash = None
     if method == 'PUT':
         if local_file:
-            with salt.utils.fopen(local_file, 'r') as ifile:
-                data = ifile.read()
+            payload_hash = salt.utils.get_hash(local_file, form='sha256')
 
     if not requesturl:
         requesturl = 'https://{0}/{1}'.format(endpoint, path)
@@ -116,6 +116,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             product='s3',
             requesturl=requesturl,
             headers=headers,
+            payload_hash=payload_hash,
         )
 
     log.debug('S3 Request: {0}'.format(requesturl))
@@ -125,12 +126,31 @@ def query(key, keyid, method='GET', params=None, headers=None,
     if not data:
         data = None
 
-    result = requests.request(method,
-                              requesturl,
-                              headers=headers,
-                              data=data,
-                              verify=verify_ssl)
-    response = result.content
+    response = None
+    if method == 'PUT':
+        if local_file:
+            with salt.utils.fopen(local_file, 'r') as data:
+                result = requests.request(method,
+                                          requesturl,
+                                          headers=headers,
+                                          data=data,
+                                          verify=verify_ssl,
+                                          stream=True)
+        response = result.content
+    elif method == 'GET' and not return_bin:
+        result = requests.request(method,
+                                  requesturl,
+                                  headers=headers,
+                                  data=data,
+                                  verify=verify_ssl,
+                                  stream=True)
+    else:
+        result = requests.request(method,
+                                  requesturl,
+                                  headers=headers,
+                                  data=data,
+                                  verify=verify_ssl)
+        response = result.content
     if result.status_code >= 400:
         # On error the S3 API response should contain error message
         log.debug('    Response content: {0}'.format(response))
@@ -175,7 +195,8 @@ def query(key, keyid, method='GET', params=None, headers=None,
     if local_file and method == 'GET':
         log.debug('Saving to local file: {0}'.format(local_file))
         with salt.utils.fopen(local_file, 'wb') as out:
-            out.write(response)
+            for chunk in result.iter_content(chunk_size=chunk_size):
+                out.write(chunk)
         return 'Saved to local file: {0}'.format(local_file)
 
     # This can be used to return a binary object wholesale