Skip to content
Permalink
Browse files

S3DownloadHandler: fix auth for requests with quoted paths/query params

  • Loading branch information
immerrr authored and dangra committed Feb 25, 2015
1 parent bb3a848 commit cdb9a0b2c00444456470a6b2302d4d2a8885b774
Showing with 19 additions and 2 deletions.
  1. +4 −2 scrapy/core/downloader/handlers/s3.py
  2. +15 −0 scrapy/tests/test_downloader_handlers.py
@@ -1,3 +1,5 @@
from urlparse import unquote

from scrapy import optional_features
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
@@ -54,8 +56,8 @@ def download_request(self, request, spider):
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=p.path,
query_args=p.query,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body)
httpreq = request.replace(url=url, headers=signed_headers)
@@ -379,6 +379,21 @@ def test_request_signing6(self):
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')

def test_request_signing7(self):
# ensure that spaces are quoted properly before signing
req = Request(
("s3://johnsmith/photos/my puppy.jpg"
"?response-content-disposition=my puppy.jpg"),
method='GET',
headers={
'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
})
httpreq = self.download_request(req, self.spider)
self.assertEqual(
httpreq.headers['Authorization'],
'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')


class FTPTestCase(unittest.TestCase):

username = "scrapy"

0 comments on commit cdb9a0b

Please sign in to comment.