Skip to content

Commit

Permalink
Cleanup leftover boto2 code in S3DownloaderHandler (#5209)
Browse files Browse the repository at this point in the history
S3DownloaderHandler.conn is a leftover attribute from 5e99758.
  • Loading branch information
laggardkernel committed Jul 27, 2021
1 parent b22a004 commit abe0b37
Showing 1 changed file with 1 addition and 13 deletions.
14 changes: 1 addition & 13 deletions scrapy/core/downloader/handlers/s3.py
@@ -1,5 +1,3 @@
from urllib.parse import unquote

from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore_available
Expand Down Expand Up @@ -59,7 +57,7 @@ def download_request(self, request, spider):
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
else:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
Expand All @@ -69,14 +67,4 @@ def download_request(self, request, spider):
self._signer.add_auth(awsrequest)
request = request.replace(
url=url, headers=awsrequest.headers.items())
else:
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body,
)
request = request.replace(url=url, headers=signed_headers)
return self._download_http(request, spider)

0 comments on commit abe0b37

Please sign in to comment.