Skip to content

Commit

Permalink
[s3] work around boto3 closing uploaded file (#1303)
Browse files Browse the repository at this point in the history
  • Loading branch information
jschneier committed Sep 17, 2023
1 parent d049794 commit 7d7bfdb
Showing 1 changed file with 10 additions and 14 deletions.
24 changes: 10 additions & 14 deletions storages/backends/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import warnings
from datetime import datetime
from datetime import timedelta
from tempfile import SpooledTemporaryFile
from urllib.parse import parse_qsl
from urllib.parse import urlencode
from urllib.parse import urlsplit
Expand Down Expand Up @@ -153,7 +152,7 @@ def closed(self):

def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
self._file = tempfile.SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".S3File",
dir=setting("FILE_UPLOAD_TEMP_DIR"),
Expand Down Expand Up @@ -478,7 +477,14 @@ def _save(self, name, content):
params["ContentEncoding"] = "gzip"

obj = self.bucket.Object(name)
obj.upload_fileobj(content, ExtraArgs=params, Config=self.transfer_config)

# Workaround file being closed errantly see: https://github.com/boto/s3transfer/issues/80
original_close = content.close
content.close = lambda: None
try:
obj.upload_fileobj(content, ExtraArgs=params, Config=self.transfer_config)
finally:
content.close = original_close
return cleaned_name

def delete(self, name):
Expand Down Expand Up @@ -649,14 +655,4 @@ class S3StaticStorage(S3Storage):


class S3ManifestStaticStorage(ManifestFilesMixin, S3StaticStorage):
"""Copy the file before saving for compatibility with ManifestFilesMixin
which does not play nicely with boto3 automatically closing the file.
See: https://github.com/boto/s3transfer/issues/80#issuecomment-562356142
"""

def _save(self, name, content):
content.seek(0)
with tempfile.SpooledTemporaryFile() as tmp:
tmp.write(content.read())
return super()._save(name, tmp)
"""Add ManifestFilesMixin with S3StaticStorage."""

0 comments on commit 7d7bfdb

Please sign in to comment.